--- loncom/homework/grades.pm	2004/04/29 07:57:47	1.193
+++ loncom/homework/grades.pm	2004/05/14 19:15:49	1.200
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # The LON-CAPA Grading handler
 #
-# $Id: grades.pm,v 1.193 2004/04/29 07:57:47 albertel Exp $
+# $Id: grades.pm,v 1.200 2004/05/14 19:15:49 albertel Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -1669,7 +1669,9 @@ KEYWORDS
 			    $partid.'</b> <font color="#999999">( ID '.$respid.
 			    ' )</font>&nbsp; &nbsp;';
 			if ($record{"resource.$partid.$respid.uploadedurl"}) {
-			    $lastsubonly.='<a href="'.&Apache::lonnet::tokenwrapper($record{"resource.$partid.$respid.uploadedurl"}).'" target="lonGRDs"><img src="/adm/lonIcons/unknown.gif" border=0"> File uploaded by student</a> <font color="red" size="1">Like all files provided by users, this file may contain virusses</font><br />';
+			    &Apache::lonnet::allowuploaded('/adm/grades',
+			      $record{"resource.$partid.$respid.uploadedurl"});
+			    $lastsubonly.='<a href="'.$record{"resource.$partid.$respid.uploadedurl"}.'" target="lonGRDs"><img src="/adm/lonIcons/unknown.gif" border=0"> File uploaded by student</a> <font color="red" size="1">Like all files provided by users, this file may contain virusses</font><br />';
 			}
 			$lastsubonly.='<b>Submitted Answer: </b>'.
 			    &cleanRecord($subval,$responsetype,$symb,$partid,
@@ -2049,11 +2051,11 @@ sub saveHandGrade {
 	    }
 	} elsif ($dropMenu eq 'reset status'
 		 && exists($record{'resource.'.$_.'.solved'})) { #don't bother if no old records -> no attempts
-	    $newrecord{'resource.'.$_.'.tries'} = 0;
-	    $newrecord{'resource.'.$_.'.solved'} = '';
-	    $newrecord{'resource.'.$_.'.award'} = '';
-	    $newrecord{'resource.'.$_.'.awarded'} = 0;
-	    $newrecord{'resource.'.$_.'.regrader'}="$ENV{'user.name'}:$ENV{'user.domain'}";
+	    foreach my $key (keys (%record)) {
+		if ($key=~/^resource\.\Q$_\E\./) { $newrecord{$key} = ''; }
+	    }
+	    $newrecord{'resource.'.$_.'.regrader'}=
+		"$ENV{'user.name'}:$ENV{'user.domain'}";
 	} elsif ($dropMenu eq '') {
 	    $pts = ($ENV{'form.GD_BOX'.$newflg.'_'.$_} ne '' ? 
 		    $ENV{'form.GD_BOX'.$newflg.'_'.$_} : 
@@ -3108,7 +3110,7 @@ sub displayPage {
 	'<td align="center"><b>&nbsp;Prob.&nbsp;</b></td>'.
 	'<td><b>&nbsp;'.($ENV{'form.vProb'} eq 'no' ? 'Title' : 'Problem Text').'/Grade</b></td></tr>';
 
-    my ($depth,$question) = (1,1);
+    my ($depth,$question,$prob) = (1,1,1);
     $iterator->next(); # skip the first BEGIN_MAP
     my $curRes = $iterator->next(); # for "current resource"
     while ($depth > 0) {
@@ -3119,7 +3121,7 @@ sub displayPage {
 	    my $parts = $curRes->parts();
             my $title = $curRes->compTitle();
 	    my $symbx = $curRes->symb();
-	    $studentTable.='<tr bgcolor="#ffffe6"><td align="center" valign="top" >'.$question.
+	    $studentTable.='<tr bgcolor="#ffffe6"><td align="center" valign="top" >'.$prob.
 		(scalar(@{$parts}) == 1 ? '' : '<br>('.scalar(@{$parts}).'&nbsp;parts)').'</td>';
 	    $studentTable.='<td valign="top">';
 	    if ($ENV{'form.vProb'} eq 'yes' ) {
@@ -3169,6 +3171,7 @@ sub displayPage {
 		    $studentTable.='<input type="hidden" name="q_'.$question.'" value="'.$partid.'" />'."\n";
 		    $question++;
 		}
+		$prob++;
 	    }
 	    $studentTable.='</td></tr>';
 
@@ -3211,7 +3214,8 @@ sub displaySubByDates {
 	    my @matchKey = sort(grep /^resource\.\Q$partid\E\..*?\.submission$/,@versionKeys);
 #	    next if ($$record{"$version:resource.$partid.solved"} eq '');
 	    foreach my $matchKey (@matchKey) {
-		if (exists $$record{$version.':'.$matchKey}) {
+		if (exists($$record{$version.':'.$matchKey}) &&
+		    $$record{$version.':'.$matchKey} ne '') {
 		    my ($responseId)=($matchKey=~ /^resource\.\Q$partid\E\.(.*?)\.submission$/);
 		    $displaySub[0].='<b>Part&nbsp;'.$partid.'&nbsp;';
 		    $displaySub[0].='<font color="#999999">(ID&nbsp;'.
@@ -3295,7 +3299,7 @@ sub updateGradeByPage {
 
     $iterator->next(); # skip the first BEGIN_MAP
     my $curRes = $iterator->next(); # for "current resource"
-    my ($depth,$question,$changeflag)= (1,1,0);
+    my ($depth,$question,$prob,$changeflag)= (1,1,1,0);
     while ($depth > 0) {
         if($curRes == $iterator->BEGIN_MAP) { $depth++; }
         if($curRes == $iterator->END_MAP) { $depth--; }
@@ -3304,7 +3308,7 @@ sub updateGradeByPage {
 	    my $parts = $curRes->parts();
             my $title = $curRes->compTitle();
 	    my $symbx = $curRes->symb();
-	    $studentTable.='<tr bgcolor="#ffffe6"><td align="center" valign="top" >'.$question.
+	    $studentTable.='<tr bgcolor="#ffffe6"><td align="center" valign="top" >'.$prob.
 		(scalar(@{$parts}) == 1 ? '' : '<br>('.scalar(@{$parts}).'&nbsp;parts)').'</td>';
 	    $studentTable.='<td valign="top">&nbsp;<b>'.$title.'</b>&nbsp;</td>';
 
@@ -3365,6 +3369,7 @@ sub updateGradeByPage {
 		'<td valign="top">'.$displayPts[1].'</td>'.
 		'</tr>';
 
+	    $prob++;
 	}
         $curRes = $iterator->next();
     }
@@ -3517,8 +3522,8 @@ sub scantron_selectphase {
           <tr bgcolor="#ffffe6">
 	    <td> Options: </td>
             <td>
-                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only skipped records <br />
-                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove any exisiting corrections
+                <input type="checkbox" name="scantron_options_redo" value="redo_skipped"/> Do only previously skipped records <br />
+                <input type="checkbox" name="scantron_options_ignore" value="ignore_corrections"/> Remove all exisiting corrections
 	    </td>
           </tr>
           <tr bgcolor="#ffffe6">
@@ -3729,7 +3734,7 @@ sub scan_data {
 }
 
 sub scantron_parse_scanline {
-    my ($line,$whichline,$scantron_config,$scan_data,$justCODE)=@_;
+    my ($line,$whichline,$scantron_config,$scan_data,$justHeader)=@_;
     my %record;
     my $questions=substr($line,$$scantron_config{'Qstart'}-1);
     my $data=substr($line,0,$$scantron_config{'Qstart'}-1);
@@ -3748,7 +3753,6 @@ sub scantron_parse_scanline {
 	    #FIXME interpret first N questions
 	}
     }
-    if ($justCODE) { return \%record; }
     $record{'scantron.ID'}=substr($data,$$scantron_config{'IDstart'}-1,
 				  $$scantron_config{'IDlength'});
     $record{'scantron.PaperID'}=
@@ -3760,6 +3764,8 @@ sub scantron_parse_scanline {
     $record{'scantron.LastName'}=
 	substr($data,$$scantron_config{'LastName'}-1,
 	       $$scantron_config{'LastNamelength'});
+    if ($justHeader) { return \%record; }
+
     my @alphabet=('A'..'Z');
     my $questnum=0;
     while ($questions) {
@@ -3827,7 +3833,7 @@ sub scantron_process_corrections {
     my ($scanlines,$scan_data)=&scantron_getfile();
     my $classlist=&Apache::loncoursedata::get_classlist();
     my $which=$ENV{'form.scantron_line'};
-    my $line=&scantron_get_line($scanlines,$which);
+    my $line=&scantron_get_line($scanlines,$scan_data,$which);
     my ($skip,$err,$errmsg);
     if ($ENV{'form.scantron_skip_record'}) {
 	$skip=1;
@@ -3850,6 +3856,8 @@ sub scantron_process_corrections {
 	    $newCODE=$ENV{'form.scantron_CODE_selectedvalue'};
 	} elsif ($resolution eq 'use_typed') {
 	    $newCODE=$ENV{'form.scantron_CODE_newvalue'};
+	} elsif ($resolution =~ /^use_closest_(\d+)/) {
+	    $newCODE=$ENV{"form.scantron_CODE_closest_$1"};
 	}
 	if ($ENV{'form.scantron_corrections'} eq 'duplicateCODE') {
 	    $args{'CODE_ignore_dup'}=1;
@@ -3871,25 +3879,79 @@ sub scantron_process_corrections {
     if ($err) {
 	$r->print("Unable to accept last correction, an error occurred :$errmsg:");
     } else {
-	&scantron_put_line($scanlines,$which,$line,$skip);
+	&scantron_put_line($scanlines,$scan_data,$which,$line,$skip);
 	&scantron_putfile($scanlines,$scan_data);
     }
 }
 
+sub reset_skipping_status {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    &scan_data($scan_data,'remember_skipping',undef,1);
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub allow_skipping {
+    my ($scan_data,$i)=@_;
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    delete($remembered{$i});
+    &scan_data($scan_data,'remember_skipping',join(':',%remembered));
+}
+
+sub should_be_skipped {
+    my ($scan_data,$i)=@_;
+    if ($ENV{'form.scantron_options_redo'} !~ /^redo_/) {
+	# not redoing old skips
+	return 0;
+    }
+    my %remembered=split(':',&scan_data($scan_data,'remember_skipping'));
+    if (exists($remembered{$i})) { return 0; }
+    return 1;
+}
+
+sub remember_current_skipped {
+    my ($scanlines,$scan_data)=&scantron_getfile();
+    my %to_remember;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	if ($scanlines->{'skipped'}[$i]) {
+	    $to_remember{$i}=1;
+	}
+    }
+    &Apache::lonnet::logthis('remembering '.join(':',%to_remember));
+    &scan_data($scan_data,'remember_skipping',join(':',%to_remember));
+    &scantron_putfile(undef,$scan_data);
+}
+
+sub check_for_error {
+    my ($r,$result)=@_;
+    if ($result ne 'ok' && $result ne 'not_found' ) {
+	$r->print("An error occured ($result) when trying to Remove the existing corrections.");
+    }
+}
 
 sub scantron_validate_file {
     my ($r) = @_;
     my ($symb,$url)=&get_symb_and_url($r);
     if (!$symb) {return '';}
     my $default_form_data=&defaultFormData($symb,$url);
+    
+    # do the detection of only doing skipped records first befroe we delete
+    # them  when doing the corrections reset
+    if ($ENV{'form.scantron_options_redo'} ne 'redo_skipped_ready') {
+	&reset_skipping_status();
+    }
+    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped') {
+	&remember_current_skipped();
+	&scantron_remove_file('skipped');
+	$ENV{'form.scantron_options_redo'}='redo_skipped_ready';
+    }
+
     if ($ENV{'form.scantron_options_ignore'} eq 'ignore_corrections') {
-	my $result=&scantron_remove('corrected');
-	&Apache::lonnet::logthis("result was $result");
-	if ($result ne 'ok' && $result ne 'not_found' ) {
-	    $r->print("An error occured ($result) when trying to Remove the existing corrections.");
-	}
+	&check_for_error($r,&scantron_remove_file('corrected'));
+	&check_for_error($r,&scantron_remove_file('skipped'));
+	&check_for_error($r,&scantron_remove_scan_data());
 	$ENV{'form.scantron_options_ignore'}='done';
     }
+
     if ($ENV{'form.scantron_corrections'}) {
 	&scantron_process_corrections($r);
     }
@@ -3916,14 +3978,10 @@ SCANTRONFORM
 			  'doublebubble',
 			  'missingbubbles');
     if (!$ENV{'form.validatepass'}) {
-	$ENV{'form.valiadatepass'} = 0;
+	$ENV{'form.validatepass'} = 0;
     }
-    my $currentphase=$ENV{'form.valiadatepass'};
+    my $currentphase=$ENV{'form.validatepass'};
 
-    if ($ENV{'form.scantron_selectfile'}=~m-^/-) {
-	#first pass copy file to classdir
-	
-    }
     my $stop=0;
     while (!$stop && $currentphase < scalar(@validate_phases)) {
 	$r->print("<p> Validating ".$validate_phases[$currentphase]."</p>");
@@ -3953,40 +4011,44 @@ SCANTRONFORM
     return '';
 }
 
-sub scantron_remove {
+sub scantron_remove_file {
     my ($which)=@_;
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $file='scantron_';
-    if ($which eq 'corrected') {
-	$file.='corrected_';
+    if ($which eq 'corrected' || $which eq 'skipped') {
+	$file.=$which.'_';
     } else {
 	return 'refused';
     }
     $file.=$ENV{'form.scantron_selectfile'};
-    &Apache::lonnet::logthis("removeing $file");
-    my $result=&Apache::lonnet::removeuserfile($cname,$cdom,$file);
+    return &Apache::lonnet::removeuserfile($cname,$cdom,$file);
+}
+
+sub scantron_remove_scan_data {
+    my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my @keys=&Apache::lonnet::getkeys('nohist_scantrondata',$cdom,$cname);
-    &Apache::lonnet::logthis('got keys '.join(':',@keys));
-    &Apache::lonnet::logthis("cdom $cdom cname $cname");
     my @todelete;
     my $filename=$ENV{'form.scantron_selectfile'};
-    &Apache::lonnet::logthis('filename '.$filename);
     foreach my $key (@keys) {
 	if ($key=~/^\Q$filename\E_/) {
+	    if ($ENV{'form.scantron_options_redo'} eq 'redo_skipped_ready' &&
+		$key=~/remember_skipping/) {
+		next;
+	    }
 	    push(@todelete,$key);
 	}
     }
-    &Apache::lonnet::logthis('todelete '.join(':',@todelete));
+    my $result;
     if (@todelete) {
-	&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
+	$result=&Apache::lonnet::del('nohist_scantrondata',\@todelete,$cdom,$cname);
     }
     return $result;
 }
 
 sub scantron_getfile {
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
     my $lines;
@@ -4029,34 +4091,48 @@ sub lonnet_putfile {
 
 sub scantron_putfile {
     my ($scanlines,$scan_data) = @_;
-    #FIXME really would prefer a scantron directory but tokenwrapper
-    # doesn't allow access to subdirs of userfiles
+    #FIXME really would prefer a scantron directory
     my $cname=$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
     my $cdom=$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
-    my $prefix='scantron_';
+    if ($scanlines) {
+	my $prefix='scantron_';
 # no need to update orig, shouldn't change
 #   &lonnet_putfile(join("\n",@{$scanlines->{'orig'}}),$prefix.'orig_'.
 #		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
-		    $prefix.'corrected_'.
-		    $ENV{'form.scantron_selectfile'});
-    &lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
-		    $prefix.'skipped_'.
-		    $ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'corrected'}}),
+			$prefix.'corrected_'.
+			$ENV{'form.scantron_selectfile'});
+	&lonnet_putfile(join("\n",@{$scanlines->{'skipped'}}),
+			$prefix.'skipped_'.
+			$ENV{'form.scantron_selectfile'});
+    }
     &Apache::lonnet::put('nohist_scantrondata',$scan_data,$cdom,$cname);
 }
 
 sub scantron_get_line {
-    my ($scanlines,$i)=@_;
-    if ($scanlines->{'skipped'}[$i]) {return undef;}
+    my ($scanlines,$scan_data,$i)=@_;
+    if (&should_be_skipped($scan_data,$i)) { return undef; }
+    if ($scanlines->{'skipped'}[$i]) { return undef; }
     if ($scanlines->{'corrected'}[$i]) {return $scanlines->{'corrected'}[$i];}
     return $scanlines->{'orig'}[$i]; 
 }
 
+sub get_todo_count {
+    my ($scanlines,$scan_data)=@_;
+    my $count=0;
+    for (my $i=0;$i<=$scanlines->{'count'};$i++) {
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
+	if ($line=~/^[\s\cz]*$/) { next; }
+	$count++;
+    }
+    return $count;
+}
+
 sub scantron_put_line {
-    my ($scanlines,$i,$newline,$skip)=@_;
+    my ($scanlines,$scan_data,$i,$newline,$skip)=@_;
     if ($skip) {
 	$scanlines->{'skipped'}[$i]=$newline;
+	&allow_skipping($scan_data,$i);
 	return;
     }
     $scanlines->{'corrected'}[$i]=$newline;
@@ -4075,7 +4151,7 @@ sub scantron_validate_ID {
 
     my %found=('ids'=>{},'usernames'=>{});
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4089,11 +4165,11 @@ sub scantron_validate_ID {
 	    if ($found{'ids'}{$found}) {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'duplicateID',$found);
-		return(1);
+		return(1,$currentphase);
 	    } elsif ($found{'usernames'}{$username}) {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'duplicateID',$username);
-		return(1);
+		return(1,$currentphase);
 	    }
 	    #FIXME store away line we previously saw the ID on to use above
 	    $found{'ids'}{$found}++;
@@ -4105,18 +4181,18 @@ sub scantron_validate_ID {
 		    &scantron_get_correction($r,$i,$scan_record,
 					     \%scantron_config,
 					     $line,'duplicateID',$username);
-		    return(1);
+		    return(1,$currentphase);
 		} elsif (!defined($username)) {
 		    &scantron_get_correction($r,$i,$scan_record,
 					     \%scantron_config,
 					     $line,'incorrectID');
-		    return(1);
+		    return(1,$currentphase);
 		}
 		$found{'usernames'}{$username}++;
 	    } else {
 		&scantron_get_correction($r,$i,$scan_record,\%scantron_config,
 					 $line,'incorrectID');
-		return(1);
+		return(1,$currentphase);
 	    }
 	}
     }
@@ -4167,7 +4243,7 @@ sub scantron_get_correction {
 	if ($error eq 'incorrectCODE') {
 	    $r->print("</p><p>The encoded CODE is not in the list of possible CODEs</p>\n");
 	} elsif ($error eq 'duplicateCODE') {
-	    $r->print("</p><p>The encoded CODE has also been used by a previous paper $arg, and CODEs were supposed to be unique</p>\n");
+	    $r->print("</p><p>The encoded CODE has also been used by a previous paper ".join(', ',@{$arg}).", and CODEs are supposed to be unique</p>\n");
 	}
 	$r->print("<p>The CODE on the form is  <tt>".
 		  $$scan_record{'scantron.CODE'}."</tt><br />\n");
@@ -4178,8 +4254,21 @@ sub scantron_get_correction {
 		  $$scan_record{'scantron.FirstName'}."</p>");
 	$r->print("<p>How should I handle this? <br /> \n");
 	$r->print("\n<br /> ");
-	$r->print("<input type='radio' name='scantron_CODE_resolution' value='use_unfound' checked='on' /> Use the CODE <b><tt>".$$scan_record{'scantron.CODE'}."</tt></b> that is was on the paper, ignoring the error.");
+	my $i=0;
+	if ($error eq 'incorrectCODE') {
+	    my ($max,$closest)=&scantron_get_closely_matching_CODEs($arg,$$scan_record{'scantron.CODE'});
+	    foreach my $testcode (@{$closest}) {
+		my $checked='';
+		if (!$i) { $checked=' checked="on" '; }
+		$r->print("<input type='radio' name='scantron_CODE_resolution' value='use_closest_$i' $checked /> Use the similar CODE <b><tt>".$testcode."</tt></b> instead.<input type='hidden' name='scantron_CODE_closest_$i' value='$testcode' />");
+		$r->print("\n<br />");
+		$i++;
+	    }
+	}
+	my $checked; if (!$i) { $checked=' checked="on" '; }
+	$r->print("<input type='radio' name='scantron_CODE_resolution' value='use_unfound' $checked /> Use the CODE <b><tt>".$$scan_record{'scantron.CODE'}."</tt></b> that is was on the paper, ignoring the error.");
 	$r->print("\n<br />");
+
 	$r->print(<<ENDSCRIPT);
 <script type="text/javascript">
 function change_radio(field) {
@@ -4202,7 +4291,6 @@ ENDSCRIPT
 	$r->print("<input type='radio' name='scantron_CODE_resolution' value='use_typed' /> Use <input type='text' size='8' name='scantron_CODE_newvalue' onfocus=\"javascript:change_radio('use_typed')\" onkeypress=\"javascript:change_radio('use_typed')\" /> as the CODE.");
 	$r->print("\n<br /><br />");
     } elsif ($error eq 'doublebubble') {
-#FIXME Need to print out who this is along with the paper info
 	$r->print("<p>There have been multiple bubbles scanned for a some question(s)</p>\n");
 	$r->print('<input type="hidden" name="scantron_questions" value="'.
 		  join(',',@{$arg}).'" />');
@@ -4249,9 +4337,38 @@ sub scantron_bubble_selector {
     $r->print('</tr></table>');
 }
 
+sub num_matches {
+    my ($orig,$code) = @_;
+    my @code=split(//,$code);
+    my @orig=split(//,$orig);
+    my $same=0;
+    for (my $i=0;$i<scalar(@code);$i++) {
+	if ($code[$i] eq $orig[$i]) { $same++; }
+    }
+    return $same;
+}
+
+sub scantron_get_closely_matching_CODEs {
+    my ($allcodes,$CODE)=@_;
+    my @CODEs;
+    foreach my $testcode (sort(keys(%{$allcodes}))) {
+	push(@{$CODEs[&num_matches($CODE,$testcode)]},$testcode);
+    }
+
+    return ($#CODEs,$CODEs[-1]);
+}
+
+sub get_codes {
+    my $old_name=$ENV{'form.scantron_CODElist'};
+    my $cdom =$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
+    my $cnum =$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
+    my %result=&Apache::lonnet::get('CODEs',[$old_name],$cdom,$cnum);
+    my %allcodes=map {(&Apache::lonprintout::num_to_letters($_),1)} split(',',$result{$old_name});
+    return %allcodes;
+}
+
 sub scantron_validate_CODE {
     my ($r,$currentphase) = @_;
-    #FIXME doesn't do anything yet
     my %scantron_config=&get_scantron_config($ENV{'form.scantron_format'});
     if ($scantron_config{'CODElocation'} &&
 	$scantron_config{'CODEstart'} &&
@@ -4265,15 +4382,11 @@ sub scantron_validate_CODE {
     
     my %usedCODEs;
 
-    my $old_name=$ENV{'form.scantron_CODElist'};
-    my $cdom =$ENV{'course.'.$ENV{'request.course.id'}.'.domain'};
-    my $cnum =$ENV{'course.'.$ENV{'request.course.id'}.'.num'};
-    my %result=&Apache::lonnet::get('CODEs',[$old_name],$cdom,$cnum);
-    my %allcodes=map {(&Apache::lonprintout::num_to_letters($_),1)} split(',',$result{$old_name});
+    my %allcodes=&get_codes();
 
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4282,17 +4395,17 @@ sub scantron_validate_CODE {
 	if (!exists($allcodes{$CODE}) && !$$scan_record{'scantron.useCODE'}) {
 	    &scantron_get_correction($r,$i,$scan_record,
 				     \%scantron_config,
-				     $line,'incorrectCODE',$CODE);
-	    return(1);
+				     $line,'incorrectCODE',\%allcodes);
+	    return(1,$currentphase);
 	}
 	if (exists($usedCODEs{$CODE}) && $ENV{'form.scantron_CODEunique'}
 	    && !$$scan_record{'scantron.CODE_ignore_dup'}) {
 	    &scantron_get_correction($r,$i,$scan_record,
 				     \%scantron_config,
-				     $line,'duplicateCODE',$CODE);
-	    return(1);
+				     $line,'duplicateCODE',$usedCODEs{$CODE});
+	    return(1,$currentphase);
 	}
-	$usedCODEs{$CODE}++;
+	push (@{$usedCODEs{$CODE}},$$scan_record{'scantron.PaperID'});
     }
     return (0,$currentphase+1);
 }
@@ -4307,7 +4420,7 @@ sub scantron_validate_doublebubble {
     my %scantron_config=&get_scantron_config($ENV{'form.scantron_format'});
     my ($scanlines,$scan_data)=&scantron_getfile();
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4357,7 +4470,7 @@ sub scantron_validate_missingbubbles {
     my $max_bubble=&scantron_get_maxbubble();
     if (!$max_bubble) { $max_bubble=2**31; }
     for (my $i=0;$i<=$scanlines->{'count'};$i++) {
-	my $line=&scantron_get_line($scanlines,$i);
+	my $line=&scantron_get_line($scanlines,$scan_data,$i);
 	if ($line=~/^[\s\cz]*$/) { next; }
 	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
 						 $scan_data);
@@ -4402,18 +4515,25 @@ SCANTRONFORM
     my @delayqueue;
     my %completedstudents;
     
+    my $count=&get_todo_count($scanlines,$scan_data);
     my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin($r,'Scantron Status',
- 				    'Scantron Progress',$scanlines->{'count'});
+ 				    'Scantron Progress',$count,
+				    'inline',undef,'scantronupload');
     &Apache::lonhtmlcommon::Update_PrgWin($r,\%prog_state,
 					  'Processing first student');
     my $start=&Time::HiRes::time();
     my $i=-1;
-    my ($uname,$udom);
+    my ($uname,$udom,$started);
     while ($i<$scanlines->{'count'}) {
  	($uname,$udom)=('','');
  	$i++;
- 	my $line=&scantron_get_line($scanlines,$i);
+ 	my $line=&scantron_get_line($scanlines,$scan_data,$i);
  	if ($line=~/^[\s\cz]*$/) { next; }
+	if ($started) {
+	    &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+						     'last student');
+	}
+	$started=1;
  	my $scan_record=&scantron_parse_scanline($line,$i,\%scantron_config,
  						 $scan_data);
  	unless ($uname=&scantron_find_student($scan_record,$scan_data,
@@ -4451,15 +4571,13 @@ SCANTRONFORM
     } continue {
 	&Apache::lonnet::delenv('form.counter');
 	&Apache::lonnet::delenv('scantron\.');
-	&Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
-						 'last student');
     }
     &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
 #    my $lasttime = &Time::HiRes::time()-$start;
 #    $r->print("<p>took $lasttime</p>");
 
     $navmap->untieHashes();
-    $r->print("</form><p>Done</p>");
+    $r->print("</form>");
     $r->print(&show_grading_menu_form($symb,$url));
     return '';
 }