--- loncom/interface/loncoursedata.pm	2002/08/28 22:02:47	1.23
+++ loncom/interface/loncoursedata.pm	2002/08/31 18:31:15	1.27
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # (Publication Handler
 #
-# $Id: loncoursedata.pm,v 1.23 2002/08/28 22:02:47 stredwic Exp $
+# $Id: loncoursedata.pm,v 1.27 2002/08/31 18:31:15 stredwic Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -693,11 +693,22 @@ sub ProcessStudentData {
         return;
     }
 
+    if(defined($cache->{$name.':keys'})) {
+	foreach (split(':::', $cache->{$name.':keys'})) {
+	    delete $cache->{$name.':'.$_};
+	}
+    }
+
+    my %courseKeys;
     # user name:domain was prepended earlier in DownloadCourseInformation
     foreach (keys %$courseData) {
+	my $currentKey =~ s/^$name//;
+	$courseKeys{$currentKey}++;
         $cache->{$_}=$courseData->{$_};
     }
 
+    $cache->{$name.':keys'} = join(':::', keys(%courseKeys));
+
     return;
 }
 
@@ -753,6 +764,13 @@ sub ExtractStudentData {
         return;
     }
 
+    my %allkeys;
+    if(defined($output->{$name.':keys'})) {
+	foreach (split(':::', $output->{$name.':keys'})) {
+	    delete $output->{$name.':'.$_};
+	}
+    }
+
     my ($username,$domain)=split(':',$name);
 
     my $Version;
@@ -768,6 +786,8 @@ sub ExtractStudentData {
 
             # Output dashes for all the parts of this problem if there
             # is no version information about the current problem.
+            $output->{$name.':'.$problemID.':NoVersion'} = 'false';
+            $allkeys{$name.':'.$problemID.':NoVersion'}++;
             if(!$LatestVersion) {
                 foreach my $part (split(/\:/,$data->{$sequence.':'.
                                                       $problemID.
@@ -775,6 +795,9 @@ sub ExtractStudentData {
                     $output->{$name.':'.$problemID.':'.$part.':tries'} = 0;
                     $output->{$name.':'.$problemID.':'.$part.':awarded'} = 0;
                     $output->{$name.':'.$problemID.':'.$part.':code'} = ' ';
+		    $allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
+		    $allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
+		    $allkeys{$name.':'.$problemID.':'.$part.':code'}++;
                     $totalProblems++;
                 }
                 $output->{$name.':'.$problemID.':NoVersion'} = 'true';
@@ -860,6 +883,7 @@ sub ExtractStudentData {
                                                  ':parts'})) {
                 $output->{$name.':'.$problemID.':'.$part.':wrong'} = 
                     $partData{$part.':tries'};
+		$allkeys{$name.':'.$problemID.':'.$part.':wrong'}++;
 
                 if($partData{$part.':code'} eq '*') {
                     $output->{$name.':'.$problemID.':'.$part.':wrong'}--;
@@ -875,14 +899,22 @@ sub ExtractStudentData {
                     $partData{$part.':code'};
                 $output->{$name.':'.$problemID.':'.$part.':awarded'} =
                     $partData{$part.':awarded'};
+		$allkeys{$name.':'.$problemID.':'.$part.':tries'}++;
+		$allkeys{$name.':'.$problemID.':'.$part.':code'}++;
+		$allkeys{$name.':'.$problemID.':'.$part.':awarded'}++;
+
                 $totalAwarded += $partData{$part.':awarded'};
                 $output->{$name.':'.$problemID.':'.$part.':timestamp'} =
                     $partData{$part.':timestamp'};
+		$allkeys{$name.':'.$problemID.':'.$part.':timestamp'}++;
+
                 foreach my $response (split(':', $data->{$sequence.':'.
                                                          $problemID.':'.
                                                          $part.':responseIDs'})) {
                     $output->{$name.':'.$problemID.':'.$part.':'.$response.
                               ':submission'}=join(':::',@submissions);
+		    $allkeys{$name.':'.$problemID.':'.$part.':'.$response.
+			     ':submission'}++;
                 }
 
                 if($partData{$part.':code'} ne 'x') {
@@ -892,6 +924,7 @@ sub ExtractStudentData {
         }
 
         $output->{$name.':'.$sequence.':problemsCorrect'} = $problemsCorrect;
+	$allkeys{$name.':'.$sequence.':problemsCorrect'}++;
         $problemsSolved += $problemsCorrect;
 	$problemsCorrect=0;
     }
@@ -899,6 +932,11 @@ sub ExtractStudentData {
     $output->{$name.':problemsSolved'} = $problemsSolved;
     $output->{$name.':totalProblems'} = $totalProblems;
     $output->{$name.':totalAwarded'} = $totalAwarded;
+    $allkeys{$name.':problemsSolved'}++;
+    $allkeys{$name.':totalProblems'}++;
+    $allkeys{$name.':totalAwarded'}++;
+
+    $output->{$name.':keys'} = join(':::', keys(%allkeys));
 
     return;
 }
@@ -1095,8 +1133,6 @@ sub DownloadStudentCourseData {
     my $title = 'LON-CAPA Statistics';
     my $heading = 'Download and Process Course Data';
     my $studentCount = scalar(@$students);
-    my %cache;
-
 
     my $WhatIWant;
     $WhatIWant = '(^version:|';
@@ -1113,6 +1149,8 @@ sub DownloadStudentCourseData {
     my $displayString;
     my $count=0;
     foreach (@$students) {
+        my %cache;
+
         if($c->aborted()) { return 'Aborted'; }
 
         if($status eq 'true') {
@@ -1130,27 +1168,28 @@ sub DownloadStudentCourseData {
 
         if($c->aborted()) { return 'Aborted'; }
 
-        if($downloadTime eq 'Not downloaded') {
-            my $courseData = 
-                &DownloadCourseInformation($_, $courseID, $downloadTime, 
-                                           $WhatIWant);
-            if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
-                foreach my $key (keys(%$courseData)) {
-                    if($key =~ /^(con_lost|error|no_such_host)/i) {
-                        $courseData->{$_.':error'} = 'No course data for '.$_;
-                        last;
-                    }
-                }
-                if($extract eq 'true') {
-                    &ExtractStudentData($courseData, \%cache, \%cache, $_);
-                } else {
-                    &ProcessStudentData(\%cache, $courseData, $_);
-                }
-                untie(%cache);
-            } else {
-                next;
-            }
-        }
+        #if($downloadTime ne 'Not downloaded') {
+	#    next;
+	#}
+	my $courseData = 
+	    &DownloadCourseInformation($_, $courseID, $downloadTime, 
+				       $WhatIWant);
+	if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
+	    foreach my $key (keys(%$courseData)) {
+		if($key =~ /^(con_lost|error|no_such_host)/i) {
+		    $courseData->{$_.':error'} = 'No course data for '.$_;
+		    last;
+		}
+	    }
+	    if($extract eq 'true') {
+		&ExtractStudentData($courseData, \%cache, \%cache, $_);
+	    } else {
+		&ProcessStudentData(\%cache, $courseData, $_);
+	    }
+	    untie(%cache);
+	} else {
+	    next;
+	}
     }
     if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); }
 
@@ -1172,8 +1211,6 @@ sub DownloadStudentCourseDataSeparate {
 
     &CheckForResidualDownload($courseID, $cacheDB, $students, $c);
 
-    my %cache;
-
     my $studentCount = scalar(@$students);
     if($status eq 'true') {
         &Apache::lonhtmlcommon::Create_PrgWin($r, $title, $heading);
@@ -1191,6 +1228,7 @@ sub DownloadStudentCourseDataSeparate {
             &Apache::lonhtmlcommon::Update_PrgWin($displayString, $r);
         }
 
+        my %cache;
         my $downloadTime='Not downloaded';
         if($checkDate eq 'true'  && 
            tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
@@ -1202,7 +1240,7 @@ sub DownloadStudentCourseDataSeparate {
             return 'Aborted';
         }
 
-        if($downloadTime eq 'Not downloaded') {
+        #if($downloadTime eq 'Not downloaded') {
             my $error = 0;
             my $courseData = 
                 &DownloadCourseInformation($_, $courseID, $downloadTime,
@@ -1226,7 +1264,7 @@ sub DownloadStudentCourseDataSeparate {
                 $downloadData{$_.':error'} = 'No course data for '.$_;
             }
             untie(%downloadData);
-        }
+        #}
     }
     if($status eq 'true') { &Apache::lonhtmlcommon::Close_PrgWin($r); }