--- loncom/interface/lonstatistics.pm	2002/07/26 16:22:09	1.33
+++ loncom/interface/lonstatistics.pm	2002/08/13 15:01:07	1.43
@@ -1,7 +1,7 @@
 # The LearningOnline Network with CAPA
 # (Publication Handler
 #
-# $Id: lonstatistics.pm,v 1.33 2002/07/26 16:22:09 stredwic Exp $
+# $Id: lonstatistics.pm,v 1.43 2002/08/13 15:01:07 stredwic Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -31,7 +31,7 @@
 # 11/1,11/4,11/16,12/14,12/16,12/18,12/20,12/31 Behrouz Minaei
 # YEAR=2002
 # 1/22,2/1,2/6,2/25,3/2,3/6,3/17,3/21,3/22,3/26,4/7,5/6 Behrouz Minaei
-# 5/12,5/14,5/15,5/19,5/26,7/16,25/7  Behrouz Minaei
+# 5/12,5/14,5/15,5/19,5/26,7/16,25/7,29/7  Behrouz Minaei
 #
 ###
 
@@ -47,7 +47,6 @@ use Apache::lonhtmlcommon;
 use Apache::lonproblemanalysis;
 use Apache::lonproblemstatistics;
 use Apache::lonstudentassessment;
-use Apache::lonchart;
 use HTML::TokeParser;
 use GDBM_File;
 
@@ -70,8 +69,10 @@ sub ProcessFormData{
     $cache->{'reportKey'} = 'false';
 
     &Apache::loncommon::get_unprocessed_cgi($ENV{'QUERY_STRING'},
-                                            ['sort','download','reportSelected',
-                                             'StudentAssessmentStudent']);
+                                            ['sort','download',
+                                             'reportSelected',
+                                             'StudentAssessmentStudent',
+                                             'ProblemStatisticsSort']);
     &CheckFormElement($cache, 'Status', 'Status', 'Active');
     &CheckFormElement($cache, 'postdata', 'reportSelected', 'Class list');
     &CheckFormElement($cache, 'reportSelected', 'reportSelected', 
@@ -82,6 +83,7 @@ sub ProcessFormData{
     &CheckFormElement($cache, 'sort', 'sort', 'fullname');
     &CheckFormElement($cache, 'download', 'download', 'false');
 
+    # student assessment
     if(defined($ENV{'form.CreateStudentAssessment'}) ||
        defined($ENV{'form.NextStudent'}) ||
        defined($ENV{'form.PreviousStudent'})) {
@@ -98,6 +100,7 @@ sub ProcessFormData{
                       'StudentAssessmentStudent', 'All Students');
     $cache->{'StudentAssessmentStudent'} = 
         &Apache::lonnet::unescape($cache->{'StudentAssessmentStudent'});
+    &CheckFormElement($cache, 'DefaultColumns', 'DefaultColumns', 'false');
 
     if(defined($ENV{'form.Section'})) {
         my @sectionsSelected = (ref($ENV{'form.Section'}) ?
@@ -108,48 +111,89 @@ sub ProcessFormData{
         $cache->{'sectionsSelected'} = $cache->{'sectionList'};
     }
 
-    foreach (keys(%ENV)) {
-        if(/form\.Analyze:::/) {
-#            $cache->{'reportSelected'} = 'Analyze';
-#            $cache->{'reportKey'} = 'Problem Analysis';
-            my ($uri, $title, $part, $problem);
-            (undef, $uri, $title, $part, $problem)=split(':::', $_);
-            $cache->{'AnalyzeURI'}     = $uri;
-            $cache->{'AnalyzeTitle'}   = $title;
-            $cache->{'AnalyzePart'}    = $part;
-            $cache->{'AnalyzeProblem'} = $problem;
-            
-            &CheckFormElement($cache, 'Interval', 'Interval', '1');
+    # Problem analysis
+    &CheckFormElement($cache, 'Interval', 'Interval', '1');
+
+    # ProblemStatistcs
+    &CheckFormElement($cache, 'DisplayCSVFormat',
+                      'DisplayFormat', 'Display Table Format');
+    &CheckFormElement($cache, 'ProblemStatisticsAscend',
+                      'ProblemStatisticsAscend', 'Ascending');
+    &CheckFormElement($cache, 'ProblemStatisticsMaps', 
+                      'ProblemStatisticsMaps', 'All Maps');
+    &CheckFormElement($cache, 'ProblemStatisticsSort',
+                      'ProblemStatisticsSort', 'Homework Sets Order');
+    &CheckFormElement($cache, 'DisplayLegend', 'DisplayLegend', 'Hide Legend');
+
+    # Search only form elements
+    my @headingColumns=();
+    my @sequenceColumns=();
+    my $foundColumn = 0;
+    if(defined($ENV{'form.ReselectColumns'})) {
+        my @reselected = (ref($ENV{'form.ReselectColumns'}) ? 
+                          @{$ENV{'form.ReselectColumns'}}
+                          : ($ENV{'form.ReselectColumns'}));
+        foreach (@reselected) {
+            if(/HeadingColumn/) {
+                push(@headingColumns, $_);
+                $foundColumn = 1;
+            } elsif(/SequenceColumn/) {
+                push(@sequenceColumns, $_);
+                $foundColumn = 1;
+            }
         }
     }
 
-    return;
+    $cache->{'reportKey'} = 'false';
+    if($cache->{'reportSelected'} eq 'Analyze') {
+        $cache->{'reportKey'} = 'Analyze';
+    } elsif($cache->{'reportSelected'} eq 'DoDiffGraph') {
+        $cache->{'reportKey'} = 'DoDiffGraph';
+    } elsif($cache->{'reportSelected'} eq 'PercentWrongGraph') {
+        $cache->{'reportKey'} = 'PercentWrongGraph';
+    }
 
-    # Select page to display
-    if(defined($ENV{'form.ProblemStatistics'}) ||
-       defined($ENV{'form.ProblemStatisticsRecalculate'}) || 
-       defined($ENV{'form.DisplayCSVFormat'})) {
-        $cache->{'GoToPage'} = 'ProblemStatistics';
-        &CheckFormElement($cache, 'DisplayCSVFormat',
-                          'DisplayFormat', 'Display Table Format');
-        &CheckFormElement($cache, 'Ascend','ProblemStatisticsAscend',
-                          'Ascending');
-        &CheckFormElement($cache, 'Maps', 'ProblemStatisticsMap', 
-                          'All Maps');
-    } elsif(defined($ENV{'form.ProblemAnalysis'})) {
-        $cache->{'GoToPage'} = 'ProblemAnalysis';
-        &CheckFormElement($cache, 'Interval', 'Interval', '1');
-    } elsif(defined($ENV{'form.DoDiffGraph'})) {
-        $cache->{'GoToPage'} = 'DoDiffGraph';
+    if(defined($ENV{'form.DoDiffGraph'})) {
+        $cache->{'reportSelected'} = 'DoDiffGraph';
+        $cache->{'reportKey'} = 'DoDiffGraph';
     } elsif(defined($ENV{'form.PercentWrongGraph'})) {
-        $cache->{'GoToPage'} = 'PercentWrongGraph';
-    } elsif(defined($ENV{'form.ActivityLog'})) {
-        $cache->{'GoToPage'} = 'ActivityLog';
-    } else {
-        $cache->{'GoToPage'} = 'Menu';
+        $cache->{'reportSelected'} = 'PercentWrongGraph';
+        $cache->{'reportKey'} = 'PercentWrongGraph';
     }
 
-    &CheckFormElement($cache, 'Status', 'Status', 'Active');
+    foreach (keys(%ENV)) {
+        if(/form\.Analyze/) {
+            $cache->{'reportSelected'} = 'Analyze';
+            $cache->{'reportKey'} = 'Analyze';
+            my $data;
+            (undef, $data)=split(':::', $_);
+            $cache->{'AnalyzeInfo'}=$data;
+        } elsif(/form\.HeadingColumn/) {
+            my $value = $_;
+            $value =~ s/form\.//;
+            push(@headingColumns, $value);
+            $foundColumn=1;
+        } elsif(/form\.SequenceColumn/) {
+            my $value = $_;
+            $value =~ s/form\.//;
+            push(@sequenceColumns, $value);
+            $foundColumn=1;
+        }
+    }
+
+    if($foundColumn) {
+        $cache->{'HeadingsFound'} = join(':', @headingColumns);
+        $cache->{'SequencesFound'} = join(':', @sequenceColumns);;
+    }
+    if(!defined($cache->{'HeadingsFound'}) || 
+       $cache->{'DefaultColumns'} ne 'false') {
+        $cache->{'HeadingsFound'}='HeadingColumnFull Name';
+    }
+    if(!defined($cache->{'SequencesFound'}) ||
+       $cache->{'DefaultColumns'} ne 'false') {
+        $cache->{'SequencesFound'}='All Sequences';
+    }
+    $cache->{'DefaultColumns'} = 'false';
 
     return;
 }
@@ -252,7 +296,7 @@ sub SpaceColumns {
 }
 
 sub PrepareData {
-    my ($c, $cacheDB, $studentInformation, $headings)=@_;
+    my ($c, $cacheDB, $studentInformation, $headings,$r)=@_;
 
     # Test for access to the cache data
     my $courseID=$ENV{'request.course.id'};
@@ -269,13 +313,13 @@ sub PrepareData {
 
     # Download class list information if not using cached data
     my %cache;
-    unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT,0640)) {
+    unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
         return "Unable to tie hash to db file.";
     }
 
     if(!$isCached) {
         my $processTopResourceMapReturn=
-            &Apache::loncoursedata::ProcessTopResourceMap(\%cache, $c);
+            &Apache::loncoursedata::ProcessTopResourceMap(\%cache, $c, $r);
         if($processTopResourceMapReturn ne 'OK') {
             untie(%cache);
             return $processTopResourceMapReturn;
@@ -315,33 +359,31 @@ sub PrepareData {
     $cache{'updateTime:columnWidth'}=24;
 
     if($cache{'download'} ne 'false') {
-        my $who = $cache{'download'};
-        my $courseData = 
-            &Apache::loncoursedata::DownloadCourseInformation(
-                                             $who, $courseID, 
-                                             $cache{$who.':lastDownloadTime'});
-        &Apache::loncoursedata::ProcessStudentData(\%cache, $courseData, $who);
+        my @who = ($cache{'download'});
         $cache{'download'} = 'false';
+        if(&Apache::loncoursedata::DownloadStudentCourseData(\@who, 'false', 
+                                                             $cacheDB, 'true', 
+                                                             'false', $courseID,
+                                                             $r, $c) ne 'OK') {
+            untie(%cache);
+            return 'Stop at download individual';
+        }
     } elsif($cache{'DownloadAll'} ne 'false') {
+        $cache{'DownloadAll'} = 'false';
         my @allStudents;
         if($cache{'DownloadAll'} eq 'sorted') {
             @allStudents = @$students;
         } else {
             @allStudents = split(':::', $cache{'NamesOfStudents'});
         }
-        foreach (@allStudents) {
-            my $courseData = 
-                &Apache::loncoursedata::DownloadCourseInformation(
-                                             $_, $courseID, 
-                                             $cache{$_.':lastDownloadTime'});
-            &Apache::loncoursedata::ProcessStudentData(\%cache, $courseData, 
-                                                       $_);
-            if($c->aborted()) {
-                untie(%cache);
-                return 'aborted'; 
-            }
+        if(&Apache::loncoursedata::DownloadStudentCourseData(\@allStudents, 
+                                                             'false', 
+                                                             $cacheDB, 'true', 
+                                                             'true', $courseID,
+                                                             $r, $c) ne 'OK') {
+            untie(%cache);
+            return 'Stop at download all';
         }
-        $cache{'DownloadAll'} = 'false';
     }
 
     if($c->aborted()) {
@@ -354,68 +396,11 @@ sub PrepareData {
     return ('OK', $students);
 }
 
-
-# Create progress
-sub Create_PrgWin {
-    my ($r)=@_;
-    $r->print(<<ENDPOP);
-    <script>
-    popwin=open('','popwin','width=400,height=100');
-    popwin.document.writeln('<html><body bgcolor="#88DDFF">'+
-      '<title>LON-CAPA Statistics</title>'+
-      '<h4>Computation Progress</h4>'+
-      '<form name=popremain>'+
-      '<input type=text size=35 name=remaining value=Starting></form>'+
-      '</body></html>');
-    popwin.document.close();
-    </script>
-ENDPOP
-
-    $r->rflush();
-}
-
-# update progress
-sub Update_PrgWin {
-    my ($totalStudents,$index,$name,$r)=@_;
-    $r->print('<script>popwin.document.popremain.remaining.value="'.
-              'Computing '.$index.'/'.$totalStudents.': '.
-              $name.'";</script>');
-    $r->rflush();
-}
-
-# close Progress Line
-sub Close_PrgWin {
-    my ($r)=@_;
-    $r->print('<script>popwin.close()</script>');
-    $r->rflush(); 
-}
-
-# For loading the colored table for display or un-colored for print
-sub setbgcolor {
-    my $PrintTable=shift;
-    my %color;
-    if ($PrintTable){
-	$color{"gb"}="#FFFFFF";
-	$color{"red"}="#FFFFFF";
-	$color{"yellow"}="#FFFFFF";
-	$color{"green"}="#FFFFFF";
-	$color{"purple"}="#FFFFFF";
-    } else {
-	$color{"gb"}="#DDFFFF";
-	$color{"red"}="#FFDDDD";
-	$color{"yellow"}="#EEFFCC";
-	$color{"green"}="#DDFFDD";
-	$color{"purple"}="#FFDDFF";
-    }
-
-    return \%color;
-}
-
 sub BuildClasslist {
-    my ($cacheDB,$students,$studentInformation,$headings)=@_;
+    my ($cacheDB,$students,$studentInformation,$headings,$r)=@_;
 
     my %cache;
-    unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER,0640)) {
+    unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
         return '<html><body>Unable to tie database.</body></html>';
     }
 
@@ -425,9 +410,11 @@ sub BuildClasslist {
 
     my $displayString = '<td align="left"><a href="/adm/statistics?';
     $displayString .= 'sort=LINKDATA">DISPLAYDATA&nbsp</a></td>'."\n";
-    $Str .= &Apache::lonhtmlcommon::CreateHeadings(\%cache, $studentInformation,
+    $Str .= &Apache::lonhtmlcommon::CreateHeadings(\%cache, 
+                                                   $studentInformation,
                                                    $headings, $displayString);
     $Str .= '</tr>'."\n";
+
     my $alternate=0;
     foreach (@$students) {
         my ($username, $domain) = split(':', $_);
@@ -462,10 +449,12 @@ sub BuildClasslist {
 
     $Str .= '</tr>'."\n";
     $Str .= '</table></td></tr></table>'."\n";
+    $r->print($Str);
+    $r->rflush();
 
     untie(%cache);
 
-    return $Str;
+    return;
 }
 
 sub CreateMainMenu {
@@ -518,6 +507,7 @@ sub BuildStatistics {
     my %reports = ('classlist'          => 'Class list',
                    'problem_statistics' => 'Problem Statistics',
                    'student_assessment' => 'Student Assessment',
+#                   'activitylog'        => 'Activity Log',
                    'reportSelected'     => 'Class list');
 
     my %cache;
@@ -525,31 +515,40 @@ sub BuildStatistics {
     my $cacheDB = "/home/httpd/perl/tmp/$ENV{'user.name'}".
                   "_$ENV{'user.domain'}_$courseID\_statistics.db";
 
-    my %color=&setbgcolor(0);
+    $r->print(&Apache::lonhtmlcommon::Title('LON-CAPA Statistics'));
+
     my ($returnValue, $students) = &PrepareData($c, $cacheDB, 
                                                 \@studentInformation, 
-                                                \@headings);
+                                                \@headings,$r);
     if($returnValue ne 'OK') {
-        $r->print('<html><body>'.$returnValue."\n".'</body></html>');
+        $r->print($returnValue."\n".'</body></html>');
         return OK;
     }
+    if(!$c->aborted()) {
+        &Apache::loncoursedata::CheckForResidualDownload($cacheDB, 
+                                                         'true', 'true',
+                                                         $courseID,
+                                                         $r, $c);
+    }
 
     my $GoToPage;
-    if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER,0640)) {
+    if(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
         $GoToPage = $cache{'reportSelected'};
         $reports{'reportSelected'} = $cache{'reportSelected'};
-#        if(defined($cache{'reportKey'}) && $cache{'reportKey'} ne 'false') {
-#            $reports{$cache{'reportKey'}} = $cache{'reportSelected'};
-#        }
+        if(defined($cache{'reportKey'}) && 
+           !exists($reports{$cache{'reportKey'}}) && 
+           $cache{'reportKey'} ne 'false') {
+            $reports{$cache{'reportKey'}} = $cache{'reportSelected'};
+        }
 
         if(defined($cache{'OptionResponses'})) {
             $reports{'problem_analysis'} = 'Problem Analysis';
         }
 
-        $r->print(&Apache::lonhtmlcommon::Title('LON-CAPA Statistics'));
         $r->print('<form name="Statistics" ');
         $r->print('method="post" action="/adm/statistics">');
         $r->print(&CreateMainMenu($cache{'Status'}, \%reports));
+        $r->rflush();
         untie(%cache);
     } else {
         $r->print('<html><body>Unable to tie database.</body></html>');
@@ -559,36 +558,33 @@ sub BuildStatistics {
     if($GoToPage eq 'Activity Log') {
         &Apache::lonproblemstatistics::Activity();
     } elsif($GoToPage eq 'Problem Statistics') {
-        $r->print(
         &Apache::lonproblemstatistics::BuildProblemStatisticsPage($cacheDB, 
                                                                   $students, 
                                                                   $courseID, 
-                                                                  $c,$r,
-								  \%color));
+                                                                  $c,$r);
     } elsif($GoToPage eq 'Problem Analysis') {
-        $r->print(
-              &Apache::lonproblemanalysis::BuildProblemAnalysisPage($cacheDB));
+        &Apache::lonproblemanalysis::BuildProblemAnalysisPage($cacheDB, $r);
     } elsif($GoToPage eq 'Student Assessment') {
-        $r->print(
-            &Apache::lonstudentassessment::BuildStudentAssessmentPage($cacheDB,
-                                                            $students,
-                                                            $courseID,
-                                                            'Statistics',
-                                                            \@headings,
-                                                            $spacing,
-                                                            \@studentInformation,
-                                                            $r, $c));
+        &Apache::lonstudentassessment::BuildStudentAssessmentPage($cacheDB,
+                                                          $students,
+                                                          $courseID,
+                                                          'Statistics',
+                                                          \@headings,
+                                                          $spacing,
+                                                          \@studentInformation,
+                                                          $r, $c);
     } elsif($GoToPage eq 'Analyze') {
-        $r->print(&Apache::lonproblemanalysis::BuildAnalyzePage($cacheDB, 
-                                                                $students, 
-                                                                $courseID));
-    } elsif($GoToPage eq 'DoDiffGraph') {
-        &Apache::lonproblemstatistics::BuildDiffGraph($r);
-    } elsif($GoToPage eq 'PercentWrongGraph') {
-        &Apache::lonproblemstatistics::BuildWrongGraph($r);
+        &Apache::lonproblemanalysis::BuildAnalyzePage($cacheDB, $students, 
+                                                      $courseID, $r);
+    } elsif($GoToPage eq 'DoDiffGraph' || $GoToPage eq 'PercentWrongGraph') {
+        my $courseDescription = $ENV{'course.'.$courseID.'.description'};
+        $courseDescription =~ s/\ /"_"/eg;
+        &Apache::lonproblemstatistics::BuildGraphicChart($GoToPage, $cacheDB,
+                                                         $courseDescription,
+                                                         $r);
     } elsif($GoToPage eq 'Class list') {
-        $r->print(&BuildClasslist($cacheDB, $students, \@studentInformation,
-                                  \@headings));
+        &BuildClasslist($cacheDB, $students, \@studentInformation,
+                        \@headings, $r);
     }
 
     $r->print('</form>'."\n");
@@ -603,6 +599,8 @@ sub BuildStatistics {
 sub handler {
     my $r=shift;
 
+#    $jr = $r;
+
     unless(&Apache::lonnet::allowed('vgr',$ENV{'request.course.id'})) {
         $ENV{'user.error.msg'}=
         $r->uri.":vgr:0:0:Cannot view grades for complete course";