--- loncom/interface/statistics/lonstathelpers.pm	2004/02/19 20:17:01	1.4
+++ loncom/interface/statistics/lonstathelpers.pm	2004/06/15 14:26:10	1.14
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: lonstathelpers.pm,v 1.4 2004/02/19 20:17:01 matthew Exp $
+# $Id: lonstathelpers.pm,v 1.14 2004/06/15 14:26:10 matthew Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -59,6 +59,8 @@ use Apache::lonlocal;
 use HTML::Entities();
 use Time::Local();
 use Spreadsheet::WriteExcel();
+use GDBM_File;
+use Storable qw(freeze thaw);
 
 ####################################################
 ####################################################
@@ -107,6 +109,7 @@ all option response and radiobutton prob
 
 Returns: A string containing html for a table which lists the sequences
 and their contents.  A radiobutton is provided for each problem.
+Skips 'survey' problems.
 
 =cut
 
@@ -116,13 +119,14 @@ sub ProblemSelector {
     my ($AcceptedResponseTypes) = @_;
     my $Str;
     $Str = "\n<table>\n";
-    foreach my $seq (&Apache::lonstatistics::Sequences_with_Assess()) {
+    foreach my $seq (&Apache::lonstatistics::Sequences_with_Assess('all')) {
         next if ($seq->{'num_assess'}<1);
         my $seq_str = '';
         foreach my $res (@{$seq->{'contents'}}) {
             next if ($res->{'type'} ne 'assessment');
             foreach my $part (@{$res->{'parts'}}) {
                 my $partdata = $res->{'partdata'}->{$part};
+                next if ($partdata->{'Survey'});
                 for (my $i=0;$i<scalar(@{$partdata->{'ResponseTypes'}});$i++){
                     my $respid = $partdata->{'ResponseIds'}->[$i];
                     my $resptype = $partdata->{'ResponseTypes'}->[$i];
@@ -145,7 +149,7 @@ sub ProblemSelector {
   $resptype.'</td><td>'.
   '<a href="'.$res->{'src'}.'">'.$title.'</a> ';
 #  '<a href="'.$res->{'src'}.'">'.$resptype.' '.$res->{'title'}.'</a> ';
-                        if ($partdata->{'option'} > 1) {
+                        if (scalar(@{$partdata->{'ResponseIds'}}) > 1) {
                             $seq_str .= &mt('response').' '.$respid;
                         }
                         $seq_str .= "</td></tr>\n";
@@ -222,7 +226,7 @@ sub get_target_from_id {
 
 =pod
 
-=item &get_prev_curr_next($target)
+=item &get_prev_curr_next($target,$AcceptableResponseTypes,$granularity)
 
 Determine the problem parts or responses preceeding and following the
 current resource.
@@ -230,7 +234,7 @@ current resource.
 Inputs: $target (see &Apache::lonstathelpers::get_target_from_id())
   $AcceptableResponseTypes, regular expression matching acceptable
                             response types,
-  $granularity, either 'part' or 'response'
+  $granularity, either 'part', 'response', or 'part_survey'
 
 Returns: three hash references, $prev, $curr, $next, which refer to the
 preceeding, current, or following problem parts or responses, depending
@@ -253,12 +257,18 @@ sub get_prev_curr_next {
     #
     # Build an array with the data we need to search through
     my @Resource;
-    foreach my $seq (&Apache::lonstatistics::Sequences_with_Assess()) {
+    foreach my $seq (&Apache::lonstatistics::Sequences_with_Assess('all')) {
         foreach my $res (@{$seq->{'contents'}}) {
             next if ($res->{'type'} ne 'assessment');
             foreach my $part (@{$res->{'parts'}}) {
                 my $partdata = $res->{'partdata'}->{$part};
-                if ($granularity eq 'part') {
+                if ($granularity eq 'part_survey' && $partdata->{'Survey'}){
+                    push (@Resource,
+                          { symb     => $res->{symb},
+                            part     => $part,
+                            resource => $res,
+                        } );
+                } elsif ($granularity eq 'part') {
                     push (@Resource,
                           { symb     => $res->{symb},
                             part     => $part,
@@ -288,7 +298,7 @@ sub get_prev_curr_next {
     my $curr_idx;
     for ($curr_idx=0;$curr_idx<$#Resource;$curr_idx++) {
         my $curr_item = $Resource[$curr_idx];
-        if ($granularity eq 'part') {
+        if ($granularity eq 'part' || $granularity eq 'part_survey') {
             if ($curr_item->{'symb'} eq $target->{'symb'} &&
                 $curr_item->{'part'} eq $target->{'part'}) {
                 last;
@@ -303,7 +313,7 @@ sub get_prev_curr_next {
         }
     }
     my $curr_item = $Resource[$curr_idx];
-    if ($granularity eq 'part') {
+    if ($granularity eq 'part' || $granularity eq 'part_survey') {
         if ($curr_item->{'symb'}     ne $target->{'symb'} ||
             $curr_item->{'part'}     ne $target->{'part'}) {
             # bogus symb - return nothing
@@ -343,6 +353,67 @@ sub get_prev_curr_next {
 
 =pod
 
+=item GetStudentAnswers($r,$problem,$Students)
+
+Determines the correct answer for a set of students on a given problem.
+The students answers are stored in the student hashes pointed to by the
+array @$Students under the key 'answer'.
+
+Inputs: $r
+$problem: hash reference containing the keys 'resource', 'part', and 'respid'.
+$Students: reference to array containing student hashes (need 'username', 
+    'domain').  
+
+Returns: nothing 
+
+=cut
+
+#####################################################
+#####################################################
+sub GetStudentAnswers {
+    my ($r,$problem,$Students,$formname,$inputname) = @_;
+    my $status_type;
+    if (defined($formname)) {
+        $status_type = 'inline';
+    } else {
+        $status_type = 'popup';
+    }    
+    my $c = $r->connection();
+    my %Answers;
+    my ($resource,$partid,$respid) = ($problem->{'resource'},
+                                      $problem->{'part'},
+                                      $problem->{'respid'});
+    # Read in the cache (if it exists) before we start timing things.
+    &Apache::lonstathelpers::ensure_proper_cache($resource->{'symb'});
+    # Open progress window
+    my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
+        ($r,'Student Answer Compilation Status',
+         'Student Answer Compilation Progress', scalar(@$Students),
+         $status_type,undef,$formname,$inputname);
+    $r->rflush();
+    foreach my $student (@$Students) {
+        last if ($c->aborted());
+        my $sname = $student->{'username'};
+        my $sdom = $student->{'domain'};
+        my $answer = &Apache::lonstathelpers::analyze_problem_as_student
+            ($resource,$sname,$sdom,$partid,$respid);
+        &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+                                                 &mt('last student'));
+        $student->{'answer'} = $answer;
+    }
+    &Apache::lonstathelpers::write_answer_cache();
+    return if ($c->aborted());
+    $r->rflush();
+    # close progress window
+    &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
+    return;
+}
+
+#####################################################
+#####################################################
+
+=pod
+
 =item analyze_problem_as_student
 
 Analyzes a homework problem for a student and returns the correct answer
@@ -354,6 +425,12 @@ Inputs: $resource: a resource object
 
 Returns: $answer
 
+If $partid and $respid are specified, $answer is simply a scalar containing
+the correct answer for the response.
+
+If $partid or $respid are undefined, $answer will be a hash reference with
+keys $partid.'.'.$respid.'.answer'.
+
 =cut
 
 #####################################################
@@ -363,6 +440,10 @@ sub analyze_problem_as_student {
     my $returnvalue;
     my $url = $resource->{'src'};
     my $symb = $resource->{'symb'};
+    my $answer = &get_from_answer_cache($sname,$sdom,$symb,$partid,$respid);
+    if (defined($answer)) {
+        return($answer);
+    }
     my $courseid = $ENV{'request.course.id'};
     my $Answ=&Apache::lonnet::ssi($url,('grade_target' => 'analyze',
                                         'grade_domain' => $sdom,
@@ -371,8 +452,30 @@ sub analyze_problem_as_student {
                                         'grade_courseid' => $courseid));
     (my $garbage,$Answ)=split(/_HASH_REF__/,$Answ,2);
     my %Answer=&Apache::lonnet::str2hash($Answ);
-    my $prefix = $partid.'.'.$respid;
-    my $key = $prefix.'.answer';
+    #
+    undef($answer);
+    foreach my $partid (@{$resource->{'parts'}}) {
+        my $partdata = $resource->{'partdata'}->{$partid};
+        foreach my $respid (@{$partdata->{'ResponseIds'}}) {
+            my $prefix = $partid.'.'.$respid;
+            my $key = $prefix.'.answer';
+            $answer->{$partid}->{$respid} = &get_answer($prefix,$key,%Answer);
+        }
+    }
+    &store_answer($sname,$sdom,$symb,undef,undef,$answer);
+    if (! defined($partid)) {
+        $returnvalue = $answer;
+    } elsif (! defined($respid)) {
+        $returnvalue = $answer->{$partid};
+    } else {
+        $returnvalue = $answer->{$partid}->{$respid};
+    }
+    return $returnvalue;
+}
+
+sub get_answer {
+    my ($prefix,$key,%Answer) = @_;
+    my $returnvalue;
     if (exists($Answer{$key})) {
         my $student_answer = $Answer{$key}->[0];
         if (! defined($student_answer)) {
@@ -390,11 +493,11 @@ sub analyze_problem_as_student {
             }
             foreach my $foil (@{$Answer{$prefix.'.shown'}}) {
                 if (ref($values{$foil}) eq 'ARRAY') {
-                    $returnvalue.=&HTML::Entities::encode($foil).'='.
-                        join(',',map {&HTML::Entities::encode($_)} @{$values{$foil}}).'&';
+                    $returnvalue.=&HTML::Entities::encode($foil,'<>&"').'='.
+                        join(',',map {&HTML::Entities::encode($_,'<>&"')} @{$values{$foil}}).'&';
                 } else {
-                    $returnvalue.=&HTML::Entities::encode($foil).'='.
-                        &HTML::Entities::encode($values{$foil}).'&';
+                    $returnvalue.=&HTML::Entities::encode($foil,'<>&"').'='.
+                        &HTML::Entities::encode($values{$foil},'<>&"').'&';
                 }
             }
             $returnvalue =~ s/ /\%20/g;
@@ -405,6 +508,190 @@ sub analyze_problem_as_student {
 }
 
 
+#####################################################
+#####################################################
+
+=pod
+
+=item Caching routines
+
+=over 4
+
+=item &load_answer_cache($symb)
+
+Loads the cache for the given symb into memory from disk.  
+Requires the cache filename be set.  
+Only should be called by &ensure_proper_cache.
+
+=cut
+
+#####################################################
+#####################################################
+{
+    my $cache_filename = undef;
+    my $current_symb = undef;
+    my %cache;
+
+sub load_answer_cache {
+    my ($symb) = @_;
+    return if (! defined($cache_filename));
+    if (! defined($current_symb) || $current_symb ne $symb) {
+        undef(%cache);
+        my $storedstring;
+        my %cache_db;
+        if (tie(%cache_db,'GDBM_File',$cache_filename,&GDBM_READER(),0640)) {
+            $storedstring = $cache_db{&Apache::lonnet::escape($symb)};
+            untie(%cache_db);
+        }
+        if (defined($storedstring)) {
+            %cache = %{thaw($storedstring)};
+        }
+    }
+    return;
+}
+
+#####################################################
+#####################################################
+
+=pod
+
+=item &get_from_answer_cache($sname,$sdom,$symb,$partid,$respid)
+
+Returns the appropriate data from the cache, or undef if no data exists.
+If $respid is undefined, a hash ref containing the answers for the given 
+$partid is returned.  If $partid is undefined, a hash ref containing answers
+for all of the parts is returned.
+
+=cut
+
+#####################################################
+#####################################################
+sub get_from_answer_cache {
+    my ($sname,$sdom,$symb,$partid,$respid) = @_;
+    &ensure_proper_cache($symb);
+    my $returnvalue;
+    if (exists($cache{$sname.':'.$sdom}) &&
+        ref($cache{$sname.':'.$sdom}) eq 'HASH') {
+        if (defined($partid) &&
+            exists($cache{$sname.':'.$sdom}->{$partid})) {
+            if (defined($respid) &&
+                exists($cache{$sname.':'.$sdom}->{$partid}->{$respid})) {
+                $returnvalue = $cache{$sname.':'.$sdom}->{$partid}->{$respid};
+            } else {
+                $returnvalue = $cache{$sname.':'.$sdom}->{$partid};
+            }
+        } else {
+            $returnvalue = $cache{$sname.':'.$sdom};
+        }
+    } else {
+        $returnvalue = undef;
+    }
+    return $returnvalue;
+}
+
+#####################################################
+#####################################################
+
+=pod
+
+=item &write_answer_cache($symb)
+
+Writes the in memory cache to disk so that it can be read in with
+&load_answer_cache($symb).
+
+=cut
+
+#####################################################
+#####################################################
+sub write_answer_cache {
+    return if (! defined($current_symb) || ! defined($cache_filename));
+    my %cache_db;
+    my $key = &Apache::lonnet::escape($current_symb);
+    if (tie(%cache_db,'GDBM_File',$cache_filename,&GDBM_WRCREAT(),0640)) {
+        my $storestring = freeze(\%cache);
+        $cache_db{$key}=$storestring;
+        $cache_db{$key.'.time'}=time;
+        untie(%cache_db);
+    }
+    undef(%cache);
+    undef($current_symb);
+    undef($cache_filename);
+    return;
+}
+
+#####################################################
+#####################################################
+
+=pod
+
+=item &ensure_proper_cache($symb)
+
+Called to make sure we have the proper cache set up.  This is called
+prior to every answer lookup.
+
+=cut
+
+#####################################################
+#####################################################
+sub ensure_proper_cache {
+    my ($symb) = @_;
+    my $cid = $ENV{'request.course.id'};
+    my $new_filename =  '/home/httpd/perl/tmp/'.
+        'problemanalsysis_'.$cid.'answer_cache.db';
+    if (! defined($cache_filename) ||
+        $cache_filename ne $new_filename ||
+        ! defined($current_symb)   ||
+        $current_symb ne $symb) {
+        $cache_filename = $new_filename;
+        # Notice: $current_symb is not set to $symb until after the cache is
+        # loaded.  This is what tells &load_answer_cache to load in a new
+        # symb cache.
+        &load_answer_cache($symb);
+        $current_symb = $symb;
+    }
+}
+
+#####################################################
+#####################################################
+
+=pod
+
+=item &store_answer($sname,$sdom,$symb,$partid,$respid,$dataset)
+
+Stores the answer data in the in memory cache.
+
+=cut
+
+#####################################################
+#####################################################
+sub store_answer {
+    my ($sname,$sdom,$symb,$partid,$respid,$dataset) = @_;
+    return if ($symb ne $current_symb);
+    if (defined($partid)) {
+        if (defined($respid)) {
+            $cache{$sname.':'.$sdom}->{$partid}->{$respid} = $dataset;
+        } else {
+            $cache{$sname.':'.$sdom}->{$partid} = $dataset;
+        }
+    } else {
+        $cache{$sname.':'.$sdom}=$dataset;
+    }
+    return;
+}
+
+}
+#####################################################
+#####################################################
+
+=pod
+
+=back
+
+=cut
+
+#####################################################
+#####################################################
+
 ##
 ## The following is copied from datecalc1.pl, part of the 
 ## Spreadsheet::WriteExcel CPAN module.
@@ -612,7 +899,7 @@ sub get_problem_data {
         while (my($key,$value) = each(%Answer)) {
             #
             # Logging code:
-            if (1) {
+            if (0) {
                 &Apache::lonnet::logthis($part.' got key "'.$key.'"');
                 if (ref($value) eq 'ARRAY') {
                     &Apache::lonnet::logthis('    @'.join(',',@$value));
@@ -655,6 +942,162 @@ sub get_problem_data {
 
 ####################################################
 ####################################################
+
+=pod
+
+=item &limit_by_time()
+
+=cut
+
+####################################################
+####################################################
+sub limit_by_time_form {
+    my $Starttime_form = '';
+    my $starttime = &Apache::lonhtmlcommon::get_date_from_form
+        ('limitby_startdate');
+    my $endtime = &Apache::lonhtmlcommon::get_date_from_form
+        ('limitby_enddate');
+    if (! defined($endtime)) {
+        $endtime = time;
+    }
+    if (! defined($starttime)) {
+        $starttime = $endtime - 60*60*24*7;
+    }
+    my $state;
+    if (&limit_by_time()) {
+        $state = '';
+    } else {
+        $state = 'disabled';
+    }
+    my $startdateform = &Apache::lonhtmlcommon::date_setter
+        ('Statistics','limitby_startdate',$starttime,undef,undef,$state);
+    my $enddateform = &Apache::lonhtmlcommon::date_setter
+        ('Statistics','limitby_enddate',$endtime,undef,undef,$state);
+    my $Str;
+    $Str .= '<script language="Javascript" >';
+    $Str .= 'function toggle_limitby_activity(state) {';
+    $Str .= '    if (state) {';
+    $Str .= '        limitby_startdate_enable();';
+    $Str .= '        limitby_enddate_enable();';
+    $Str .= '    } else {';
+    $Str .= '        limitby_startdate_disable();';
+    $Str .= '        limitby_enddate_disable();';
+    $Str .= '    }';    
+    $Str .= '}';
+    $Str .= '</script>';
+    $Str .= '<fieldset>';
+    my $timecheckbox = '<input type="checkbox" name="limit_by_time" ';
+    if (&limit_by_time()) {
+        $timecheckbox .= ' checked ';
+    } 
+    $timecheckbox .= 'OnChange="javascript:toggle_limitby_activity(this.checked);" ';
+    $timecheckbox .= ' />';
+    $Str .= '<legend>'.&mt('[_1] Limit by time',$timecheckbox).'</legend>';
+    $Str .= &mt('Start Time: [_1]',$startdateform).'<br />';
+    $Str .= &mt('&nbsp;End Time: [_1]',$enddateform).'<br />';
+    $Str .= '</fieldset>';
+    return $Str;
+}
+
+sub limit_by_time {
+    if (exists($ENV{'form.limit_by_time'}) &&
+        $ENV{'form.limit_by_time'} ne '' ) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+sub get_time_limits {
+    my $starttime = &Apache::lonhtmlcommon::get_date_from_form
+        ('limitby_startdate');
+    my $endtime = &Apache::lonhtmlcommon::get_date_from_form
+        ('limitby_enddate');
+    return ($starttime,$endtime);
+}
+
+
+
+####################################################
+####################################################
+
+=pod
+
+=item sections_description 
+
+Inputs: @Sections, an array of sections
+
+Returns: A text description of the sections selected.
+
+=cut
+
+####################################################
+####################################################
+sub sections_description {
+    my @Sections = @_;
+    my $sectionstring = '';
+    if (scalar(@Sections) > 1) {
+        if (scalar(@Sections) > 2) {
+            my $last = pop(@Sections);
+            $sectionstring = "Sections ".join(', ',@Sections).', and '.$last;
+        } else {
+            $sectionstring = "Sections ".join(' and ',@Sections);
+        }
+    } else {
+        if ($Sections[0] eq 'all') {
+            $sectionstring = "All sections";
+        } else {
+            $sectionstring = "Section ".$Sections[0];
+        }
+    }
+    return $sectionstring;
+}
+
+####################################################
+####################################################
+
+=pod
+
+=item &manage_caches
+
+Inputs: $r, apache request object
+
+Returns: An array of scalars containing html for buttons.
+
+=cut
+
+####################################################
+####################################################
+sub manage_caches {
+    my ($r,$formname,$inputname) = @_;
+    &Apache::loncoursedata::clear_internal_caches();
+    if (exists($ENV{'form.ClearCache'}) || 
+        exists($ENV{'form.updatecaches'}) ||
+        (exists($ENV{'form.firstrun'}) &&
+         $ENV{'form.firstrun'} ne 'no')) {
+        &Apache::lonstatistics::Gather_Full_Student_Data($r,$formname,
+                                                         $inputname);
+    }
+    #
+    if (! exists($ENV{'form.firstrun'})) {
+        $r->print('<input type="hidden" name="firstrun" value="yes" />');
+    } else {
+        $r->print('<input type="hidden" name="firstrun" value="no" />');
+    }
+    my @Buttons = 
+        ('<input type="submit" name="ClearCache" '.
+             'value="'.&mt('Clear Caches').'" />',
+         '<input type="submit" name="updatecaches" '.
+             'value="'.&mt('Update Caches').'" />');
+    #
+    return @Buttons;
+}
+
+
+
+
+####################################################
+####################################################
 
 =pod