--- loncom/interface/loncoursedata.pm	2015/03/10 02:45:56	1.196
+++ loncom/interface/loncoursedata.pm	2021/03/04 23:54:36	1.206
@@ -1,6 +1,6 @@
 # The LearningOnline Network with CAPA
 #
-# $Id: loncoursedata.pm,v 1.196 2015/03/10 02:45:56 raeburn Exp $
+# $Id: loncoursedata.pm,v 1.206 2021/03/04 23:54:36 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -1082,6 +1082,9 @@ sub store_updatetime {
     $dbh->do($request);
 }
 
+my $requested_max_packet = 0;
+my $max_allowed_packet;
+
 sub store_student_data {
     my ($sname,$sdom,$courseid,$student_data) = @_;
     #
@@ -1102,12 +1105,33 @@ sub store_student_data {
     my $starttime = Time::HiRes::time;
     my $elapsed = 0;
     my $rows_stored;
-    my $store_parameters_command  = 'INSERT IGNORE INTO '.$parameters_table.
+    my $store_parameters_prefix  = 'INSERT IGNORE INTO '.$parameters_table.
         ' VALUES '."\n";
     my $num_parameters = 0;
-    my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table.
+    my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table.
         ' VALUES '."\n";
     return ('error',undef) if (! defined($dbh));
+    unless ($requested_max_packet) {
+        (undef,$max_allowed_packet) = $dbh->selectrow_array(
+                                             qq{show variables LIKE ? },
+                                             undef,
+                                             "max_allowed_packet");
+        if ($max_allowed_packet !~ /^\d+$/) {
+            $max_allowed_packet = '';
+        }
+        $requested_max_packet = 1;
+    }
+    my @store_parameters_values = ();
+    my $curr_params_values = '';
+    my $curr_params_length = 0;
+    my @store_performance_values = ();
+    my $curr_perf_values = '';
+    my $curr_perf_length = 0;
+    my ($max_param,$max_perf);
+    if ($max_allowed_packet) {
+        $max_param = $max_allowed_packet - length($store_parameters_prefix);
+        $max_perf = $max_allowed_packet - length($store_performance_prefix);
+    }
     while (my ($current_symb,$param_hash) = each(%{$student_data})) {
         #
         # make sure the symb is set up properly
@@ -1120,10 +1144,39 @@ sub store_student_data {
                                               $symb_id,$student_id,
                                               $parameter)."',".
                                                   $dbh->quote($value)."),\n";
-                $num_parameters ++;
                 if ($sql_parameter !~ /''/) {
-                    $store_parameters_command .= $sql_parameter;
+                    if ($max_param) {
+                        my $length = length($sql_parameter);
+                        if ($length > $max_param) {
+                            &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size");
+                            &Apache::lonnet::logthis("symb_id: $symb_id");
+                            &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                            if ($parameter =~ /\.submission$/) {
+                                $sql_parameter = "('".join("','",
+                                              $symb_id,$student_id,
+                                              $parameter)."',".
+                                                  $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),\n";
+                                $length = length($sql_parameter);
+                                &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission");
+                                &Apache::lonnet::logthis("See grading interface for the actual submission");
+                            } else {
+                                &Apache::lonnet::logthis("Skipping this item");
+                                next;
+                            }
+                        }
+                        if ($length + $curr_params_length > $max_param) {
+                            push(@store_parameters_values,$curr_params_values);
+                            $curr_params_values = $sql_parameter;
+                            $curr_params_length = $length;
+                        } else {
+                            $curr_params_values .= $sql_parameter;
+                            $curr_params_length += $length;
+                        }
+                    } else {
+                        $curr_params_values .= $sql_parameter;
+                    }
                     #$rows_stored++;
+                    $num_parameters ++;
                 }
             }
         }
@@ -1165,31 +1218,66 @@ sub store_student_data {
                 "('".join("','",$symb_id,$student_id,$part_id,$part,
                                 $solved,$tries,$awarded,$award,
                                 $awarddetail,$timestamp)."'),\n";
-            $store_performance_command .= $sql_performance;
+            if ($max_perf) {
+                my $length = length($sql_performance);
+                if ($length > $max_perf) {
+                            &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size");
+                            &Apache::lonnet::logthis("symb_id: $symb_id");
+                            &Apache::lonnet::logthis("Skipping this item.  You may want to increase the max_allowed_packet size from the current: $max_allowed_packet");
+                            next;
+                } else {
+                    if ($length + $curr_perf_length > $max_perf) {
+                        push(@store_performance_values,$curr_perf_values);
+                        $curr_perf_values = $sql_performance;
+                        $curr_perf_length = $length;
+                    } else {
+                        $curr_perf_values .= $sql_performance;
+                        $curr_perf_length += $length;
+                    }
+                }
+            } else {
+                $curr_perf_values .= $sql_performance;
+            }
             $rows_stored++;
         }
     }
+    if ($curr_params_values ne '') {
+        push(@store_parameters_values,$curr_params_values);
+    }
+    if ($curr_perf_values ne '') {
+        push(@store_performance_values,$curr_perf_values);
+    }
     if (! $rows_stored) { return ($returnstatus, undef); }
-    $store_parameters_command =~ s|,\n$||;
-    $store_performance_command =~ s|,\n$||;
     my $start = Time::HiRes::time;
-    $dbh->do($store_performance_command);
-    if ($dbh->err()) {
-        &Apache::lonnet::logthis('performance bigass insert error:'.
-                                 $dbh->errstr());
-        &Apache::lonnet::logthis('command = '.$/.$store_performance_command);
-        $returnstatus = 'error: unable to insert performance into database';
-        return ($returnstatus,$student_data);
+    foreach my $item (@store_performance_values) {
+        $item =~ s|,\n$||;
+        if ($item ne '') {
+            $dbh->do($store_performance_prefix.$item);
+            if ($dbh->err()) {
+                &Apache::lonnet::logthis('performance insert error:'.
+                                         $dbh->errstr());
+                &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item);
+                $returnstatus = 'error: unable to insert performance into database';
+                return ($returnstatus,$student_data);
+            }
+        }
     }
-    $dbh->do($store_parameters_command) if ($num_parameters>0);
-    if ($dbh->err()) {
-        &Apache::lonnet::logthis('parameters bigass insert error:'.
-                                 $dbh->errstr());
-        &Apache::lonnet::logthis('command = '.$/.$store_parameters_command);
-        &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
-        &Apache::lonnet::logthis('student_id = '.$student_id);
-        $returnstatus = 'error: unable to insert parameters into database';
-        return ($returnstatus,$student_data);
+    if ($num_parameters > 0) {
+        foreach my $item (@store_parameters_values) {
+            $item =~ s|,\n$||;
+            if ($item ne '') {
+                $dbh->do($store_parameters_prefix.$item);
+                if ($dbh->err()) {
+                     &Apache::lonnet::logthis('parameters insert error:'.
+                                              $dbh->errstr());
+                     &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item);
+                     &Apache::lonnet::logthis('rows_stored = '.$rows_stored);
+                     &Apache::lonnet::logthis('student_id = '.$student_id);
+                     $returnstatus = 'error: unable to insert parameters into database';
+                     return ($returnstatus,$student_data);
+                }
+            }
+        }
     }
     $elapsed += Time::HiRes::time - $start;
     return ($returnstatus,$student_data);
@@ -1291,6 +1379,196 @@ sub ensure_current_full_data {
     return $status;
 }
 
+sub ensure_current_groups {
+    my ($courseid) = @_;  
+    my ($cdom,$cnum);
+    if (defined($courseid)) {
+        my %coursehash = &Apache::lonnet::coursedescription($courseid);
+        $cdom = $coursehash{'domain'};
+        $cnum = $coursehash{'num'};
+    } elsif ($env{'request.course.id'}) {
+        $courseid = $env{'request.course.id'};
+        $cdom = $env{'course.'.$courseid.'.domain'};
+        $cnum = $env{'course.'.$courseid.'.num'};
+    }
+    if ($cdom eq '' || $cnum eq '') {
+        return 'error: invalid course';
+    }
+    &setup_table_names($courseid);
+    my @CurrentTables = &Apache::lonmysql::tables_in_db();
+    unless (grep(/^\Q$groupnames_table\E$/,@CurrentTables)) {
+        return;
+    }
+    # Get the update time for the groupnames table
+    my $getuserdir = 1;
+    my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+        ($cdom,$cnum,'coursegroups.db',$getuserdir);
+    my %tableinfo = &Apache::lonmysql::table_information($groupnames_table);
+    my $updatetime;
+    if ($tableinfo{'Update_time'}) {
+        $updatetime = $tableinfo{'Update_time'};
+    }
+    if (! defined($updatetime) || $modifiedtime > $updatetime) {
+        my (%groups_in_sql,%removegroups,$addgroup);
+        my %curr_groups = &Apache::longroup::coursegroups($cdom,$cnum);
+        my @Result = &Apache::lonmysql::get_rows($groupnames_table);
+        foreach my $row (@Result) {
+            my ($id,$name) = @{$row};
+            unless (exists($curr_groups{$name})) {
+                $groups_in_sql{$name}=$id;
+            } elsif ($id) {
+                $removegroups{$id} = $name;
+            }
+        }
+        foreach my $group (keys(%curr_groups)) {
+            unless (exists($groups_in_sql{$group})) {
+                $addgroup = 1;
+                last;
+            }
+        }
+        if (keys(%removegroups)) {
+            my $dbh = &Apache::lonmysql::get_dbh();
+            foreach my $group_id (keys(%removegroups)) {
+                my $command = 'DELETE FROM '.$groupnames_table.' WHERE group_id='.
+                              $group_id;
+                $dbh->do($command);
+                if ($dbh->err()) {
+                    &Apache::lonnet::logthis("error ".$dbh->errstr().
+                                             " occurred executing \n".
+                                             "SQL command: $command");
+                }
+            }
+        }
+        if ($addgroup) {
+            &populate_groupnames_table($courseid);
+        }
+    }
+    return;
+}
+
+sub ensure_current_students_groups {
+    my ($courseid) = @_;
+    my ($cdom,$cnum);
+    if (defined($courseid)) {
+        my %coursehash = &Apache::lonnet::coursedescription($courseid);
+        $cdom = $coursehash{'domain'};
+        $cnum = $coursehash{'num'};
+    } elsif ($env{'request.course.id'}) {
+        $courseid = $env{'request.course.id'};
+        $cdom = $env{'course.'.$courseid.'.domain'};
+        $cnum = $env{'course.'.$courseid.'.num'};
+    }
+    &setup_table_names($courseid);
+    my @CurrentTables = &Apache::lonmysql::tables_in_db();
+    unless (grep(/^\Q$students_groups_table\E$/,@CurrentTables)) {
+        return;
+    }
+    # Get the update time for the groupnames table
+    my $getuserdir = 1;
+    my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+        ($cdom,$cnum,'groupmembership.db',$getuserdir);
+    my %tableinfo = &Apache::lonmysql::table_information($students_groups_table);
+    my $updatetime;
+    if ($tableinfo{'Update_time'}) {
+        $updatetime = $tableinfo{'Update_time'};
+    }
+    if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) {
+        if (&Apache::lonmysql::drop_table($students_groups_table)) {
+            if (&init_dbs($courseid)) {
+                return "error creating $students_groups_table\n";
+            } else {
+                &populate_students_groups_table($courseid);
+            }
+        }
+    }
+    return;
+}
+
+sub ensure_current_sections {
+    my ($courseid) = @_;
+    my ($cdom,$cnum);
+    if (defined($courseid)) {
+        my %coursehash = &Apache::lonnet::coursedescription($courseid);
+        $cdom = $coursehash{'domain'};
+        $cnum = $coursehash{'num'};
+    } elsif ($env{'request.course.id'}) {
+        $courseid = $env{'request.course.id'};
+        $cdom = $env{'course.'.$courseid.'.domain'};
+        $cnum = $env{'course.'.$courseid.'.num'};
+    }
+    &setup_table_names($courseid);
+    my @CurrentTables = &Apache::lonmysql::tables_in_db();
+    unless (grep(/^\Q$student_table\E$/,@CurrentTables)) {
+        return;
+    }
+    # Get the update time for the student table
+    my $getuserdir = 1;
+    my $modifiedtime = &Apache::lonnet::GetFileTimestamp
+        ($cdom,$cnum,'classlist.db',$getuserdir);
+    my %tableinfo = &Apache::lonmysql::table_information($student_table);
+    my $updatetime;
+    if ($tableinfo{'Update_time'}) {
+        $updatetime = $tableinfo{'Update_time'};
+    }
+    if ((!defined($updatetime)) || ($modifiedtime > $updatetime)) {
+        &update_student_table($cdom,$cnum);
+    }
+    return;
+}
+
+sub update_student_table {
+    my ($cdom,$cnum) = @_;
+    return unless (($cdom ne '') && ($cnum ne ''));
+    my (%roster,%sqldata);
+    my $classlist = &get_classlist($cdom,$cnum);
+    while (my ($student,$data) = each (%$classlist)) {
+        my ($section,$start,$end) = ($data->[&CL_SECTION()],
+                                     $data->[&CL_START()],
+                                     $data->[&CL_END()]);
+        if ($section eq '' || $section =~ /^\s*$/) {
+            $section = 'none';
+        }
+        if ($start eq '') { $start = 0; }
+        if ($end eq '')   { $end   = 0; }
+        $roster{$student}{'section'} = $section;
+        $roster{$student}{'start'} = $start;
+        $roster{$student}{'end'} = $end;
+    }
+    my $dbh = &Apache::lonmysql::get_dbh();
+    my $statement = "SELECT student_id,student,section,start,end FROM $student_table";
+    my $sth = $dbh->prepare($statement);
+    $sth->execute();
+    if ($sth->err()) {
+        &Apache::lonnet::logthis("Unable to execute MySQL request:");
+        &Apache::lonnet::logthis("\n".$statement."\n");
+        &Apache::lonnet::logthis("error is:".$sth->errstr());
+        return undef;
+    }
+    foreach my $row (@{$sth->fetchall_arrayref}) {
+        my ($id,$student,$section,$start,$end) = (@$row);
+        if (ref($roster{$student}) eq 'HASH') {
+            if (($roster{$student}{'section'} ne $section) ||
+                ($roster{$student}{'start'} ne $start) ||
+                ($roster{$student}{'end'} ne $end)) {
+                $sqldata{$id} = {
+                                  section => $roster{$student}{'section'},
+                                  start   => $roster{$student}{'start'},
+                                  end     => $roster{$student}{'end'},
+                                };
+            }
+        }
+    }
+    $sth->finish();
+    if (keys(%sqldata)) { 
+        foreach my $id (sort { $a <=> $b } keys(%sqldata)) {
+            my $request = "UPDATE $student_table SET section='$sqldata{$id}{section}'".
+                          ", start='$sqldata{$id}{start}'".
+                          ", end='$sqldata{$id}{end}' WHERE student_id='$id'";
+            $dbh->do($request);
+        }
+    }
+    return;
+}
 
 sub get_student_data_from_performance_cache {
     my ($sname,$sdom,$symb,$courseid)=@_;
@@ -1487,22 +1765,13 @@ sub get_problem_statistics {
     #
     $request = 'SELECT MAX(tries),MIN(tries) FROM '.$stats_table.
         ' WHERE awarded>0';
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($max,$min) = &execute_SQL_request($dbh,$request);
     #
     $request = 'SELECT SUM(awarded) FROM '.$stats_table;
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($Solved) = &execute_SQL_request($dbh,$request);
     #
     $request = 'SELECT SUM(awarded) FROM '.$stats_table.
         " WHERE solved='correct_by_override'";
-    if (defined($time_requirements)) {
-        $request .= ' AND '.$time_requirements;
-    }
     my ($solved) = &execute_SQL_request($dbh,$request);
     #
     $Solved -= $solved;
@@ -1585,7 +1854,7 @@ sub populate_weight_table {
     my @resources;
     foreach my $seq (@sequences) {
         push(@resources,$navmap->retrieveResources($seq,
-                                                   sub {shift->is_problem();},
+                                                   sub {shift->is_gradable();},
                                                    0,0,0));
     }
     if (! scalar(@resources)) {
@@ -1743,7 +2012,7 @@ sub rank_students_by_scores_on_resources
         $limits =~ s/( AND )$//;   # Remove extra conjunction
         $request .= "WHERE $limits";
     } 
-    $request .= " $award_clause GROUP BY a.student_id ORDER BY score";
+    $request .= " $award_clause GROUP BY a.student_id ORDER BY score, b.student";
     #&Apache::lonnet::logthis('request = '.$/.$request);
     my $sth = $dbh->prepare($request) or die "Can't prepare $request";
     $sth->execute();
@@ -2294,14 +2563,15 @@ sub CL_STATUS   { return 7; }
 sub CL_TYPE     { return 8; }
 sub CL_LOCKEDTYPE   { return 9; }
 sub CL_CREDITS  { return 10; }
-sub CL_GROUP    { return 11; }
-sub CL_PERMANENTEMAIL { return 12; }
-sub CL_ROLE     { return 13; }
-sub CL_EXTENT   { return 14; }
-sub CL_PHOTO   { return 15; }
-sub CL_THUMBNAIL { return 16; }
-sub CL_AUTHORQUOTA { return 17; }
-sub CL_AUTHORUSAGE { return 18; }
+sub CL_INSTSEC { return 11; }
+sub CL_GROUP    { return 12; }
+sub CL_PERMANENTEMAIL { return 13; }
+sub CL_ROLE     { return 14; }
+sub CL_EXTENT   { return 15; }
+sub CL_PHOTO   { return 16; }
+sub CL_THUMBNAIL { return 17; }
+sub CL_AUTHORQUOTA { return 18; }
+sub CL_AUTHORUSAGE { return 19; }
 
 sub get_classlist {
     my ($cdom,$cnum) = @_;
@@ -2321,9 +2591,9 @@ sub get_classlist {
         }
         my ($sname,$sdom) = split(/:/,$student);
         my @Values = split(/:/,$info);
-        my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits);
+        my ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec);
         if (@Values > 2) {
-            ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits) = @Values;
+            ($end,$start,$id,$section,$fullname,$type,$lockedtype,$credits,$instsec) = @Values;
         } else { # We have to get the data ourselves
             ($end,$start) = @Values;
             $section = &Apache::lonnet::getsection($sdom,$sname,$cid);
@@ -2363,12 +2633,12 @@ sub get_classlist {
         }
         $classlist{$student} = 
             [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
-             $lockedtype,$credits];
+             $lockedtype,$credits,$instsec];
     }
     if (wantarray()) {
         return (\%classlist,['domain','username','end','start','id',
                              'section','fullname','status','type',
-                             'lockedtype','credits']);
+                             'lockedtype','credits','instsec']);
     } else {
         return \%classlist;
     }
@@ -2989,7 +3259,7 @@ Inputs: $starttime, $endtime, $table
 Returns: $time_limits
 
 
-=item C<&limit_by_section_and_status()C<
+=item C<&limit_by_section_and_status()C>
 
 Build SQL WHERE condition which limits the data collected by section and
 student status.
@@ -3099,7 +3369,7 @@ $env{'course.'.$cid.'.domain'}, and $env
 Returns a reference to a hash which contains:
  keys    '$sname:$sdom'
  values  [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type,
-          $lockedtype,$credits]
+          $lockedtype,$credits,$instsec]
 
 The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used
 as indices into the returned list to future-proof clients against