--- loncom/interface/loncoursedata.pm 2018/03/14 17:07:29 1.201.2.2 +++ loncom/interface/loncoursedata.pm 2021/03/05 17:18:40 1.201.2.5 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: loncoursedata.pm,v 1.201.2.2 2018/03/14 17:07:29 raeburn Exp $ +# $Id: loncoursedata.pm,v 1.201.2.5 2021/03/05 17:18:40 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -777,6 +777,8 @@ sub symb_is_for_task { return ($symb =~ /\.task$/); } +my $requested_max_packet = 0; +my $max_allowed_packet; sub update_full_student_data { my ($sname,$sdom,$courseid) = @_; @@ -964,9 +966,26 @@ sub update_full_student_data { } ## ## Store the response data - $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table. + my $store_prefix = 'INSERT IGNORE INTO '.$fulldump_response_table. ' VALUES '."\n"; $store_rows = 0; + unless ($requested_max_packet) { + (undef,$max_allowed_packet) = $dbh->selectrow_array( + qq{show variables LIKE ? }, + undef, + "max_allowed_packet"); + if ($max_allowed_packet !~ /^\d+$/) { + $max_allowed_packet = ''; + } + $requested_max_packet = 1; + } + my @store_values = (); + my $curr_values = ''; + my $curr_length = 0; + my ($max_values); + if ($max_allowed_packet) { + $max_values = $max_allowed_packet - length($store_prefix); + } while (my ($symb_id,$hash1) = each (%$respdata)) { while (my ($part_id,$hash2) = each (%$hash1)) { while (my ($resp_id,$hash3) = each (%$hash2)) { @@ -978,7 +997,7 @@ sub update_full_student_data { # and odd number of '\' cause insert errors to occur. # Best trap this somehow... $submission = $dbh->quote($submission); - $store_command .= "('". + my $sql_values = "('". join("','",$symb_id,$part_id, $resp_id,$student_id, $transaction, @@ -986,20 +1005,51 @@ sub update_full_student_data { $data->{'response_specific'}, $data->{'response_specific_value'}, $data->{'response_specific_2'}, - $data->{'response_specific_value_2'}). - "',".$submission."),"; + $data->{'response_specific_value_2'})."',"; + if ($max_values) { + my $length = length($sql_values) + length($submission."),"); + if ($length > $max_values) { + &Apache::lonnet::logthis("SQL responsedata insert for student: $sname would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id, part_id: $part_id, resp_id: $resp_id"); + &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + $sql_values .= $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),"; + $length = length($sql_values); + &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission"); + &Apache::lonnet::logthis("See grading interface for the actual submission"); + } else { + $sql_values .= $submission."),"; + } + if ($length + $curr_length > $max_values) { + push(@store_values,$curr_values); + $curr_values = $sql_values; + $curr_length = $length; + } else { + $curr_values .= $sql_values; + $curr_length += $length; + } + } else { + $curr_values .= $sql_values.$submission."),"; + } $store_rows++; } } } } if ($store_rows) { - chop($store_command); - $dbh->do($store_command); - if ($dbh->err) { - $returnstatus = 'error saving response data'; - &Apache::lonnet::logthis('insert error '.$dbh->errstr()); - &Apache::lonnet::logthis("While attempting\n".$store_command); + if ($curr_values ne '') { + push(@store_values,$curr_values); + } + foreach my $item (@store_values) { + chop($item); + if ($item ne '') { + $dbh->do($store_prefix.$item); + if ($dbh->err) { + $returnstatus = 'error saving response data'; + &Apache::lonnet::logthis('insert error '.$dbh->errstr()); + &Apache::lonnet::logthis("While attempting\n".$store_prefix.$item); + last; + } + } } } ## @@ -1012,7 +1062,7 @@ sub update_full_student_data { $returnstatus = 'error saving current data:'.$status; } elsif ($status ne 'okay') { $returnstatus .= ' error saving current data:'.$status; - } + } ## ## Update the students time...... if ($returnstatus eq 'okay') { @@ -1102,12 +1152,33 @@ sub store_student_data { my $starttime = Time::HiRes::time; my $elapsed = 0; my $rows_stored; - my $store_parameters_command = 'INSERT IGNORE INTO '.$parameters_table. + my $store_parameters_prefix = 'INSERT IGNORE INTO '.$parameters_table. ' VALUES '."\n"; my $num_parameters = 0; - my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table. + my $store_performance_prefix = 'INSERT IGNORE INTO '.$performance_table. ' VALUES '."\n"; return ('error',undef) if (! defined($dbh)); + unless ($requested_max_packet) { + (undef,$max_allowed_packet) = $dbh->selectrow_array( + qq{show variables LIKE ? }, + undef, + "max_allowed_packet"); + if ($max_allowed_packet !~ /^\d+$/) { + $max_allowed_packet = ''; + } + $requested_max_packet = 1; + } + my @store_parameters_values = (); + my $curr_params_values = ''; + my $curr_params_length = 0; + my @store_performance_values = (); + my $curr_perf_values = ''; + my $curr_perf_length = 0; + my ($max_param,$max_perf); + if ($max_allowed_packet) { + $max_param = $max_allowed_packet - length($store_parameters_prefix); + $max_perf = $max_allowed_packet - length($store_performance_prefix); + } while (my ($current_symb,$param_hash) = each(%{$student_data})) { # # make sure the symb is set up properly @@ -1120,10 +1191,39 @@ sub store_student_data { $symb_id,$student_id, $parameter)."',". $dbh->quote($value)."),\n"; - $num_parameters ++; if ($sql_parameter !~ /''/) { - $store_parameters_command .= $sql_parameter; + if ($max_param) { + my $length = length($sql_parameter); + if ($length > $max_param) { + &Apache::lonnet::logthis("SQL parameter insert for student: $sname for parameter: $parameter would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + if ($parameter =~ /\.submission$/) { + $sql_parameter = "('".join("','", + $symb_id,$student_id, + $parameter)."',". + $dbh->quote('WARNING: Submission too large -- see grading interface for actual submission')."),\n"; + $length = length($sql_parameter); + &Apache::lonnet::logthis("Placeholder inserted instead of value of actual submission"); + &Apache::lonnet::logthis("See grading interface for the actual submission"); + } else { + &Apache::lonnet::logthis("Skipping this item"); + next; + } + } + if ($length + $curr_params_length > $max_param) { + push(@store_parameters_values,$curr_params_values); + $curr_params_values = $sql_parameter; + $curr_params_length = $length; + } else { + $curr_params_values .= $sql_parameter; + $curr_params_length += $length; + } + } else { + $curr_params_values .= $sql_parameter; + } #$rows_stored++; + $num_parameters ++; } } } @@ -1165,31 +1265,66 @@ sub store_student_data { "('".join("','",$symb_id,$student_id,$part_id,$part, $solved,$tries,$awarded,$award, $awarddetail,$timestamp)."'),\n"; - $store_performance_command .= $sql_performance; + if ($max_perf) { + my $length = length($sql_performance); + if ($length > $max_perf) { + &Apache::lonnet::logthis("SQL performance insert for student: $sname would exceed max_allowed_packet size"); + &Apache::lonnet::logthis("symb_id: $symb_id"); + &Apache::lonnet::logthis("Skipping this item. You may want to increase the max_allowed_packet size from the current: $max_allowed_packet"); + next; + } else { + if ($length + $curr_perf_length > $max_perf) { + push(@store_performance_values,$curr_perf_values); + $curr_perf_values = $sql_performance; + $curr_perf_length = $length; + } else { + $curr_perf_values .= $sql_performance; + $curr_perf_length += $length; + } + } + } else { + $curr_perf_values .= $sql_performance; + } $rows_stored++; } } + if ($curr_params_values ne '') { + push(@store_parameters_values,$curr_params_values); + } + if ($curr_perf_values ne '') { + push(@store_performance_values,$curr_perf_values); + } if (! $rows_stored) { return ($returnstatus, undef); } - $store_parameters_command =~ s|,\n$||; - $store_performance_command =~ s|,\n$||; my $start = Time::HiRes::time; - $dbh->do($store_performance_command); - if ($dbh->err()) { - &Apache::lonnet::logthis('performance bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_performance_command); - $returnstatus = 'error: unable to insert performance into database'; - return ($returnstatus,$student_data); + foreach my $item (@store_performance_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_performance_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('performance insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_performance_prefix.$item); + $returnstatus = 'error: unable to insert performance into database'; + return ($returnstatus,$student_data); + } + } } - $dbh->do($store_parameters_command) if ($num_parameters>0); - if ($dbh->err()) { - &Apache::lonnet::logthis('parameters bigass insert error:'. - $dbh->errstr()); - &Apache::lonnet::logthis('command = '.$/.$store_parameters_command); - &Apache::lonnet::logthis('rows_stored = '.$rows_stored); - &Apache::lonnet::logthis('student_id = '.$student_id); - $returnstatus = 'error: unable to insert parameters into database'; - return ($returnstatus,$student_data); + if ($num_parameters > 0) { + foreach my $item (@store_parameters_values) { + $item =~ s|,\n$||; + if ($item ne '') { + $dbh->do($store_parameters_prefix.$item); + if ($dbh->err()) { + &Apache::lonnet::logthis('parameters insert error:'. + $dbh->errstr()); + &Apache::lonnet::logthis('command = '.$/.$store_parameters_prefix.$item); + &Apache::lonnet::logthis('rows_stored = '.$rows_stored); + &Apache::lonnet::logthis('student_id = '.$student_id); + $returnstatus = 'error: unable to insert parameters into database'; + return ($returnstatus,$student_data); + } + } + } } $elapsed += Time::HiRes::time - $start; return ($returnstatus,$student_data);