--- loncom/interface/loncoursedata.pm 2003/09/26 19:23:14 1.90 +++ loncom/interface/loncoursedata.pm 2004/03/09 21:42:01 1.112.2.1 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: loncoursedata.pm,v 1.90 2003/09/26 19:23:14 matthew Exp $ +# $Id: loncoursedata.pm,v 1.112.2.1 2004/03/09 21:42:01 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -73,8 +73,6 @@ and/or itself. =item &get_sequence_assessment_data() -AT THIS TIME THE USE OF THIS FUNCTION IS *NOT* RECOMMENDED - Use lonnavmaps to build a data structure describing the order and assessment contents of each sequence in the current course. @@ -190,6 +188,10 @@ sub get_sequence_assessment_data { $title =~ s/\:/\&\#058;/g; $symb = $curRes->symb(); $src = $curRes->src(); + # Grab the filename if there is not title available + if (! defined($title) || $title eq '') { + ($title) = ($src=~ m:/([^/]*)$:); + } my $parts = $curRes->parts(); my %partdata; foreach my $part (@$parts) { @@ -197,6 +199,10 @@ sub get_sequence_assessment_data { my @Ids = $curRes->responseIds($part); $partdata{$part}->{'ResponseTypes'}= \@Responses; $partdata{$part}->{'ResponseIds'} = \@Ids; + # Count how many responses of each type there are in this part + foreach (@Responses) { + $partdata{$part}->{$_}++; + } } my $assessment = { title => $title, src => $src, @@ -456,7 +462,7 @@ The response table holds data (documente associated with a particular response id which is stored when a student attempts a problem. The following are the columns of the table, in order: 'symb_id','part_id','response_id','student_id','transaction','tries', -'awarddetail', 'awarded','response_specific' (data particular to the response +'awarddetail', 'response_specific' (data particular to the response type), 'response_specific_value', and 'submission (the text of the students submission). The primary key is based on the first five columns listed above. @@ -707,13 +713,8 @@ sub init_dbs { { name => 'transaction', type => 'MEDIUMINT UNSIGNED', restrictions => 'NOT NULL' }, - { name => 'tries', - type => 'SMALLINT UNSIGNED', - restrictions => 'NOT NULL' }, { name => 'awarddetail', type => 'TINYTEXT' }, - { name => 'awarded', - type => 'TINYTEXT' }, # { name => 'message', # type => 'CHAR' }, { name => 'response_specific', @@ -1040,7 +1041,8 @@ sub get_student_id { $have_read_student_table = 1; } if (! exists($ids_by_student{$student})) { - &Apache::lonmysql::store_row($student_table,[undef,$student,undef]); + &Apache::lonmysql::store_row($student_table, + [undef,$student,undef]); undef(%ids_by_student); my @Result = &Apache::lonmysql::get_rows($student_table); foreach (@Result) { @@ -1070,6 +1072,36 @@ sub get_student { =pod +=item &clear_internal_caches() + +Causes the internal caches used in get_student_id, get_student, +get_symb_id, get_symb, get_part_id, and get_part to be undef'd. + +Needs to be called before the first operation with the MySQL database +for a given Apache request. + +=cut + +################################################ +################################################ +sub clear_internal_caches { + $have_read_part_table = 0; + undef(%ids_by_part); + undef(%parts_by_id); + $have_read_symb_table = 0; + undef(%ids_by_symb); + undef(%symbs_by_id); + $have_read_student_table = 0; + undef(%ids_by_student); + undef(%students_by_id); +} + + +################################################ +################################################ + +=pod + =item &update_full_student_data($sname,$sdom,$courseid) Does a lonnet::dump on a student to populate the courses tables. @@ -1147,7 +1179,7 @@ sub update_full_student_data { # We can deal with 'timestamp' right away my @timestamp_storage = ($symb_id,$student_id, $transaction,$value); - my $store_command = 'INSERT INTO '.$fulldump_timestamp_table. + my $store_command = 'INSERT IGNORE INTO '.$fulldump_timestamp_table. " VALUES ('".join("','",@timestamp_storage)."');"; $dbh->do($store_command); if ($dbh->err()) { @@ -1183,9 +1215,7 @@ sub update_full_student_data { } # deal with response specific data if (defined($resp_id) && - $field =~ /^(tries| - awarddetail| - awarded| + $field =~ /^(awarddetail| submission| submissiongrading| molecule)$/x) { @@ -1195,10 +1225,7 @@ sub update_full_student_data { # However, there is one wrinkle: submissions which end in # and odd number of '\' cause insert errors to occur. # Best trap this somehow... - my ($offensive_string) = ($value =~ /(\\+)$/); - if (length($offensive_string) % 2) { - $value =~ s/\\$/\\\\/; - } + $value = $dbh->quote($value); } if ($field eq 'submissiongrading' || $field eq 'molecule') { @@ -1212,7 +1239,7 @@ sub update_full_student_data { } ## ## Store the part data - my $store_command = 'INSERT INTO '.$fulldump_part_table. + my $store_command = 'INSERT IGNORE INTO '.$fulldump_part_table. ' VALUES '."\n"; my $store_rows = 0; while (my ($symb_id,$hash1) = each (%$partdata)) { @@ -1240,22 +1267,28 @@ sub update_full_student_data { } ## ## Store the response data - $store_command = 'INSERT INTO '.$fulldump_response_table. + $store_command = 'INSERT IGNORE INTO '.$fulldump_response_table. ' VALUES '."\n"; $store_rows = 0; while (my ($symb_id,$hash1) = each (%$respdata)) { while (my ($part_id,$hash2) = each (%$hash1)) { while (my ($resp_id,$hash3) = each (%$hash2)) { while (my ($transaction,$data) = each (%$hash3)) { - $store_command .= "('".join("','",$symb_id,$part_id, - $resp_id,$student_id, - $transaction, - $data->{'tries'}, - $data->{'awarddetail'}, - $data->{'awarded'}, - $data->{'response_specific'}, - $data->{'response_specific_value'}, - $data->{'submission'})."'),"; + my $submission = $data->{'submission'}; + # We have to be careful with user supplied input. + # most of the time we are okay because it is escaped. + # However, there is one wrinkle: submissions which end in + # and odd number of '\' cause insert errors to occur. + # Best trap this somehow... + $submission = $dbh->quote($submission); + $store_command .= "('". + join("','",$symb_id,$part_id, + $resp_id,$student_id, + $transaction, + $data->{'awarddetail'}, + $data->{'response_specific'}, + $data->{'response_specific_value'}). + "',".$submission."),"; $store_rows++; } } @@ -1353,8 +1386,10 @@ sub update_student_data { my @Results = &store_student_data($sname,$sdom,$courseid,\%student_data); # # Set the students update time - &Apache::lonmysql::replace_row($studentdata_table, + if ($Results[0] eq 'okay') { + &Apache::lonmysql::replace_row($studentdata_table, [$student_id,$time_of_retrieval,undef,undef,undef]); + } # return @Results; } @@ -1379,10 +1414,10 @@ sub store_student_data { my $starttime = Time::HiRes::time; my $elapsed = 0; my $rows_stored; - my $store_parameters_command = 'INSERT INTO '.$parameters_table. + my $store_parameters_command = 'INSERT IGNORE INTO '.$parameters_table. ' VALUES '."\n"; my $num_parameters = 0; - my $store_performance_command = 'INSERT INTO '.$performance_table. + my $store_performance_command = 'INSERT IGNORE INTO '.$performance_table. ' VALUES '."\n"; return ('error',undef) if (! defined($dbh)); while (my ($current_symb,$param_hash) = each(%{$student_data})) { @@ -1433,6 +1468,13 @@ sub store_student_data { chop $store_performance_command; chop $store_performance_command; my $start = Time::HiRes::time; + $dbh->do($store_performance_command); + if ($dbh->err()) { + &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr()); + &Apache::lonnet::logthis('command = '.$store_performance_command); + $returnstatus = 'error: unable to insert performance into database'; + return ($returnstatus,$student_data); + } $dbh->do($store_parameters_command) if ($num_parameters>0); if ($dbh->err()) { &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr()); @@ -1442,13 +1484,6 @@ sub store_student_data { $returnstatus = 'error: unable to insert parameters into database'; return ($returnstatus,$student_data); } - $dbh->do($store_performance_command); - if ($dbh->err()) { - &Apache::lonnet::logthis(' bigass insert error:'.$dbh->errstr()); - &Apache::lonnet::logthis('command = '.$store_performance_command); - $returnstatus = 'error: unable to insert performance into database'; - return ($returnstatus,$student_data); - } $elapsed += Time::HiRes::time - $start; return ($returnstatus,$student_data); } @@ -1678,6 +1713,33 @@ sub get_student_data_from_performance_ca $studentdata->{$symb}->{$base.'.awarddetail'} = $awarddetail; $studentdata->{$symb}->{'timestamp'} = $time if (defined($time) && $time ne ''); } + ## Get misc parameters + $request = 'SELECT c.symb,a.parameter,a.value '. + "FROM $student_table AS b ". + "LEFT JOIN $parameters_table AS a ON b.student_id=a.student_id ". + "LEFT JOIN $symb_table AS c ON c.symb_id = a.symb_id ". + "WHERE student='$student'"; + if (defined($symb) && $symb ne '') { + $request .= " AND c.symb=".$dbh->quote($symb); + } + $sth = $dbh->prepare($request); + $sth->execute(); + if ($sth->err()) { + &Apache::lonnet::logthis("Unable to execute MySQL request:"); + &Apache::lonnet::logthis("\n".$request."\n"); + &Apache::lonnet::logthis("error is:".$sth->errstr()); + if (defined($symb) && $symb ne '') { + $studentdata = $studentdata->{$symb}; + } + return $studentdata; + } + # + foreach my $row (@{$sth->fetchall_arrayref}) { + $rows_retrieved++; + my ($symb,$parameter,$value) = (@$row); + $studentdata->{$symb}->{$parameter} = $value; + } + # if (defined($symb) && $symb ne '') { $studentdata = $studentdata->{$symb}; } @@ -1825,6 +1887,7 @@ sub get_problem_statistics { return if (! defined($symb) || ! defined($part)); $courseid = $ENV{'request.course.id'} if (! defined($courseid)); # + &setup_table_names($courseid); my $symb_id = &get_symb_id($symb); my $part_id = &get_part_id($part); my $stats_table = $courseid.'_problem_stats'; @@ -1832,21 +1895,10 @@ sub get_problem_statistics { my $dbh = &Apache::lonmysql::get_dbh(); return undef if (! defined($dbh)); # - # A) Number of Students attempting problem - # B) Total number of tries of students attempting problem - # C) Mod (largest number of tries for solving the problem) - # D) Mean (average number of tries for solving the problem) - # E) Number of students to solve the problem - # F) Number of students to solve the problem by override - # G) Number of students unable to solve the problem - # H) Degree of difficulty : 1-(E+F)/B - # I) Standard deviation of number of tries - # J) Skew of tries: sqrt(sum(Xi-D)^3)/A - # $dbh->do('DROP TABLE '.$stats_table); # May return an error my $request = 'CREATE TEMPORARY TABLE '.$stats_table. - ' SELECT student_id,solved,award,tries FROM '.$performance_table. + ' SELECT student_id,solved,award,awarded,tries FROM '.$performance_table. ' WHERE symb_id='.$symb_id.' AND part_id='.$part_id; if (defined($students)) { $request .= ' AND ('. @@ -1858,16 +1910,22 @@ sub get_problem_statistics { } # &Apache::lonnet::logthis($request); $dbh->do($request); +# &Apache::lonnet::logthis('request = '.$/.$request); + $request = 'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) '. + 'FROM '.$stats_table; my ($num,$tries,$mod,$mean,$STD) = &execute_SQL_request - ($dbh, - 'SELECT COUNT(*),SUM(tries),MAX(tries),AVG(tries),STD(tries) FROM '. - $stats_table); - my ($Solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '. - $stats_table. - " WHERE solved='correct_by_student'"); - my ($solved) = &execute_SQL_request($dbh,'SELECT COUNT(tries) FROM '. - $stats_table. - " WHERE solved='correct_by_override'"); + ($dbh,$request); +# &Apache::lonnet::logthis('request = '.$/.$request); + $request = 'SELECT SUM(awarded) FROM '.$stats_table; + my ($Solved) = &execute_SQL_request($dbh,$request); +# &Apache::lonnet::logthis('request = '.$/.$request); + $request = 'SELECT SUM(awarded) FROM '.$stats_table. + " WHERE solved='correct_by_override'"; +# &Apache::lonnet::logthis('request = '.$/.$request); + my ($solved) = &execute_SQL_request($dbh,$request); +# $Solved = int($Solved); +# $solved = int($solved); + # $num = 0 if (! defined($num)); $tries = 0 if (! defined($tries)); $mod = 0 if (! defined($mod)); @@ -1887,7 +1945,7 @@ sub get_problem_statistics { $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10; } # - $dbh->do('DROP TABLE '.$stats_table); # May return an error +# $dbh->do('DROP TABLE '.$stats_table); # May return an error # # Store in metadata # @@ -1931,6 +1989,143 @@ sub execute_SQL_request { return (); } +sub get_student_data { + my ($students,$courseid) = @_; + $courseid = $ENV{'request.course.id'} if (! defined($courseid)); + &setup_table_names($courseid); + my $dbh = &Apache::lonmysql::get_dbh(); + return undef if (! defined($dbh)); + my $request = 'SELECT '. + 'student_id, student '. + 'FROM '.$student_table; + if (defined($students)) { + $request .= ' WHERE ('. + join(' OR ', map {'student_id='. + &get_student_id($_->{'username'}, + $_->{'domain'}) + } @$students + ).')'; + } + $request.= ' ORDER BY student_id'; + my $sth = $dbh->prepare($request); + $sth->execute(); + if ($dbh->err) { + &Apache::lonnet::logthis('error = '.$dbh->errstr()); + return undef; + } + my $dataset = $sth->fetchall_arrayref(); + if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) { + return $dataset; + } +} + +sub RD_student_id { return 0; } +sub RD_awarddetail { return 1; } +sub RD_response_eval { return 2; } +sub RD_submission { return 3; } +sub RD_timestamp { return 4; } +sub RD_tries { return 5; } +sub RD_sname { return 6; } + +sub get_response_data { + my ($students,$symb,$response,$courseid) = @_; + return undef if (! defined($symb) || + ! defined($response)); + $courseid = $ENV{'request.course.id'} if (! defined($courseid)); + # + &setup_table_names($courseid); + my $symb_id = &get_symb_id($symb); + my $response_id = &get_part_id($response); + # + my $dbh = &Apache::lonmysql::get_dbh(); + return undef if (! defined($dbh)); + my $request = 'SELECT '. + 'a.student_id, a.awarddetail, a.response_specific_value, '. + 'a.submission, b.timestamp, c.tries, d.student '. + 'FROM '.$fulldump_response_table.' AS a '. + 'LEFT JOIN '.$fulldump_timestamp_table.' AS b '. + 'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '. + 'a.transaction = b.transaction '. + 'LEFT JOIN '.$fulldump_part_table.' AS c '. + 'ON a.symb_id=c.symb_id AND a.student_id=c.student_id AND '. + 'a.part_id=c.part_id AND a.transaction = c.transaction '. + 'LEFT JOIN '.$student_table.' AS d '. + 'ON a.student_id=d.student_id '. + 'WHERE '. + 'a.symb_id='.$symb_id.' AND a.response_id='.$response_id; + if (defined($students)) { + $request .= ' AND ('. + join(' OR ', map {'a.student_id='. + &get_student_id($_->{'username'}, + $_->{'domain'}) + } @$students + ).')'; + } + $request .= ' ORDER BY b.timestamp'; +# &Apache::lonnet::logthis("request =\n".$request); + my $sth = $dbh->prepare($request); + $sth->execute(); + if ($dbh->err) { + &Apache::lonnet::logthis('error = '.$dbh->errstr()); + return undef; + } + my $dataset = $sth->fetchall_arrayref(); + if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) { + # Clear the \'s from around the submission + for (my $i =0;$i[$i]->[3] =~ s/(\'$|^\')//g; + } + return $dataset; + } +} + +sub RT_student_id { return 0; } +sub RT_awarded { return 1; } +sub RT_tries { return 2; } +sub RT_timestamp { return 3; } + +sub get_response_time_data { + my ($students,$symb,$part,$courseid) = @_; + return undef if (! defined($symb) || + ! defined($part)); + $courseid = $ENV{'request.course.id'} if (! defined($courseid)); + # + &setup_table_names($courseid); + my $symb_id = &get_symb_id($symb); + my $part_id = &get_part_id($part); + # + my $dbh = &Apache::lonmysql::get_dbh(); + return undef if (! defined($dbh)); + my $request = 'SELECT '. + 'a.student_id, a.awarded, a.tries, b.timestamp '. + 'FROM '.$fulldump_part_table.' AS a '. + 'NATURAL LEFT JOIN '.$fulldump_timestamp_table.' AS b '. +# 'ON a.symb_id=b.symb_id AND a.student_id=b.student_id AND '. +# 'a.transaction = b.transaction '. + 'WHERE '. + 'a.symb_id='.$symb_id.' AND a.part_id='.$part_id; + if (defined($students)) { + $request .= ' AND ('. + join(' OR ', map {'a.student_id='. + &get_student_id($_->{'username'}, + $_->{'domain'}) + } @$students + ).')'; + } + $request .= ' ORDER BY b.timestamp'; +# &Apache::lonnet::logthis("request =\n".$request); + my $sth = $dbh->prepare($request); + $sth->execute(); + if ($dbh->err) { + &Apache::lonnet::logthis('error = '.$dbh->errstr()); + return undef; + } + my $dataset = $sth->fetchall_arrayref(); + if (ref($dataset) eq 'ARRAY' && scalar(@$dataset)>0) { + return $dataset; + } + +} ################################################ ################################################ @@ -2032,7 +2227,7 @@ $ENV{'course.'.$cid.'.domain'}, and $ENV Returns a reference to a hash which contains: keys '$sname:$sdom' - values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status] + values [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type] The constant values CL_SDOM, CL_SNAME, CL_END, etc. can be used as indices into the returned list to future-proof clients against @@ -2051,6 +2246,7 @@ sub CL_ID { return 4; } sub CL_SECTION { return 5; } sub CL_FULLNAME { return 6; } sub CL_STATUS { return 7; } +sub CL_TYPE { return 8; } sub get_classlist { my ($cid,$cdom,$cnum) = @_; @@ -2067,9 +2263,9 @@ sub get_classlist { } my ($sname,$sdom) = split(/:/,$student); my @Values = split(/:/,$info); - my ($end,$start,$id,$section,$fullname); + my ($end,$start,$id,$section,$fullname,$type); if (@Values > 2) { - ($end,$start,$id,$section,$fullname) = @Values; + ($end,$start,$id,$section,$fullname,$type) = @Values; } else { # We have to get the data ourselves ($end,$start) = @Values; $section = &Apache::lonnet::getsection($sdom,$sname,$cid); @@ -2106,11 +2302,11 @@ sub get_classlist { $status='Active'; } $classlist{$student} = - [$sdom,$sname,$end,$start,$id,$section,$fullname,$status]; + [$sdom,$sname,$end,$start,$id,$section,$fullname,$status,$type]; } if (wantarray()) { return (\%classlist,['domain','username','end','start','id', - 'section','fullname','status']); + 'section','fullname','status','type']); } else { return \%classlist; }