--- loncom/interface/statistics/lonproblemstatistics.pm 2002/11/26 16:59:03 1.37
+++ loncom/interface/statistics/lonproblemstatistics.pm 2003/03/26 21:55:18 1.46
@@ -1,7 +1,6 @@
# The LearningOnline Network with CAPA
-# (Publication Handler
#
-# $Id: lonproblemstatistics.pm,v 1.37 2002/11/26 16:59:03 minaeibi Exp $
+# $Id: lonproblemstatistics.pm,v 1.46 2003/03/26 21:55:18 matthew Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -26,12 +25,6 @@
# http://www.lon-capa.org/
#
# (Navigate problems for statistical reports
-# YEAR=2001
-# 5/5,7/9,7/25/1,8/11,9/13,9/26,10/5,10/9,10/22,10/26 Behrouz Minaei
-# 11/1,11/4,11/16,12/14,12/16,12/18,12/20,12/31 Behrouz Minaei
-# YEAR=2002
-# 1/22,2/1,2/6,2/25,3/2,3/26,4/7,5/6 Behrouz Minaei
-# 5/12,5/26,7/16,7/29,8/5,10/31 Behrouz Minaei
#
###
@@ -41,467 +34,716 @@ use strict;
use Apache::lonnet();
use Apache::lonhtmlcommon;
use Apache::loncoursedata;
-use GDBM_File;
+use Apache::lonstatistics;
+use Spreadsheet::WriteExcel;
+#######################################################
+#######################################################
-sub InitializeProblemStatistics {
- my ($cacheDB, $students, $courseID, $c, $r)=@_;
- my %cache;
-
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- $r->print('Unable to tie database1.');
- return ('ERROR', undef);
- }
-
- # Remove students who don't have the proper section.
- my @sectionsSelected = split(':',$cache{'sectionsSelected'});
- for(my $studentIndex=((scalar @$students)-1); $studentIndex>=0;
- $studentIndex--) {
- my $value = $cache{$students->[$studentIndex].':section'};
- my $found = 0;
- foreach (@sectionsSelected) {
- if($_ eq 'none') {
- if($value eq '' || !defined($value) || $value eq ' ') {
- $found = 1;
- last;
- }
- } else {
- if($value eq $_) {
- $found = 1;
- last;
- }
- }
- }
- if($found == 0) {
- splice(@$students, $studentIndex, 1);
- }
- }
+sub CreateInterface {
+ my $Str = '';
+ $Str .= '
'."\n";
+ $Str .= '';
+ $Str .= 'Sections ';
+ $Str .= 'Sequences and Folders ';
+ $Str .= 'Output ';
+ $Str .= ' '."\n";
+ #
+ $Str .= ''."\n";
+ $Str .= &Apache::lonstatistics::SectionSelect('Section','multiple',5);
+ $Str .= ' ';
+ #
+ my $only_seq_with_assessments = sub {
+ my $s=shift;
+ if ($s->{'num_assess'} < 1) {
+ return 0;
+ } else {
+ return 1;
+ }
+ };
+ $Str .= &Apache::lonstatistics::MapSelect('Maps','multiple,all',5,
+ $only_seq_with_assessments);
+ $Str .= ' '."\n";
+ $Str .= &CreateAndParseOutputSelector();
+ $Str .= ' '."\n";
+ $Str .= '
'."\n";
+ return $Str;
+}
- my $isNotCached = 0;
- my $lastStatus = (defined($cache{'StatisticsLastStatus'})) ?
- $cache{'StatisticsLastStatus'} : 'Nothing';
- my $whichStudents = join(':::',sort(@$students));
- if(!defined($cache{'StatisticsCached'}) ||
- $lastStatus ne $cache{'Status'} ||
- $whichStudents ne $cache{'StatisticsWhichStudents'}) {
- $isNotCached = 1;
- }
+#######################################################
+#######################################################
- untie(%cache);
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
- $r->print('Unable to tie database.2');
- return ('ERROR', undef);
- }
- if($isNotCached && defined($cache{'StatisticsCached'})) {
- my @statkeys = split(':::', $cache{'StatisticsKeys'});
- delete $cache{'StatisticsKeys'};
- delete $cache{'StatisticsCached'};
- foreach(@statkeys) {
- delete $cache{$_};
- }
- }
+=pod
- untie(%cache);
- if($isNotCached) {
- &Apache::loncoursedata::DownloadStudentCourseDataSeparate($students,
- 'true',
- $cacheDB,
- 'true',
- 'true',
- $courseID,
- $r, $c);
- }
- if($c->aborted()) { return ('ERROR', undef); }
+=item &CreateAndParseOutputSelector()
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- $r->print('Unable to tie database.3');
- return ('ERROR', undef);
- }
- my $problemData;
- if($isNotCached) {
- ($problemData) = &ExtractStudentData(\%cache, $students);
- &CalculateStatistics($problemData, \%cache, $courseID);
- }
- untie(%cache);
+=cut
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_WRCREAT(),0640)) {
- $r->print('Unable to tie database.4');
- return ('ERROR', undef);
+#######################################################
+#######################################################
+my $output_mode;
+my $show;
+
+my @OutputOptions =
+ (
+ { name => 'problem statistics grouped by sequence',
+ value => 'HTML problem statistics grouped',
+ description => 'Output statistics for the problem parts.',
+ mode => 'html',
+ show => 'grouped',
+ },
+ { name => 'problem statistics ungrouped',
+ value => 'HTML problem statistics ungrouped',
+ description => 'Output statistics for the problem parts.',
+ mode => 'html',
+ show => 'ungrouped',
+ },
+ { name => 'problem statistics, Excel',
+ value => 'Excel problem statistics',
+ description => 'Output statistics for the problem parts '.
+ 'in an Excel workbook',
+ mode => 'excel',
+ show => 'all',
+ },
+ { name => 'Degree of Difficulty Plot',
+ value => 'plot deg diff',
+ description => 'Generate a plot of the degree of difficulty of each '.
+ 'problem part.',
+ mode => 'plot',
+ show => 'deg of diff',
+ },
+ { name => 'Percent Wrong Plot',
+ value => 'plot per wrong',
+ description => 'Generate a plot showing the percent of students who '.
+ 'were unable to complete each problem part',
+ mode => 'plot',
+ show => 'per wrong',
+ },
+ );
+
+sub OutputDescriptions {
+ my $Str = '';
+ $Str .= "Output Modes \n";
+ $Str .= "\n";
+ foreach my $outputmode (@OutputOptions) {
+ $Str .=" ".$outputmode->{'name'}." \n";
+ $Str .=" ".$outputmode->{'description'}." \n";
}
- if($isNotCached) {
- foreach(keys(%$problemData)) {
- $cache{$_} = $problemData->{$_};
+ $Str .= " \n";
+ return $Str;
+}
+
+sub CreateAndParseOutputSelector {
+ my $Str = '';
+ my $elementname = 'outputmode';
+ #
+ # Format for output options is 'mode, restrictions';
+ my $selected = 'html, with links';
+ if (exists($ENV{'form.'.$elementname})) {
+ if (ref($ENV{'form.'.$elementname} eq 'ARRAY')) {
+ $selected = $ENV{'form.'.$elementname}->[0];
+ } else {
+ $selected = $ENV{'form.'.$elementname};
}
- $cache{'StatisticsKeys'} = join(':::', keys(%$problemData));
- $cache{'StatisticsCached'} = 'true';
- $cache{'StatisticsLastStatus'} = $cache{'Status'};
- $cache{'StatisticsWhichStudents'} = $whichStudents;
}
- untie(%cache);
-
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- $r->print('Unable to tie database.5');
- return ('ERROR', undef);
+ #
+ # Set package variables describing output mode
+ $output_mode = 'html';
+ $show = 'all';
+ foreach my $option (@OutputOptions) {
+ next if ($option->{'value'} ne $selected);
+ $output_mode = $option->{'mode'};
+ $show = $option->{'show'};
+ }
+ #
+ # Build the form element
+ $Str = qq//;
+ foreach my $option (@OutputOptions) {
+ $Str .= "\n".' {'value'} eq $selected);
+ $Str .= ">".$option->{'name'}."<\/option>";
}
-
- my $orderedProblems = &SortProblems(\%cache,
- $cache{'ProblemStatisticsSort'},
- $cache{'SortProblems'},
- $cache{'ProblemStatisticsAscend'});
- untie(%cache);
-
- return ('OK', $orderedProblems);
+ $Str .= "\n ";
+ return $Str;
}
-sub BuildProblemStatisticsPage {
- my ($cacheDB, $students, $courseID, $c, $r)=@_;
-
- my @Header = ("Homework Sets Order","#Stdnts","Tries","Mod",
- "Mean","#YES","#yes","%Wrng","DoDiff",
- "S.D.","Skew.","D.F.1st","D.F.2nd");
- my $color=&setbgcolor(0);
- my %cache;
+###############################################
+###############################################
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- $r->print('Unable to tie database.6');
- return;
- }
- my $Ptr = '';
- $Ptr .= '';
- $Ptr .= 'Select Map '."\n";
- $Ptr .= '';
- $Ptr .= &Apache::lonhtmlcommon::MapOptions(\%cache, 'Statistics',
- 'Statistics');
- $Ptr .= ' '."\n";
- $Ptr .= 'Sorting Type: '."\n";
- $Ptr .= ''."\n";
- $Ptr .= &Apache::lonhtmlcommon::AscendOrderOptions(
- $cache{'ProblemStatisticsAscend'},
- 'ProblemStatistics',
- 'Statistics');
- $Ptr .= ' '."\n";
- $Ptr .= 'Select Sections ';
- $Ptr .= ' '."\n";
- $Ptr .= ''."\n";
- my @sections = split(':',$cache{'sectionList'});
- my @sectionsSelected = split(':',$cache{'sectionsSelected'});
- $Ptr .= &Apache::lonhtmlcommon::MultipleSectionSelect(\@sections,
- \@sectionsSelected,
- 'Statistics');
- $Ptr .= ' '."\n";
- $Ptr .= &ProblemStatisticsButtons($cache{'DisplayFormat'},
- $cache{'DisplayLegend'},
- $cache{'SortProblems'});
- $Ptr .= '
';
- if($cache{'DisplayLegend'} eq 'Show Legend') {
- $Ptr .= &ProblemStatisticsLegend();
+###############################################
+###############################################
+sub Gather_Student_Data {
+ my ($r) = @_;
+ my $c = $r->connection();
+ #
+ my @Sequences = &Apache::lonstatistics::Sequences_with_Assess();
+ #
+ my @Students = @Apache::lonstatistics::Students;
+ #
+ # Open the progress window
+ my %prog_state=&Apache::lonhtmlcommon::Create_PrgWin
+ ($r,'Statistics Compilation Status',
+ 'Statistics Compilation Progress', scalar(@Students));
+ #
+ while (my $student = shift @Students) {
+ return if ($c->aborted());
+ my ($status,undef) = &Apache::loncoursedata::ensure_current_data
+ ($student->{'username'},$student->{'domain'},
+ $ENV{'request.course.id'});
+ &Apache::lonhtmlcommon::Increment_PrgWin($r,\%prog_state,
+ 'last student');
}
- $r->print($Ptr);
+ &Apache::lonhtmlcommon::Close_PrgWin($r,\%prog_state);
$r->rflush();
- untie(%cache);
+}
- my ($result, $orderedProblems) =
- &InitializeProblemStatistics($cacheDB, $students, $courseID, $c, $r);
- if($result ne 'OK') {
- return;
- }
+###############################################
+###############################################
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- $r->print('Unable to tie database.6');
+###############################################
+###############################################
+sub BuildProblemStatisticsPage {
+ my ($r,$c)=@_;
+ #
+ $output_mode = 'html';
+ $show = 'grouped';
+ #
+ $r->print(&CreateInterface());
+ $r->print(' ');
+ $r->print(' ');
+ if (! exists($ENV{'form.statsfirstcall'})) {
return;
}
- &BuildStatisticsTable(\%cache, $cache{'DisplayFormat'},
- $cache{'SortProblems'}, $orderedProblems,
- \@Header, $r, $color);
- untie(%cache);
-
+ #
+ &Gather_Student_Data($r);
+ #
+ #
+ if ($output_mode eq 'html') {
+ $r->print("".
+ $ENV{'course.'.$ENV{'request.course.id'}.'.description'}.
+ " \n");
+ $r->print("".localtime(time)." ");
+ $r->rflush();
+ if ($show eq 'grouped') {
+ &output_html_grouped_by_sequence($r);
+ } elsif ($show eq 'ungrouped') {
+ &output_html_ungrouped($r);
+ }
+ } elsif ($output_mode eq 'excel') {
+ $r->print("Preparing Excel Spreadsheet ");
+ &output_excel($r);
+ } elsif ($output_mode eq 'plot') {
+ if ($show eq 'deg of diff') {
+ &plot_statistics($r,'DoDiff');
+ } elsif ($show eq 'per wrong') {
+ &plot_statistics($r,'%Wrng');
+ }
+ } else {
+ $r->print("Not implemented ");
+ }
return;
}
-sub BuildGraphicChart {
- my ($graph,$cacheDB,$courseDescription,$students,$courseID,$r,$c)=@_;
- my %cache;
- my $max;
- unless(tie(%cache,'GDBM_File',$cacheDB,&GDBM_READER(),0640)) {
- return 'Unable to tie database.7';
- }
+###############################################
+###############################################
- my $title = '';
- if($graph eq 'DoDiffGraph') {
- $title = 'Degree-of-Difficulty';
- } else {
- $title = 'Wrong-Percentage';
+###############################################
+###############################################
+sub output_html_grouped_by_sequence {
+ my ($r) = @_;
+ my $problem_num = 0;
+ #$r->print(&ProblemStatisticsLegend());
+ my @Header = ("Title","Part","#Stdnts","Tries","Mod",
+ "Mean","#YES","#yes","%Wrng","DoDiff",
+ "S.D.","Skew.");#,"D.F.1st","D.F.2nd");
+ # #FFFFE6 #EEFFCC #DDFFFF FFDDDD #DDFFDD #FFDDFF
+ foreach my $sequence (&Apache::lonstatistics::Sequences_with_Assess()) {
+ my $show_part = 0;
+ next if ($sequence->{'num_assess'}<1);
+ $r->print("".$sequence->{'title'}." ");
+ $r->print(''."\n");
+ $r->print(''."\n");
+ $r->print(''.
+ join(" ",@Header)." \n");
+ foreach my $resource (@{$sequence->{'contents'}}) {
+ next if ($resource->{'type'} ne 'assessment');
+ foreach my $part (@{$resource->{'parts'}}) {
+ $problem_num++;
+ my ($num,$tries,$mod,$mean,$Solved,$solved,$DegOfDiff,$STD,
+ $SKEW) = &Apache::loncoursedata::get_problem_statistics
+ (undef,$resource->{'symb'},$part,
+ $ENV{'request.course.id'});
+ #
+ $show_part = 1 if ($part ne '0');
+ $part = ' ' if ($part == 0);
+ #
+ my $wrongpercent = 0;
+ if (defined($num) && $num > 0) {
+ $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
+ }
+ my $option = '';
+ $option .= 'no part' if (! $show_part);
+ $r->print(''.&statistics_html_table_data
+ ($resource,$part,$num,$tries,$mod,$mean,$Solved,
+ $solved,$wrongpercent,$DegOfDiff,$STD,$SKEW,
+ $option).
+ " \n");
+ }
+ }
+ $r->print("
\n");
+ $r->print("
\n");
+ $r->rflush();
}
+ #
+ return;
+}
- my $currentSequence = -1;
- my $sortProblems = 'Sort Within Sequence';
+###############################################
+###############################################
- my ($result, $orderedProblems) =
- &InitializeProblemStatistics($cacheDB, $students, $courseID, $c, $r);
- if($result ne 'OK') {
- return;
+###############################################
+###############################################
+sub output_html_ungrouped {
+ my ($r,$option) = @_;
+ #
+ my $problem_num = 0;
+ my $show_container = 0;
+ my $show_part = 0;
+ #$r->print(&ProblemStatisticsLegend());
+ my @Header = ("Title","Part","#Stdnts","Tries","Mod",
+ "Mean","#YES","#yes","%Wrng","DoDiff",
+ "S.D.","Skew");#,"D.F.1st","D.F.2nd");
+ #
+ my $sortby = undef;
+ foreach (@Header) {
+ if ($ENV{'form.sortby'} eq $_) {
+ $sortby = $_;
+ }
+ }
+ if (! defined($sortby) || $sortby eq '') {
+ $sortby = 'Container';
+ }
+ # If there is more than one sequence, list their titles
+ my @Sequences = &Apache::lonstatistics::Sequences_with_Assess();
+ if (@Sequences > 1) {
+ unshift(@Header,"Container");
+ $show_container = 1;
+ }
+ #
+ # If the option for showing the problem number is needed, push that
+ # on the list too
+ if (defined($option) && $option =~ /show probnum/) {
+ unshift(@Header,"P#");
}
-
- my @values = ();
-
- foreach(@$orderedProblems) {
- my ($sequence,$problem,$part)=split(':', $_);
- if($cache{'StatisticsMaps'} ne 'All Maps' &&
- $cache{'StatisticsMaps'} ne $cache{$sequence.':title'}) {
- next;
- }
-
- if( $currentSequence == -1 ||
- ($sortProblems eq 'Sort Within Sequence' &&
- $currentSequence != $sequence)) {
- if($currentSequence != -1) {
- &DrawGraph(\@values,$courseDescription,$title,$max,$r);
- }
- if($sortProblems eq 'Sort Within Sequence') {
- $r->print(''.$cache{$sequence.':title'}.' '."\n");
+ #
+ $r->print(''."\n");
+ $r->rflush();
+ #
+ # Compile the data
+ my @Statsarray;
+ foreach my $sequence (@Sequences) {
+ next if ($sequence->{'num_assess'}<1);
+ foreach my $resource (@{$sequence->{'contents'}}) {
+ next if ($resource->{'type'} ne 'assessment');
+ foreach my $part (@{$resource->{'parts'}}) {
+ $problem_num++;
+ my ($num,$tries,$mod,$mean,$Solved,$solved,$DegOfDiff,$STD,
+ $SKEW) = &Apache::loncoursedata::get_problem_statistics
+ (undef,$resource->{'symb'},$part,
+ $ENV{'request.course.id'});
+ #
+ $show_part = 1 if ($part ne '0');
+ $part = ' ' if ($part == 0);
+ #
+ my $wrongpercent = 0;
+ if (defined($num) && $num > 0) {
+ $wrongpercent=int(10*100*($num-$Solved+$solved)/$num)/10;
+ }
+ push (@Statsarray,
+ { 'sequence' => $sequence,
+ 'resource' => $resource,
+ 'Title' => $resource->{'title'},
+ 'Part' => $part,
+ '#Stdnts' => $num,
+ 'Tries' => $tries,
+ 'Mod' => $mod,
+ 'Mean' => $mean,
+ '#YES' => $Solved,
+ '#yes' => $solved,
+ '%Wrng' => $wrongpercent,
+ 'DoDiff' => $DegOfDiff,
+ 'S.D.' => $STD,
+ 'Skew' => $SKEW,
+ 'problem_num' => $problem_num,
+ });
}
-
- $currentSequence = $sequence;
- @values = ();
- $max=0;
- }
- my $data = 0;
- if($graph eq 'DoDiffGraph') {
- $data = sprintf("%.2f", $cache{$_.':degreeOfDifficulty'}),
+ }
+ }
+ #
+ # Table Headers
+ $r->print(''."\n");
+ my $Str = '';
+ foreach (@Header) {
+ next if ($_ eq 'Part' && !$show_part);
+ # Do not allow sorting on some fields
+ if ($_ eq $sortby || /^(Part|P\#)$/) {
+ $Str .= ''.$_.' ';
} else {
- $data = sprintf("%.1f", $cache{$_.':percentWrong'}),
+ $Str .= ''.
+ ''.
+ $_.' ';
}
- if($max < $data) {
- $max = $data;
+ }
+ $r->print(''.$Str." \n");
+ #
+ # Sort the data
+ my @OutputOrder;
+ if ($sortby eq 'Container') {
+ @OutputOrder = @Statsarray;
+ } else {
+ # $sortby is already defined, so we can charge ahead
+ if ($sortby =~ /^(title|part)$/i) {
+ # Alpha comparison
+ @OutputOrder = sort {
+ lc($a->{$sortby}) cmp lc($b->{$sortby}) ||
+ lc($a->{'Title'}) cmp lc($b->{'Title'}) ||
+ lc($a->{'Part'}) cmp lc($b->{'Part'});
+ } @Statsarray;
+ } else {
+ # Numerical comparison
+ @OutputOrder = sort {
+ my $retvalue = 0;
+ if ($b->{$sortby} eq 'nan') {
+ if ($a->{$sortby} ne 'nan') {
+ $retvalue = -1;
+ } else {
+ $retvalue = 0;
+ }
+ }
+ if ($a->{$sortby} eq 'nan') {
+ if ($b->{$sortby} ne 'nan') {
+ $retvalue = 1;
+ }
+ }
+ if ($retvalue eq '0') {
+ $retvalue = $b->{$sortby} <=> $a->{$sortby} ||
+ lc($a->{'Title'}) <=> lc($b->{'Title'}) ||
+ lc($a->{'Part'}) <=> lc($b->{'Part'});
+ }
+ $retvalue;
+ } @Statsarray;
}
- push(@values, $data);
}
- untie(%cache);
-
- &DrawGraph(\@values,$courseDescription,$title,$max,$r);
+ $option .= ',no part' if (! $show_part);
+ foreach my $row (@OutputOrder) {
+ $r->print('');
+ if (defined($option) && $option =~ /show probnum/) {
+ $r->print(''.$row->{'problem_num'}.' ');
+ }
+ if ($show_container) {
+ $r->print(''
+ .$row->{'sequence'}->{'title'}.' ');
+ }
+ $r->print(&statistics_html_table_data
+ ($row->{'resource'},$row->{'Part'},$row->{'#Stdnts'},
+ $row->{'Tries'},$row->{'Mod'},$row->{'Mean'},
+ $row->{'#YES'},$row->{'#yes'},$row->{"\%Wrng"},
+ $row->{'DoDiff'},$row->{'S.D.'},$row->{'Skew'},
+ $option));
+ $r->print(" \n");
+ }
+ $r->print("
\n");
+ $r->print("
\n");
+ $r->rflush();
+ #
return;
}
-sub DrawGraph {
- my ($values,$courseDescription,$title,$Max,$r)=@_;
- my $sendValues = join(',', @$values);
- my $sendCount = scalar(@$values);
- $r->print(" The Maximum Value is: $Max");
- if ( $Max > 1 ) {
- if ($Max % 10) {
- if ( int($Max) < $Max ) {
- $Max++;
- $Max = int($Max);
- }
- }
- #(10 - $Max % 10);
- } else { $Max = 1; }
-
- my @GData = ('','Problem_number',$title,$Max,$sendCount,$sendValues);
+###############################################
+###############################################
-# $r->print(''."\n");
- $r->print(' '."\n");
- $r->print(' ');
-# $r->print('