--- loncom/interface/statistics/lonproblemstatistics.pm 2005/03/10 17:33:57 1.101 +++ loncom/interface/statistics/lonproblemstatistics.pm 2012/03/15 20:53:18 1.121 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: lonproblemstatistics.pm,v 1.101 2005/03/10 17:33:57 matthew Exp $ +# $Id: lonproblemstatistics.pm,v 1.121 2012/03/15 20:53:18 www Exp $ # # Copyright Michigan State University Board of Trustees # @@ -50,8 +50,9 @@ Excel files, and plots. package Apache::lonproblemstatistics; use strict; -use Apache::lonnet(); +use Apache::lonnet; use Apache::loncommon(); +use Apache::lonquickgrades(); use Apache::lonhtmlcommon; use Apache::loncoursedata; use Apache::lonstatistics; @@ -60,6 +61,8 @@ use Apache::lonlocal; use Spreadsheet::WriteExcel; use Apache::lonstathelpers(); use Time::HiRes; +use LONCAPA; + my @StatsArray; my %SeqStat; # keys are symbs, values are hash refs @@ -121,11 +124,11 @@ my @Fields = ( align => 'left', color => '#FFFFE6', special => 'link', - sortable => 'yes', + sortable => 'yes', selectable => 'no', defaultselected => 'yes', }, - { name => 'part', + { name => 'part', title => 'Part', align => 'left', color => '#FFFFE6', @@ -373,7 +376,7 @@ my @SeqFields = ( align => 'left', color => '#FFFFE6', special => 'no', - sortable => 'no', + sortable => 'no', selectable => 'yes', defaultselected => 'no', }, @@ -508,7 +511,7 @@ my @SeqFields = ( long_title => 'KR-21 reliability statistic', selectable => 'yes', defaultselected => 'no', - }, + }, ); my %SelectedFields; @@ -516,35 +519,35 @@ my %SelectedFields; sub parse_field_selection { # # Pull out the defaults - if (! defined($ENV{'form.fieldselections'})) { - $ENV{'form.fieldselections'} = []; + if (! defined($env{'form.fieldselections'})) { + $env{'form.fieldselections'} = []; foreach my $field (@Fields) { next if ($field->{'selectable'} ne 'yes'); if ($field->{'defaultselected'} eq 'yes') { - push(@{$ENV{'form.fieldselections'}},$field->{'name'}); + push(@{$env{'form.fieldselections'}},$field->{'name'}); } } } # # Make sure the data we are plotting is there my %NeededFields; - if (exists($ENV{'form.plot'}) && $ENV{'form.plot'} ne '' && - $ENV{'form.plot'} ne 'none') { - if ($ENV{'form.plot'} eq 'degrees') { + if (exists($env{'form.plot'}) && $env{'form.plot'} ne '' && + $env{'form.plot'} ne 'none') { + if ($env{'form.plot'} eq 'degrees') { $NeededFields{'deg_of_diff'}++; $NeededFields{'deg_of_disc'}++; - } elsif ($ENV{'form.plot'} eq 'tries statistics') { + } elsif ($env{'form.plot'} eq 'tries statistics') { $NeededFields{'mean_tries'}++; $NeededFields{'std_tries'}++; $NeededFields{'problem_num'}++; } else { - $NeededFields{$ENV{'form.plot'}}++; + $NeededFields{$env{'form.plot'}}++; } } # # This should not happen, but in case it does... - if (ref($ENV{'form.fieldselections'}) ne 'ARRAY') { - $ENV{'form.fieldselections'} = [$ENV{'form.fieldselections'}]; + if (ref($env{'form.fieldselections'}) ne 'ARRAY') { + $env{'form.fieldselections'} = [$env{'form.fieldselections'}]; } # # Set the field data and the selected fields (for easier checking) @@ -559,7 +562,7 @@ sub parse_field_selection { $field->{'selected'} = 'yes'; $SelectedFields{$field->{'name'}}++; } - foreach my $selection (@{$ENV{'form.fieldselections'}}) { + foreach my $selection (@{$env{'form.fieldselections'}}) { if ($selection eq $field->{'name'} || $selection eq 'all') { $field->{'selected'} = 'yes'; $SelectedFields{$field->{'name'}}++; @@ -575,7 +578,7 @@ sub parse_field_selection { } sub field_selection_input { - my $Str = ''."\n"; $Str .= ''."\n"; foreach my $field (@Fields) { next if ($field->{'selectable'} ne 'yes'); @@ -608,39 +611,43 @@ sub CreateInterface { &parse_field_selection(); # my $Str = ''; - $Str .= &Apache::lonhtmlcommon::breadcrumbs - (undef,'Overall Problem Statistics','Statistics_Overall_Key'); - $Str .= ''."\n"; - $Str .= ''; - $Str .= ''; - $Str .= ''; - $Str .= ''; - $Str .= ''; - $Str .= ''; - $Str .= ''."\n"; + $Str .= '

'; + $Str .= &Apache::loncommon::start_data_table(); + $Str .= &Apache::loncommon::start_data_table_header_row(); + $Str .= '

'; + $Str .= ''; + $Str .= ''; + $Str .= ''; + $Str .= ''; + $Str .= ''; + $Str .= ''; + $Str .= &Apache::loncommon::end_data_table_header_row(); # - $Str .= ''."\n"; - $Str .= '
'.&mt('Sections').''.&mt('Enrollment Status').''.&mt('Sequences and Folders').''.&mt('Statistics').''. - &Apache::lonstathelpers::limit_by_time_form().'
'.&mt('Sections').''.&mt('Groups').''.&mt('Access Status').''.&mt('Sequences and Folders').''.&mt('Statistics').''.&mt('Plot Graph').''.&mt('Time Period').'
'."\n"; + $Str .= &Apache::loncommon::start_data_table_row(); + $Str .= ''."\n"; $Str .= &Apache::lonstatistics::SectionSelect('Section','multiple',5); - $Str .= ''; + $Str .= ''; + $Str .= &Apache::lonstatistics::GroupSelect('Group','multiple',5); + $Str .= ''; $Str .= &Apache::lonhtmlcommon::StatusOptions(undef,undef,5); - $Str .= ''; + $Str .= ''; # $Str .= &Apache::lonstatistics::map_select('Maps','multiple,all',5); - $Str .= ''.&field_selection_input(); - $Str .= '
'."\n"; - # - $Str .= '

'.&mt('Status: [_1]', - '' - ). - '

'; + $Str .= ''; + $Str .= &field_selection_input(); + $Str .= ''; + $Str .= &plot_dropdown(); + $Str .= ''."\n"; + $Str .= ''; + $Str .= &Apache::lonstathelpers::limit_by_time_form(); + $Str .= ''."\n"; + $Str .= &Apache::loncommon::end_data_table_row(); + $Str .= &Apache::loncommon::end_data_table(); # + $Str .= '

'; $Str .= ''; - $Str .= ' 'x5; - $Str .= 'Plot '.&plot_dropdown().(' 'x10); + $Str .= (' 'x10); # return $Str; } @@ -661,6 +668,11 @@ Main interface to problem statistics. my $navmap; my @sequences; +sub clean_up { + undef($navmap); + undef(@sequences); +} + sub BuildProblemStatisticsPage { my ($r,$c)=@_; undef($navmap); @@ -669,6 +681,7 @@ sub BuildProblemStatisticsPage { my %Saveable_Parameters = ('Status' => 'scalar', 'statsoutputmode' => 'scalar', 'Section' => 'array', + 'Groups' => 'array', 'StudentData' => 'array', 'Maps' => 'array', 'fieldselections'=> 'array'); @@ -684,12 +697,16 @@ sub BuildProblemStatisticsPage { undef(%SeqStat); # # Finally let the user know we are here + $r->print(&Apache::lonhtmlcommon::breadcrumbs('Overall Problem Statistics', + 'Statistics_Overall_Key')); + &Apache::lonquickgrades::startGradeScreen($r,'statistics'); + my $interface = &CreateInterface($r); $r->print($interface); - $r->print(''); # - my @CacheButtonHTML = + my @CacheButtonHTML = &Apache::lonstathelpers::manage_caches($r,'Statistics','stats_status'); my $Str; foreach my $html (@CacheButtonHTML) { @@ -697,14 +714,16 @@ sub BuildProblemStatisticsPage { } # $r->print($Str); - if (! exists($ENV{'form.firstrun'})) { - $r->print('

'. + if (! exists($env{'form.firstrun'})) { + $r->print('

'. &mt('Press "Generate Statistics" when you are ready.'). - '

'. + '

'. + '

'. &mt('It may take some time to update the student data '. - 'for the first analysis. Future analysis this session '. - ' will not have this delay.'). + 'for the first analysis. Future analysis this session '. + 'will not have this delay.'). '

'); + &clean_up(); return; } $r->rflush(); @@ -713,22 +732,29 @@ sub BuildProblemStatisticsPage { # it does not slow things down noticably. &Apache::loncoursedata::populate_weight_table(); # - ($navmap,@sequences) = + ($navmap,@sequences) = &Apache::lonstatistics::selected_sequences_with_assessments(); if (! ref($navmap)) { - $r->print('

'.&mt('A course-wide error occured.').'

'. + $r->print('
'.&mt('A course-wide error occurred.').'
'. '

'.$navmap.'

'); + &clean_up(); return; } - if (exists($ENV{'form.Excel'})) { + if (exists($env{'form.Excel'})) { + $r->print('

'. + &Apache::lonstatistics::section_and_enrollment_description(). + '

'); &Excel_output($r); - } else { + } else { $r->print(''.' 'x5); $r->rflush(); + $r->print('

'. + &Apache::lonstatistics::section_and_enrollment_description(). + '

'); my $count = 0; foreach my $seq (@sequences) { - my @resources = + my @resources = &Apache::lonstathelpers::get_resources($navmap,$seq); $count += scalar(@resources); } @@ -742,9 +768,9 @@ sub BuildProblemStatisticsPage { $r->rflush(); } # - my $sortby = $ENV{'form.sortby'}; + my $sortby = $env{'form.sortby'}; $sortby = 'container' if (! defined($sortby) || $sortby =~ /^\s*$/); - my $plot = $ENV{'form.plot'}; + my $plot = $env{'form.plot'}; if ($plot eq '' || $plot eq 'none') { undef($plot); } @@ -759,6 +785,7 @@ sub BuildProblemStatisticsPage { &output_sequence_statistics($r); } } + &clean_up(); return; } @@ -772,7 +799,7 @@ sub output_sequence_statistics { ''."\n". ''); $r->print(&sequence_html_header()); - foreach my $seq (@sequences) { + foreach my $seq (@sequences) { last if ($c->aborted); &compute_sequence_statistics($seq); $r->print(&sequence_html_output($seq)); @@ -819,7 +846,7 @@ sub output_html_stats { my ($r)=@_; &compute_all_statistics($r); $r->print(&html_preamble()); - &sort_data($ENV{'form.sortby'}); + &sort_data($env{'form.sortby'}); # my $count=0; foreach my $data (@StatsArray) { @@ -841,7 +868,7 @@ sub output_html_stats { sub html_preamble { my $Str=''; $Str .= "

". - $ENV{'course.'.$ENV{'request.course.id'}.'.description'}. + $env{'course.'.$env{'request.course.id'}.'.description'}. "

\n"; my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits(); if (defined($starttime) || defined($endtime)) { @@ -965,7 +992,7 @@ sub sequence_html_output { sub make_plot { my ($r,$plot) = @_; &compute_all_statistics($r); - &sort_data($ENV{'form.sortby'}); + &sort_data($env{'form.sortby'}); if ($plot eq 'degrees') { °rees_plot($r); } elsif ($plot eq 'tries statistics') { @@ -995,7 +1022,7 @@ sub make_single_stat_plot { } # # Build up the data sets to plot - my @Labels; + my @Labels; my @Data; my $max = 1; foreach my $data (@StatsArray) { @@ -1035,7 +1062,7 @@ sub degrees_plot { my $plot = ''; my $ymax = 0; my $ymin = 0; - my @Disc; my @Diff; my @Labels; + my @Disc; my @Diff; my @Labels; foreach my $data (@StatsArray) { push(@Labels,$data->{'problem_num'}); my $disc = $data->{'deg_of_disc'}; @@ -1125,7 +1152,7 @@ sub degrees_plot { END - my $plotresult = + my $plotresult = '

'.&Apache::lonxml::xmlparse($r,'web',$plot).'

'.$/; $r->print($plotresult); return; @@ -1210,7 +1237,7 @@ sub tries_data_plot { END - my $plotresult = + my $plotresult = '

'.&Apache::lonxml::xmlparse($r,'web',$plot).'

'.$/; $r->print($plotresult); return; @@ -1219,8 +1246,8 @@ END sub plot_dropdown { my $current = ''; # - if (defined($ENV{'form.plot'})) { - $current = $ENV{'form.plot'}; + if (defined($env{'form.plot'})) { + $current = $env{'form.plot'}; } # my @Additional_Plots = ( @@ -1273,7 +1300,7 @@ sub Excel_output { return if (! defined($excel_workbook)); # # Add a worksheet - my $sheetname = $ENV{'course.'.$ENV{'request.course.id'}.'.description'}; + my $sheetname = $env{'course.'.$env{'request.course.id'}.'.description'}; if (length($sheetname) > 31) { $sheetname = substr($sheetname,0,31); } @@ -1286,17 +1313,17 @@ sub Excel_output { # # Put the course description in the header $excel_sheet->write($rows_output,$cols_output++, - $ENV{'course.'.$ENV{'request.course.id'}.'.description'}, + $env{'course.'.$env{'request.course.id'}.'.description'}, $format->{'h1'}); $cols_output += 3; # # Put a description of the sections listed my $sectionstring = ''; $excel_sheet->write($rows_output,$cols_output++, - &Apache::lonstathelpers::sections_description - (@Apache::lonstatistics::SelectedSections), + &Apache::lonstatistics::section_and_enrollment_description('plaintext'), $format->{'h3'}); - $cols_output += scalar(@Apache::lonstatistics::SelectedSections); + $cols_output += scalar(&Apache::lonstatistics::get_selected_sections()); + $cols_output += scalar(&Apache::lonstatistics::get_selected_groups()); # # Time restrictions my $time_string; @@ -1321,11 +1348,11 @@ sub Excel_output { $excel_sheet->write($rows_output,$cols_output++, 'Compiled on '.localtime(time)); # - $rows_output++; + $rows_output++; $cols_output=0; ## ## Sequence Statistics - ## + ## &write_headers($excel_sheet,$format,\$rows_output,\$cols_output, \@SeqFields); foreach my $seq (@sequences) { @@ -1403,7 +1430,7 @@ sub write_headers { foreach my $field (@{$Fields}) { next if ($field->{'selected'} ne 'yes'); next if ($field->{'name'} eq 'problem_num'); - # Use english for excel as I am not sure how well excel handles + # Use english for excel as I am not sure how well excel handles # other character sets.... $excel_sheet->write($$rows_output,$$cols_output, $field->{'title'}, @@ -1427,7 +1454,7 @@ sub compute_statistics_on_sequence { my @Data; foreach my $res (&Apache::lonstathelpers::get_resources($navmap,$seq)) { foreach my $part (@{$res->parts}) { - next if ($res->is_survey($part)); + next if (($res->is_survey($part)) || ($res->is_anonsurvey($part))) ; # # This is where all the work happens my $data = &get_statistics($seq,$res,$part,scalar(@StatsArray)+1); @@ -1516,13 +1543,13 @@ sub sort_data { =item &get_statistics() -Wrapper routine from the call to loncoursedata::get_problem_statistics. +Wrapper routine from the call to loncoursedata::get_problem_statistics. Calls lonstathelpers::get_time_limits() to limit the data set by time and &compute_discrimination_factor Inputs: $sequence, $resource, $part, $problem_num -Returns: Hash reference with statistics data from +Returns: Hash reference with statistics data from loncoursedata::get_problem_statistics. =cut @@ -1534,10 +1561,11 @@ sub get_statistics { # my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits(); my $symb = $resource->symb; - my $courseid = $ENV{'request.course.id'}; + my $courseid = $env{'request.course.id'}; # my $data = &Apache::loncoursedata::get_problem_statistics - (\@Apache::lonstatistics::SelectedSections, + ([&Apache::lonstatistics::get_selected_sections()], + [&Apache::lonstatistics::get_selected_groups()], $Apache::lonstatistics::enrollment_status, $symb,$part,$courseid,$starttime,$endtime); $data->{'symb'} = $symb; @@ -1546,25 +1574,24 @@ sub get_statistics { $data->{'container'} = $sequence->compTitle; $data->{'title'} = $resource->compTitle; $data->{'title.link'} = $resource->src.'?symb='. - &Apache::lonnet::escape($resource->symb); + &escape($resource->symb); # if ($SelectedFields{'deg_of_disc'}) { - $data->{'deg_of_disc'} = + $data->{'deg_of_disc'} = &compute_discrimination_factor($resource,$part,$sequence); } # # Store in metadata if computations were done for all students if ($data->{'num_students'} > 1) { - my @Sections = @Apache::lonstatistics::SelectedSections; + my @Sections = &Apache::lonstatistics::get_selected_sections(); my $sections = '"'.join(' ',@Sections).'"'; $sections =~ s/&+/_/g; # Ensure no special characters $data->{'sections'}=$sections; - $data->{'course'} = $ENV{'request.course.id'}; + $data->{'course'} = $env{'request.course.id'}; my $urlres=(&Apache::lonnet::decode_symb($resource->symb))[2]; - $data->{'urlres'}=$urlres; - my %storestats = + my %storestats = &LONCAPA::lonmetadata::dynamic_metadata_storage($data); - my ($dom,$user) = $urlres=~/^(\w+)\/(\w+)/; + my ($dom,$user) = ($urlres=~m{^($LONCAPA::domain_re)/($LONCAPA::username_re)}); &Apache::lonnet::put('nohist_resevaldata',\%storestats,$dom,$user); } # @@ -1572,16 +1599,16 @@ sub get_statistics { ($data->{'num_solved'}+0.1); # # Get the due date for research purposes (commented out most of the time) -# $data->{'duedate'} = -# &Apache::lonnet::EXT('resource.'.$part.'.duedate',$symb); -# $data->{'opendate'} = -# &Apache::lonnet::EXT('resource.'.$part.'.opendate',$symb); -# $data->{'maxtries'} = -# &Apache::lonnet::EXT('resource.'.$part.'.maxtries',$symb); -# $data->{'hinttries'} = -# &Apache::lonnet::EXT('resource.'.$part.'.hinttries',$symb); - $data->{'weight'} = - &Apache::lonnet::EXT('resource.'.$part.'.weight',$symb); +# my $duedate = &Apache::lonnet::EXT('resource.'.$part.'.duedate',$symb);; +# my $opendate = &Apache::lonnet::EXT('resource.'.$part.'.opendate',$symb); +# my $maxtries = &Apache::lonnet::EXT('resource.'.$part.'.maxtries',$symb); +# my $hinttries = &Apache::lonnet::EXT('resource.'.$part.'.hinttries',$symb); + my $weight = &Apache::lonnet::EXT('resource.'.$part.'.weight',$symb); + $data->{'weight'} = $weight; +# $data->{'duedate'} = $duedate; +# $data->{'opendate'} = $opendate; +# $data->{'maxtries'} = $maxtries; +# $data->{'hinttries'} = $hinttries; # $data->{'resptypes'} = join(',',@{$resource->{'partdata'}->{$part}->{'ResponseTypes'}}); return $data; } @@ -1612,30 +1639,31 @@ sub compute_discrimination_factor { # # rank my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits(); - my $ranking = + my $ranking = &Apache::loncoursedata::rank_students_by_scores_on_resources (\@Resources, - \@Apache::lonstatistics::SelectedSections, + [&Apache::lonstatistics::get_selected_sections()], + [&Apache::lonstatistics::get_selected_groups()], $Apache::lonstatistics::enrollment_status,undef, - $starttime,$endtime); + $starttime,$endtime, $symb); # # compute their percent scores on the problems in the sequence, my $number_to_grab = int(scalar(@{$ranking})/4); my $num_students = scalar(@{$ranking}); - my @BottomSet = map { $_->[&Apache::loncoursedata::RNK_student()]; + my @BottomSet = map { $_->[&Apache::loncoursedata::RNK_student()]; } @{$ranking}[0..$number_to_grab]; - my @TopSet = - map { - $_->[&Apache::loncoursedata::RNK_student()]; - } @{$ranking}[($num_students-$number_to_grab)..($num_students-1)]; + my @TopSet = + map { + $_->[&Apache::loncoursedata::RNK_student()]; + } @{$ranking}[-$number_to_grab..0]; if (! @BottomSet || (@BottomSet == 1 && $BottomSet[0] eq '') || ! @TopSet || (@TopSet == 1 && $TopSet[0] eq '')) { return 'nan'; } - my ($bottom_sum,$bottom_max) = + my ($bottom_sum,$bottom_max) = &Apache::loncoursedata::get_sum_of_scores($symb,$part,\@BottomSet, undef,$starttime,$endtime); - my ($top_sum,$top_max) = + my ($top_sum,$top_max) = &Apache::loncoursedata::get_sum_of_scores($symb,$part,\@TopSet, undef,$starttime,$endtime); my $deg_of_disc; @@ -1658,10 +1686,10 @@ sub compute_discrimination_factor { ## ## K=the number of items in your test ## M=the mean score on the test -## s=the standard deviation of the scores on your test +## s=the standard deviation of the scores on your test ## ## then: -## +## ## KR-21 rk= [K/(K-1)] * [1- (M*(K-M))/(K*s^2))] ## ############################################### @@ -1678,9 +1706,10 @@ sub compute_sequence_statistics { my ($starttime,$endtime) = &Apache::lonstathelpers::get_time_limits(); # # First compute statistics based on student scores - my ($smin,$smax,$sMean,$sSTD,$scount,$sMAX) = + my ($smin,$smax,$sMean,$sSTD,$scount,$sMAX) = &Apache::loncoursedata::score_stats - (\@Apache::lonstatistics::SelectedSections, + ([&Apache::lonstatistics::get_selected_sections()], + [&Apache::lonstatistics::get_selected_groups()], $Apache::lonstatistics::enrollment_status, \@Resources,$starttime,$endtime,undef); $SeqStat{$symb}->{'title'} = $seq->compTitle; @@ -1692,10 +1721,11 @@ sub compute_sequence_statistics { $SeqStat{$symb}->{'max_possible'} = $sMAX; # # Compute statistics based on the number of correct problems - # 'correct' is taken to mean + # 'correct' is taken to mean my ($cmin,$cmax,$cMean,$cSTD,$ccount)= &Apache::loncoursedata::count_stats - (\@Apache::lonstatistics::SelectedSections, + ([&Apache::lonstatistics::get_selected_sections()], + [&Apache::lonstatistics::get_selected_groups()], $Apache::lonstatistics::enrollment_status, \@Resources,$starttime,$endtime,undef); my $K = $part_count;