--- loncom/interface/statistics/lonproblemanalysis.pm 2003/10/17 22:05:45 1.42 +++ loncom/interface/statistics/lonproblemanalysis.pm 2003/10/20 20:42:39 1.43 @@ -1,6 +1,6 @@ # The LearningOnline Network with CAPA # -# $Id: lonproblemanalysis.pm,v 1.42 2003/10/17 22:05:45 matthew Exp $ +# $Id: lonproblemanalysis.pm,v 1.43 2003/10/20 20:42:39 matthew Exp $ # # Copyright Michigan State University Board of Trustees # @@ -36,6 +36,7 @@ use Apache::lonstatistics; use Apache::lonlocal; use HTML::Entities(); use Time::Local(); +use Spreadsheet::WriteExcel(); my $plotcolors = ['#33ff00', '#0033cc', '#990000', '#aaaa66', '#663399', '#ff9933', @@ -77,18 +78,32 @@ sub BuildProblemAnalysisPage { $r->print(''); $r->print(' 'x5); + $r->print(''); + $r->print(' 'x5); # $r->print('
'); # my ($symb,$part,$resid) = &get_problem_symb( &Apache::lonnet::unescape($ENV{'form.problemchoice'}) ); + $r->rflush(); # my $resource = &get_resource_from_symb($symb); if (defined($resource)) { $r->print('

'.$resource->{'src'}.'

'); my %Data = &get_problem_data($resource->{'src'}); + my $PerformanceData = + &Apache::loncoursedata::get_optionresponse_data + (\@Students,$symb,$resid); my $ORdata = $Data{$part.'.'.$resid}; + if (exists($ENV{'form.ExcelOutput'}) && + defined($PerformanceData)) { + my $result = &prepare_excel_sheet($r,$resource, + $PerformanceData,$ORdata); + $r->print($result); + $r->rflush(); + } ## ## Render the problem my $base; @@ -102,21 +117,21 @@ sub BuildProblemAnalysisPage { ''. $rendered_problem. ''); + $r->rflush(); ## ## Analyze the problem - my $PerformanceData = - &Apache::loncoursedata::get_optionresponse_data - (\@Students,$symb,$resid); if (defined($PerformanceData) && ref($PerformanceData) eq 'ARRAY') { if ($ENV{'form.AnalyzeOver'} eq 'Tries') { - my $analysis_html = &tries_analysis($PerformanceData, + my $analysis_html = &tries_analysis($r,$PerformanceData, $ORdata); $r->print($analysis_html); + $r->rflush(); } elsif ($ENV{'form.AnalyzeOver'} eq 'Time') { my $analysis_html = &time_analysis($PerformanceData, $ORdata); - $r->print($analysis_html); + $r->print($analysis_html); + $r->rflush(); } else { $r->print('

'. &mt('The analysis you have selected is '. @@ -173,13 +188,13 @@ sub build_foil_index { eight => 8, nine => 9, ten => 10,); - my $a1 = $a; - my $b1 = $b; - if (exists($Numbers{lc($a)})) { - $a1 = $Numbers{lc($a)}; + my $a1 = lc($a); + my $b1 = lc($b); + if (exists($Numbers{$a})) { + $a1 = $Numbers{$a}; } - if (exists($Numbers{lc($b)})) { - $b1 = $Numbers{lc($b)}; + if (exists($Numbers{$b})) { + $b1 = $Numbers{$b}; } $a1 cmp $b1; }; @@ -287,7 +302,7 @@ sub build_option_index { ######################################################### ######################################################### sub tries_analysis { - my ($PerformanceData,$ORdata) = @_; + my ($r,$PerformanceData,$ORdata) = @_; my $mintries = 1; my $maxtries = $ENV{'form.NumPlots'}; my ($table,$Foils,$Concepts) = &build_foil_index($ORdata); @@ -298,7 +313,7 @@ sub tries_analysis { '

'.$table; $ENV{'form.AnalyzeAs'} = 'Foils'; } - my %ResponseData = &analyze_option_data_by_tries($PerformanceData, + my %ResponseData = &analyze_option_data_by_tries($r,$PerformanceData, $mintries,$maxtries); my $analysis = ''; if ($ENV{'form.AnalyzeAs'} eq 'Foils') { @@ -460,7 +475,7 @@ sub Tries_Concept_Analysis { } sub analyze_option_data_by_tries { - my ($PerformanceData,$mintries,$maxtries) = @_; + my ($r,$PerformanceData,$mintries,$maxtries) = @_; my %Trydata; $mintries = 1 if (! defined($mintries) || $mintries < 1); $maxtries = $mintries if (! defined($maxtries) || $maxtries < $mintries); @@ -468,6 +483,7 @@ sub analyze_option_data_by_tries { next if (! defined($row)); my $tries = &get_tries_from_row($row); my %Row = &Process_Row($row); + next if (! %Row); while (my ($foilid,$href) = each(%Row)) { if (! ref($href)) { $Trydata{$foilid}->[$tries] += $href; @@ -719,13 +735,394 @@ sub Concept_Time_Analysis { ## ######################################################### ######################################################### + sub prepare_excel_sheet { - my ($PerformanceData,$ORdata) = @_; + my ($r,$resource,$PerformanceData,$ORdata) = @_; + my $response = '

'.&mt('Excel Spreadsheet Creation').'

'; my (undef,$Foils,$Concepts) = &build_foil_index($ORdata); + # + # Create excel worksheet + my $filename = '/prtspool/'. + $ENV{'user.name'}.'_'.$ENV{'user.domain'}.'_'. + time.'_'.rand(1000000000).'.xls'; + my $workbook = Spreadsheet::WriteExcel->new('/home/httpd'.$filename); + if (! defined($workbook)) { + $r->log_error("Error creating excel spreadsheet $filename: $!"); + $r->print('

'.&mt("Unable to create new Excel file. ". + "This error has been logged. ". + "Please alert your LON-CAPA administrator"). + '

'); + return undef; + } + # + $workbook->set_tempdir('/home/httpd/perl/tmp'); + # + # Define some potentially useful formats + my $format; + $format->{'header'} = $workbook->add_format(bold => 1, + bottom => 1, +# text_wrap => 1, + align => 'center'); + $format->{'bold'} = $workbook->add_format(bold=>1); + $format->{'h1'} = $workbook->add_format(bold=>1, size=>18); + $format->{'h2'} = $workbook->add_format(bold=>1, size=>16); + $format->{'h3'} = $workbook->add_format(bold=>1, size=>14); + $format->{'date'} = $workbook->add_format(num_format=> + 'mmm d yyyy hh:mm AM/PM'); + # + # Create and populate main worksheets + my $problem_data_sheet = $workbook->addworksheet('Problem Data'); + my $student_data_sheet = $workbook->addworksheet('Student Data'); + my $response_data_sheet = $workbook->addworksheet('Response Data'); + foreach my $sheet ($problem_data_sheet,$student_data_sheet, + $response_data_sheet) { + $sheet->write(0,0,$resource->{'title'},$format->{'h2'}); + $sheet->write(1,0,$resource->{'src'},$format->{'h3'}); + } + # + my $result; + $result = &build_problem_data_worksheet($problem_data_sheet,$format, + $Concepts,$ORdata); + if ($result ne 'okay') { + # Do something useful + } + $result = &build_student_data_worksheet($student_data_sheet,$format); + if ($result ne 'okay') { + # Do something useful + } + $result = &build_response_data_worksheet($response_data_sheet,$format, + $PerformanceData,$Foils, + $ORdata); + if ($result ne 'okay') { + # Do something useful + } + $response_data_sheet->activate(); + # + # Close the excel file + $workbook->close(); + # + # Write a link to allow them to download it + $result .= '
'. + 'Your Excel spreadsheet.'."\n"; + return $result; +} + +sub build_problem_data_worksheet { + my ($worksheet,$format,$Concepts,$ORdata) = @_; + my $rows_output = 3; + my $cols_output = 0; + $worksheet->write($rows_output++,0,'Problem Structure',$format->{'h3'}); + ## + ## + my @Headers; + if (@$Concepts > 1) { + @Headers = ("Concept\nNumber",'Concept',"Foil\nNumber", + 'Foil Name','Foil Text','Correct value'); + } else { + @Headers = ('Foil Number','FoilName','Foil Text','Correct value'); + } + $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'}); + my %Foildata = %{$ORdata->{'Foils'}}; + my $conceptindex = 1; + my $foilindex = 1; + foreach my $concept (@$Concepts) { + my @FoilsInConcept = @{$concept->{'foils'}}; + my $firstfoil = shift(@FoilsInConcept); + if (@$Concepts > 1) { + $worksheet->write_row($rows_output++,0, + [$conceptindex, + $concept->{'name'}, + $foilindex++, + $Foildata{$firstfoil}->{'name'}, + $Foildata{$firstfoil}->{'text'}, + $Foildata{$firstfoil}->{'value'},]); + } else { + $worksheet->write_row($rows_output++,0, + [ $foilindex++, + $Foildata{$firstfoil}->{'name'}, + $Foildata{$firstfoil}->{'text'}, + $Foildata{$firstfoil}->{'value'},]); + } + foreach my $foilid (@FoilsInConcept) { + if (@$Concepts > 1) { + $worksheet->write_row($rows_output++,0, + ['', + '', + $foilindex, + $Foildata{$foilid}->{'name'}, + $Foildata{$foilid}->{'text'}, + $Foildata{$foilid}->{'value'},]); + } else { + $worksheet->write_row($rows_output++,0, + [$foilindex, + $Foildata{$foilid}->{'name'}, + $Foildata{$foilid}->{'text'}, + $Foildata{$foilid}->{'value'},]); + } + } continue { + $foilindex++; + } + } continue { + $conceptindex++; + } + $rows_output++; + $rows_output++; + ## + ## Option data output + $worksheet->write($rows_output++,0,'Options',$format->{'header'}); + foreach my $string (@{$ORdata->{'Options'}}) { + $worksheet->write($rows_output++,0,$string); + } + return 'okay'; +} + +sub build_student_data_worksheet { + my ($worksheet,$format) = @_; + my $rows_output = 3; + my $cols_output = 0; + $worksheet->write($rows_output++,0,'Student Data',$format->{'h3'}); + my @Headers = ('full name','username','domain','section', + "student\nnumber",'identifier'); + $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'}); + my @Students = @Apache::lonstatistics::Students; + my $studentrows = &Apache::loncoursedata::get_student_data(\@Students); + my %ids; + foreach my $row (@$studentrows) { + my ($mysqlid,$student) = @$row; + $ids{$student}=$mysqlid; + } + foreach my $student (@Students) { + my $name_domain = $student->{'username'}.':'.$student->{'domain'}; + $worksheet->write_row($rows_output++,0, + [$student->{'fullname'}, + $student->{'username'},$student->{'domain'}, + $student->{'section'},$student->{'id'}, + $ids{$name_domain}]); + } + return; +} + +sub build_response_data_worksheet { + my ($worksheet,$format,$PerformanceData,$Foils,$ORdata)=@_; + my $rows_output = 3; + my $cols_output = 0; + $worksheet->write($rows_output++,0,'Response Data',$format->{'h3'}); + $worksheet->set_column(1,1,20); + $worksheet->set_column(2,2,13); + my @Headers = ('identifier','time','award detail','attempt'); + foreach my $foil (@$Foils) { + push (@Headers,$foil.' submission'); + push (@Headers,$foil.' grading'); + } + $worksheet->write_row($rows_output++,0,\@Headers,$format->{'header'}); + # + foreach my $row (@$PerformanceData) { + next if (! defined($row)); + my ($student,$award,$grading,$submission,$time,$tries) = @$row; + my @Foilgrades = split('&',$grading); + my @Foilsubs = split('&',$submission); + my %ResponseData; + for (my $j=0;$j<=$#Foilgrades;$j++) { + my ($foilid,$correct) = split('=',$Foilgrades[$j]); + my (undef,$submission) = split('=',$Foilsubs[$j]); + $submission = &Apache::lonnet::unescape($submission); + $ResponseData{$foilid.' submission'}=$submission; + $ResponseData{$foilid.' award'}=$correct; + } + $worksheet->write($rows_output,$cols_output++,$student); + $worksheet->write($rows_output,$cols_output++, + &calc_serial($time),$format->{'date'}); + $worksheet->write($rows_output,$cols_output++,$award); + $worksheet->write($rows_output,$cols_output++,$tries); + foreach my $foilid (@$Foils) { + $worksheet->write($rows_output,$cols_output++, + $ResponseData{$foilid.' submission'}); + $worksheet->write($rows_output,$cols_output++, + $ResponseData{$foilid.' award'}); + } + $rows_output++; + $cols_output = 0; + } + return; } -sub process_data_for_excel_sheet { - my ($PerformanceData,$Foils,$Concepts,$ORdata)=@_; + +## +## The following is copied from datecalc1.pl, part of the +## Spreadsheet::WriteExcel CPAN module. +## +## +###################################################################### +# +# Demonstration of writing date/time cells to Excel spreadsheets, +# using UNIX/Perl time as source of date/time. +# +# Copyright 2000, Andrew Benham, adsb@bigfoot.com +# +###################################################################### +# +# UNIX/Perl time is the time since the Epoch (00:00:00 GMT, 1 Jan 1970) +# measured in seconds. +# +# An Excel file can use exactly one of two different date/time systems. +# In these systems, a floating point number represents the number of days +# (and fractional parts of the day) since a start point. The floating point +# number is referred to as a 'serial'. +# The two systems ('1900' and '1904') use different starting points: +# '1900'; '1.00' is 1 Jan 1900 BUT 1900 is erroneously regarded as +# a leap year - see: +# http://support.microsoft.com/support/kb/articles/Q181/3/70.asp +# for the excuse^H^H^H^H^H^Hreason. +# '1904'; '1.00' is 2 Jan 1904. +# +# The '1904' system is the default for Apple Macs. Windows versions of +# Excel have the option to use the '1904' system. +# +# Note that Visual Basic's "DateSerial" function does NOT erroneously +# regard 1900 as a leap year, and thus its serials do not agree with +# the 1900 serials of Excel for dates before 1 Mar 1900. +# +# Note that StarOffice (at least at version 5.2) does NOT erroneously +# regard 1900 as a leap year, and thus its serials do not agree with +# the 1900 serials of Excel for dates before 1 Mar 1900. +# +###################################################################### +# +# Calculation description +# ======================= +# +# 1900 system +# ----------- +# Unix time is '0' at 00:00:00 GMT 1 Jan 1970, i.e. 70 years after 1 Jan 1900. +# Of those 70 years, 17 (1904,08,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68) +# were leap years with an extra day. +# Thus there were 17 + 70*365 days = 25567 days between 1 Jan 1900 and +# 1 Jan 1970. +# In the 1900 system, '1' is 1 Jan 1900, but as 1900 was not a leap year +# 1 Jan 1900 should really be '2', so 1 Jan 1970 is '25569'. +# +# 1904 system +# ----------- +# Unix time is '0' at 00:00:00 GMT 1 Jan 1970, i.e. 66 years after 1 Jan 1904. +# Of those 66 years, 17 (1904,08,12,16,20,24,28,32,36,40,44,48,52,56,60,64,68) +# were leap years with an extra day. +# Thus there were 17 + 66*365 days = 24107 days between 1 Jan 1904 and +# 1 Jan 1970. +# In the 1904 system, 2 Jan 1904 being '1', 1 Jan 1970 is '24107'. +# +###################################################################### +# +# Copyright (c) 2000, Andrew Benham. +# This program is free software. It may be used, redistributed and/or +# modified under the same terms as Perl itself. +# +# Andrew Benham, adsb@bigfoot.com +# London, United Kingdom +# 11 Nov 2000 +# +###################################################################### + +# Use 1900 date system on all platforms other than Apple Mac (for which +# use 1904 date system). +my $DATE_SYSTEM = ($^O eq 'MacOS') ? 1 : 0; + +#----------------------------------------------------------- +# calc_serial() +# +# Called with (up to) 2 parameters. +# 1. Unix timestamp. If omitted, uses current time. +# 2. GMT flag. Set to '1' to return serial in GMT. +# If omitted, returns serial in appropriate timezone. +# +# Returns date/time serial according to $DATE_SYSTEM selected +#----------------------------------------------------------- +sub calc_serial { + my $time = (defined $_[0]) ? $_[0] : time(); + my $gmtflag = (defined $_[1]) ? $_[1] : 0; + + # Divide timestamp by number of seconds in a day. + # This gives a date serial with '0' on 1 Jan 1970. + my $serial = $time / 86400; + + # Adjust the date serial by the offset appropriate to the + # currently selected system (1900/1904). + if ($DATE_SYSTEM == 0) { # use 1900 system + $serial += 25569; + } else { # use 1904 system + $serial += 24107; + } + + unless ($gmtflag) { + # Now have a 'raw' serial with the right offset. But this + # gives a serial in GMT, which is false unless the timezone + # is GMT. We need to adjust the serial by the appropriate + # timezone offset. + # Calculate the appropriate timezone offset by seeing what + # the differences between localtime and gmtime for the given + # time are. + + my @gmtime = gmtime($time); + my @ltime = localtime($time); + + # For the first 7 elements of the two arrays, adjust the + # date serial where the elements differ. + for (0 .. 6) { + my $diff = $ltime[$_] - $gmtime[$_]; + if ($diff) { + $serial += _adjustment($diff,$_); + } + } + } + + # Perpetuate the error that 1900 was a leap year by decrementing + # the serial if we're using the 1900 system and the date is prior to + # 1 Mar 1900. This has the effect of making serial value '60' + # 29 Feb 1900. + + # This fix only has any effect if UNIX/Perl time on the platform + # can represent 1900. Many can't. + + unless ($DATE_SYSTEM) { + $serial-- if ($serial < 61); # '61' is 1 Mar 1900 + } + return $serial; +} + +sub _adjustment { + # Based on the difference in the localtime/gmtime array elements + # number, return the adjustment required to the serial. + + # We only look at some elements of the localtime/gmtime arrays: + # seconds unlikely to be different as all known timezones + # have an offset of integral multiples of 15 minutes, + # but it's easy to do. + # minutes will be different for timezone offsets which are + # not an exact number of hours. + # hours very likely to be different. + # weekday will differ when localtime/gmtime difference + # straddles midnight. + # + # Assume that difference between localtime and gmtime is less than + # 5 days, then don't have to do maths for day of month, month number, + # year number, etc... + + my ($delta,$element) = @_; + my $adjust = 0; + + if ($element == 0) { # Seconds + $adjust = $delta/86400; # 60 * 60 * 24 + } elsif ($element == 1) { # Minutes + $adjust = $delta/1440; # 60 * 24 + } elsif ($element == 2) { # Hours + $adjust = $delta/24; # 24 + } elsif ($element == 6) { # Day of week number + # Catch difference straddling Sat/Sun in either direction + $delta += 7 if ($delta < -4); + $delta -= 7 if ($delta > 4); + + $adjust = $delta; + } + return $adjust; } ######################################################### @@ -910,7 +1307,7 @@ sub get_time_from_row { sub get_tries_from_row { my ($row) = @_; if (ref($row)) { - return $row->[4]; + return $row->[5]; } return undef; } @@ -918,8 +1315,8 @@ sub get_tries_from_row { sub Process_Row { my ($row) = @_; my %RowData; - my ($award,$grading,$submission,$time,$tries) = @$row; - next if ($award eq 'MISSING_ANSWER'); + my ($student_id,$award,$grading,$submission,$time,$tries) = @$row; + return undef if ($award eq 'MISSING_ANSWER'); if ($award =~ /(APPROX_ANS|EXACT_ANS)/) { $RowData{'_correct'} = 1; } @@ -933,7 +1330,6 @@ sub Process_Row { $RowData{$foilid}->{'_correct'}++; } else { $submission = &Apache::lonnet::unescape($submission); - $submission =~ s/\%20/ /g; $RowData{$foilid}->{$submission}++; } $RowData{$foilid}->{'_total'}++; @@ -973,6 +1369,7 @@ sub get_problem_data { if ($key=~ /^foil\.text\.(.*)$/) { my $foil = $1; $Partdata{$part}->{'Foils'}->{$foil}->{'name'}=$foil; + $value =~ s/(\s*$|^\s*)//g; $Partdata{$part}->{'Foils'}->{$foil}->{'text'}=$value; } elsif ($key =~ /^foil\.value\.(.*)$/) { my $foil = $1;