File:
[LON-CAPA] /
loncom /
metadata_database /
LONCAPA /
lonmetadata.pm
Revision
1.11:
download - view:
text,
annotated -
select for diffs
Thu Jun 17 20:33:21 2004 UTC (20 years, 3 months ago) by
www
Branches:
MAIN
CVS tags:
version_1_3_X,
version_1_3_3,
version_1_3_2,
version_1_3_1,
version_1_3_0,
version_1_2_X,
version_1_2_99_1,
version_1_2_99_0,
version_1_2_1,
version_1_2_0,
version_1_1_99_5,
version_1_1_99_4,
version_1_1_99_3,
version_1_1_99_2,
version_1_1_99_1,
HEAD
Ignored "correct" evaluation data
# The LearningOnline Network with CAPA
#
# $Id: lonmetadata.pm,v 1.11 2004/06/17 20:33:21 www Exp $
#
# Copyright Michigan State University Board of Trustees
#
# This file is part of the LearningOnline Network with CAPA (LON-CAPA).
#
# LON-CAPA is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LON-CAPA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with LON-CAPA; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
#
# /home/httpd/html/adm/gpl.txt
#
# http://www.lon-capa.org/
#
######################################################################
package LONCAPA::lonmetadata;
use strict;
use DBI;
######################################################################
######################################################################
=pod
=head1 Name
lonmetadata
=head1 Synopsis
lonmetadata holds a description of the metadata table and provides
wrappers for the storage and retrieval of metadata to/from the database.
=head1 Description
=head1 Methods
=over 4
=cut
######################################################################
######################################################################
=pod
=item Old table creation command
CREATE TABLE IF NOT EXISTS metadata
(title TEXT,
author TEXT,
subject TEXT,
url TEXT,
keywords TEXT,
version TEXT,
notes TEXT,
abstract TEXT,
mime TEXT,
language TEXT,
creationdate DATETIME,
lastrevisiondate DATETIME,
owner TEXT,
copyright TEXT,
FULLTEXT idx_title (title),
FULLTEXT idx_author (author),
FULLTEXT idx_subject (subject),
FULLTEXT idx_url (url),
FULLTEXT idx_keywords (keywords),
FULLTEXT idx_version (version),
FULLTEXT idx_notes (notes),
FULLTEXT idx_abstract (abstract),
FULLTEXT idx_mime (mime),
FULLTEXT idx_language (language),
FULLTEXT idx_owner (owner),
FULLTEXT idx_copyright (copyright))
TYPE=MYISAM;
=cut
######################################################################
######################################################################
my @Metadata_Table_Description =
(
{ name => 'title', type=>'TEXT'},
{ name => 'author', type=>'TEXT'},
{ name => 'subject', type=>'TEXT'},
{ name => 'url', type=>'TEXT', restrictions => 'NOT NULL' },
{ name => 'keywords', type=>'TEXT'},
{ name => 'version', type=>'TEXT'},
{ name => 'notes', type=>'TEXT'},
{ name => 'abstract', type=>'TEXT'},
{ name => 'mime', type=>'TEXT'},
{ name => 'language', type=>'TEXT'},
{ name => 'creationdate', type=>'DATETIME'},
{ name => 'lastrevisiondate', type=>'DATETIME'},
{ name => 'owner', type=>'TEXT'},
{ name => 'copyright', type=>'TEXT'},
#--------------------------------------------------
{ name => 'dependencies', type=>'TEXT'},
{ name => 'modifyinguser', type=>'TEXT'},
{ name => 'authorspace', type=>'TEXT'},
{ name => 'lowestgradelevel', type=>'INT'},
{ name => 'highestgradelevel', type=>'INT'},
{ name => 'standards', type=>'TEXT'},
{ name => 'count', type=>'INT'},
{ name => 'course', type=>'INT'},
{ name => 'course_list', type=>'TEXT'},
{ name => 'goto', type=>'INT'},
{ name => 'goto_list', type=>'TEXT'},
{ name => 'comefrom', type=>'INT'},
{ name => 'comefrom_list', type=>'TEXT'},
{ name => 'sequsage', type=>'INT'},
{ name => 'sequsage_list', type=>'TEXT'},
{ name => 'stdno', type=>'INT'},
{ name => 'stdno_list', type=>'TEXT'},
{ name => 'avetries', type=>'FLOAT'},
{ name => 'avetries_list', type=>'TEXT'},
{ name => 'difficulty', type=>'FLOAT'},
{ name => 'difficulty_list',type=>'TEXT'},
{ name => 'disc', type=>'FLOAT'},
{ name => 'disc_list', type=>'TEXT'},
{ name => 'clear', type=>'FLOAT'},
{ name => 'technical', type=>'FLOAT'},
{ name => 'correct', type=>'FLOAT'},
{ name => 'helpful', type=>'FLOAT'},
{ name => 'depth', type=>'FLOAT'},
{ name => 'hostname', type=> 'TEXT'},
#--------------------------------------------------
);
my @Fulltext_indicies = qw/
title
author
subject
url
keywords
version
notes
abstract
mime
language
owner
copyright/;
######################################################################
######################################################################
=pod
=item &describe_metadata_storage
Input: None
Returns: An array of hash references describing the columns and indicies
of the metadata table(s).
=cut
######################################################################
######################################################################
sub describe_metadata_storage {
return (\@Metadata_Table_Description,\@Fulltext_indicies);
}
######################################################################
######################################################################
=pod
=item create_metadata_storage()
Inputs: table name (optional): the name of the table. Default is 'metadata'.
Returns: A perl string which, when executed by MySQL, will cause the
metadata storage to be initialized.
=cut
######################################################################
######################################################################
sub create_metadata_storage {
my ($tablename) = @_;
$tablename = 'metadata' if (! defined($tablename));
my $request = "CREATE TABLE IF NOT EXISTS ".$tablename." ";
#
# Process the columns (this code is stolen from lonmysql.pm)
my @Columns;
my $col_des; # mysql column description
foreach my $coldata (@Metadata_Table_Description) {
my $column = $coldata->{'name'};
$col_des = '';
if (lc($coldata->{'type'}) =~ /(enum|set)/) { # 'enum' or 'set'
$col_des.=$column." ".$coldata->{'type'}."('".
join("', '",@{$coldata->{'values'}})."')";
} else {
$col_des.=$column." ".$coldata->{'type'};
if (exists($coldata->{'size'})) {
$col_des.="(".$coldata->{'size'}.")";
}
}
# Modifiers
if (exists($coldata->{'restrictions'})){
$col_des.=" ".$coldata->{'restrictions'};
}
if (exists($coldata->{'default'})) {
$col_des.=" DEFAULT '".$coldata->{'default'}."'";
}
$col_des.=' AUTO_INCREMENT' if (exists($coldata->{'auto_inc'}) &&
($coldata->{'auto_inc'} eq 'yes'));
$col_des.=' PRIMARY KEY' if (exists($coldata->{'primary_key'}) &&
($coldata->{'primary_key'} eq 'yes'));
} continue {
# skip blank items.
push (@Columns,$col_des) if ($col_des ne '');
}
foreach my $colname (@Fulltext_indicies) {
my $text = 'FULLTEXT idx_'.$colname.' ('.$colname.')';
push (@Columns,$text);
}
$request .= "(".join(", ",@Columns).") TYPE=MyISAM";
return $request;
}
######################################################################
######################################################################
=pod
=item store_metadata()
Inputs: database handle ($dbh), a table name, and a hash or hash reference
containing the metadata for a single resource.
Returns: 1 on success, 0 on failure to store.
=cut
######################################################################
######################################################################
{
##
## WARNING: The following cleverness may cause trouble in cases where
## the dbi connection is dropped and recreated - a stale statement
## handler may linger around and cause trouble.
##
## In most scripts, this will work fine. If the dbi is going to be
## dropped and (possibly) later recreated, call &clear_sth. Yes it
## is annoying but $sth appearantly does not have a link back to the
## $dbh, so we can't check our validity.
##
my $sth = undef;
my $sth_table = undef;
sub create_statement_handler {
my $dbh = shift();
my $tablename = shift();
$tablename = 'metadata' if (! defined($tablename));
$sth_table = $tablename;
my $request = 'INSERT INTO '.$tablename.' VALUES(';
foreach (@Metadata_Table_Description) {
$request .= '?,';
}
chop $request;
$request.= ')';
$sth = $dbh->prepare($request);
return;
}
sub clear_sth { $sth=undef; $sth_table=undef;}
sub store_metadata {
my ($dbh,$tablename,@Metadata)=@_;
my $errors = '';
if (! defined($sth) ||
( defined($tablename) && ($sth_table ne $tablename)) ||
(! defined($tablename) && $sth_table ne 'metadata')) {
&create_statement_handler($dbh,$tablename);
}
my $successcount = 0;
foreach my $mdata (@Metadata) {
next if (ref($mdata) ne "HASH");
my @MData;
foreach my $field (@Metadata_Table_Description) {
my $fname = $field->{'name'};
if (exists($mdata->{$fname}) &&
defined($mdata->{$fname}) &&
$mdata->{$fname} ne '') {
if ($mdata->{$fname} eq 'nan' ||
$mdata->{$fname} eq '') {
push(@MData,'NULL');
} else {
push(@MData,$mdata->{$fname});
}
} else {
push(@MData,undef);
}
}
$sth->execute(@MData);
if (! $sth->err) {
$successcount++;
} else {
$errors = join(',',$errors,$sth->errstr);
}
$errors =~ s/^,//;
}
if (wantarray()) {
return ($successcount,$errors);
} else {
return $successcount;
}
}
}
######################################################################
######################################################################
=pod
=item lookup_metadata()
Inputs: database handle ($dbh) and a hash or hash reference containing
metadata which will be used for a search.
Returns: scalar with error string on failure, array reference on success.
The array reference is the same one returned by $sth->fetchall_arrayref().
=cut
######################################################################
######################################################################
sub lookup_metadata {
my ($dbh,$condition,$fetchparameter,$tablename) = @_;
$tablename = 'metadata' if (! defined($tablename));
my $error;
my $returnvalue=[];
my $request = 'SELECT * FROM '.$tablename;
if (defined($condition)) {
$request .= ' WHERE '.$condition;
}
my $sth = $dbh->prepare($request);
if ($sth->err) {
$error = $sth->errstr;
}
if (! $error) {
$sth->execute();
if ($sth->err) {
$error = $sth->errstr;
} else {
$returnvalue = $sth->fetchall_arrayref($fetchparameter);
if ($sth->err) {
$error = $sth->errstr;
}
}
}
return ($error,$returnvalue);
}
######################################################################
######################################################################
=pod
=item delete_metadata()
Removes a single metadata record, based on its url.
Inputs: $dbh, the database handler.
$tablename, the name of the metadata table to remove from. default: 'metadata'
$url, the url of the resource to remove from the metadata database.
Returns: undef on success, dbh errorstr on failure.
=cut
######################################################################
######################################################################
sub delete_metadata {
my ($dbh,$tablename,$url) = @_;
$tablename = 'metadata' if (! defined($tablename));
my $error;
my $delete_command = 'DELETE FROM '.$tablename.' WHERE url='.
$dbh->quote($url);
$dbh->do($delete_command);
if ($dbh->err) {
$error = $dbh->errstr();
}
return $error;
}
######################################################################
######################################################################
=pod
=item update_metadata
Updates metadata record in mysql database. It does not matter if the record
currently exists. Fields not present in the new metadata will be taken
from the current record, if it exists. To delete an entry for a key, set
it to "" or undef.
Inputs:
$dbh, database handle
$newmetadata, hash reference containing the new metadata
$tablename, metadata table name. Defaults to 'metadata'.
Returns:
$error on failure. undef on success.
=cut
######################################################################
######################################################################
sub update_metadata {
my ($dbh,$tablename,$newmetadata)=@_;
my $error;
$tablename = 'metadata' if (! defined($tablename));
if (! exists($newmetadata->{'url'})) {
$error = 'Unable to update: no url specified';
}
return $error if (defined($error));
#
# Retrieve current values
my $row;
($error,$row) = &lookup_metadata($dbh,
' url='.$dbh->quote($newmetadata->{'url'}),
undef,$tablename);
return $error if ($error);
my %metadata = &LONCAPA::lonmetadata::metadata_col_to_hash(@{$row->[0]});
#
# Update metadata values
while (my ($key,$value) = each(%$newmetadata)) {
$metadata{$key} = $value;
}
#
# Delete old data (deleting a nonexistant record does not produce an error.
$error = &delete_metadata($dbh,$tablename,$newmetadata->{'url'});
return $error if (defined($error));
#
# Store updated metadata
my $success;
($success,$error) = &store_metadata($dbh,$tablename,\%metadata);
return $error;
}
######################################################################
######################################################################
=pod
=item metdata_col_to_hash
Input: Array of metadata columns
Return: Hash with the metadata columns as keys and the array elements
passed in as values
=cut
######################################################################
######################################################################
sub metadata_col_to_hash {
my @cols=@_;
my %hash=();
for (my $i=0; $i<=$#Metadata_Table_Description;$i++) {
$hash{$Metadata_Table_Description[$i]->{'name'}}=$cols[$i];
}
return %hash;
}
######################################################################
######################################################################
=pod
=item nohist_resevaldata.db data structure
The nohist_resevaldata.db file has the following possible keys:
Statistics Data (values are integers, perl times, or real numbers)
------------------------------------------
$course___$resource___avetries
$course___$resource___count
$course___$resource___difficulty
$course___$resource___stdno
$course___$resource___timestamp
Evaluation Data (values are on a 1 to 5 scale)
------------------------------------------
$username@$dom___$resource___clear
$username@$dom___$resource___comments
$username@$dom___$resource___depth
$username@$dom___$resource___technical
$username@$dom___$resource___helpful
$username@$dom___$resource___correct
Course Context Data
------------------------------------------
$course___$resource___course course id
$course___$resource___comefrom resource preceeding this resource
$course___$resource___goto resource following this resource
$course___$resource___usage resource containing this resource
New statistical data storage
------------------------------------------
$course&$sec&$numstud___$resource___stats
$sec is a string describing the sections: all, 1 2, 1 2 3,...
Value is a '&' deliminated list of key=value pairs.
Possible keys are (currently) disc,course,sections,difficulty,
stdno, timestamp
=cut
######################################################################
######################################################################
=pod
=item &process_reseval_data
Process a nohist_resevaldata hash into a more complex data structure.
Input: Hash reference containing reseval data
Returns: Hash with the following structure:
$hash{$url}->{'statistics'}->{$courseid}->{'avetries'} = $value
$hash{$url}->{'statistics'}->{$courseid}->{'count'} = $value
$hash{$url}->{'statistics'}->{$courseid}->{'difficulty'} = $value
$hash{$url}->{'statistics'}->{$courseid}->{'stdno'} = $value
$hash{$url}->{'statistics'}->{$courseid}->{'timestamp'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'clear'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'comments'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'depth'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'technical'} = $value
$hash{$url}->{'evaluation'}->{$username}->{'helpful'} = $value
$hash{$url}->{'course'} = \@Courses
$hash{$url}->{'comefrom'} = \@Resources
$hash{$url}->{'goto'} = \@Resources
$hash{$url}->{'usage'} = \@Resources
$hash{$url}->{'stats'}->{$courseid\_$section}->{$key} = $value
=cut
######################################################################
######################################################################
sub process_reseval_data {
my ($evaldata) = @_;
my %DynamicData;
#
# Process every stored element
while (my ($storedkey,$value) = each(%{$evaldata})) {
my ($source,$file,$type) = split('___',$storedkey);
$source = &unescape($source);
$file = &unescape($file);
$value = &unescape($value);
" got ".$file."\n ".$type." ".$source."\n";
if ($type =~ /^(avetries|count|difficulty|stdno|timestamp)$/) {
#
# Statistics: $source is course id
$DynamicData{$file}->{'statistics'}->{$source}->{$type}=$value;
} elsif ($type =~ /^(clear|comments|depth|technical|helpful|correct)$/){
#
# Evaluation $source is username, check if they evaluated it
# more than once. If so, pad the entry with a space.
while(exists($DynamicData{$file}->{'evaluation'}->{$type}->{$source})) {
$source .= ' ';
}
$DynamicData{$file}->{'evaluation'}->{$type}->{$source}=$value;
} elsif ($type =~ /^(course|comefrom|goto|usage)$/) {
#
# Context $source is course id or resource
push(@{$DynamicData{$file}->{$type}},&unescape($source));
} elsif ($type eq 'stats') {
#
# Statistics storage...
# $source is $cid\_$sec\_$stdno
# $value is stat1=value&stat2=value&stat3=value,....
#
my ($cid,$sec,$stdno)=split('&',$source);
my $crssec = $cid.'&'.$sec;
my @Data = split('&',$value);
my %Statistics;
while (my ($key,$value) = split('=',pop(@Data))) {
$Statistics{$key} = $value;
}
$sec =~ s:("$|^")::g;
$Statistics{'sections'} = $sec;
#
# Only store the data if the number of students is greater
# than the data already stored
if (! exists($DynamicData{$file}->{'stats'}->{$crssec}) ||
$DynamicData{$file}->{'stats'}->{$crssec}->{'stdno'}<$stdno){
$DynamicData{$file}->{'stats'}->{$crssec}=\%Statistics;
}
}
}
return %DynamicData;
}
######################################################################
######################################################################
=pod
=item &process_dynamic_metadata
Inputs: $url: the url of the item to process
$DynamicData: hash reference for the results of &process_reseval_data
Returns: Hash containing the following keys:
avetries, avetries_list, difficulty, difficulty_list, stdno, stdno_list,
course, course_list, goto, goto_list, comefrom, comefrom_list,
usage, clear, technical, correct, helpful, depth, comments
Each of the return keys is associated with either a number or a string
The *_list items are comma-seperated strings. 'comments' is a string
containing generically marked-up comments.
=cut
######################################################################
######################################################################
sub process_dynamic_metadata {
my ($url,$DynamicData) = @_;
my %data;
my $resdata = $DynamicData->{$url};
#
# Get the statistical data - Use a weighted average
foreach my $type (qw/avetries difficulty disc/) {
my $studentcount;
my $sum;
my @Values;
my @Students;
#
# Old data
foreach my $coursedata (values(%{$resdata->{'statistics'}}),
values(%{$resdata->{'stats'}})) {
if (ref($coursedata) eq 'HASH' && exists($coursedata->{$type})) {
$studentcount += $coursedata->{'stdno'};
$sum += ($coursedata->{$type}*$coursedata->{'stdno'});
push(@Values,$coursedata->{$type});
push(@Students,$coursedata->{'stdno'});
}
}
if (exists($resdata->{'stats'})) {
foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
my $coursedata = $resdata->{'stats'}->{$identifier};
$studentcount += $coursedata->{'stdno'};
$sum += $coursedata->{$type}*$coursedata->{'stdno'};
push(@Values,$coursedata->{$type});
push(@Students,$coursedata->{'stdno'});
}
}
#
# New data
if (defined($studentcount) && $studentcount>0) {
$data{$type} = $sum/$studentcount;
$data{$type.'_list'} = join(',',@Values);
}
}
#
# Find out the number of students who have completed the resource...
my $stdno;
foreach my $coursedata (values(%{$resdata->{'statistics'}}),
values(%{$resdata->{'stats'}})) {
if (ref($coursedata) eq 'HASH' && exists($coursedata->{'stdno'})) {
$stdno += $coursedata->{'stdno'};
}
}
if (exists($resdata->{'stats'})) {
#
# For the number of students, take the maximum found for the class
my $current_course;
my $coursemax=0;
foreach my $identifier (sort(keys(%{$resdata->{'stats'}}))) {
my $coursedata = $resdata->{'stats'}->{$identifier};
if (! defined($current_course)) {
$current_course = $coursedata->{'course'};
}
if ($current_course ne $coursedata->{'course'}) {
$stdno += $coursemax;
$coursemax = 0;
$current_course = $coursedata->{'course'};
}
if ($coursemax < $coursedata->{'stdno'}) {
$coursemax = $coursedata->{'stdno'};
}
}
$stdno += $coursemax; # pick up the final course in the list
}
$data{'stdno'}=$stdno;
#
# Get the context data
foreach my $type (qw/course goto comefrom/) {
if (defined($resdata->{$type}) &&
ref($resdata->{$type}) eq 'ARRAY') {
$data{$type} = scalar(@{$resdata->{$type}});
$data{$type.'_list'} = join(',',@{$resdata->{$type}});
}
}
if (defined($resdata->{'usage'}) &&
ref($resdata->{'usage'}) eq 'ARRAY') {
$data{'sequsage'} = scalar(@{$resdata->{'usage'}});
$data{'sequsage_list'} = join(',',@{$resdata->{'usage'}});
}
#
# Get the evaluation data
foreach my $type (qw/clear technical correct helpful depth/) {
my $count;
my $sum;
foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{$type}})){
$sum += $resdata->{'evaluation'}->{$type}->{$evaluator};
$count++;
}
if ($count > 0) {
$data{$type}=$sum/$count;
}
}
#
# put together comments
my $comments = '<div class="LCevalcomments">';
foreach my $evaluator (keys(%{$resdata->{'evaluation'}->{'comments'}})){
$comments .=
'<p>'.
'<b>'.$evaluator.'</b>:'.
$resdata->{'evaluation'}->{'comments'}->{$evaluator}.
'</p>';
}
$comments .= '</div>';
$data{'comments'} = $comments;
#
if (exists($resdata->{'stats'})) {
$data{'stats'} = $resdata->{'stats'};
}
#
return %data;
}
sub dynamic_metadata_storage {
my ($data) = @_;
my %Store;
my $courseid = $data->{'course'};
my $sections = $data->{'sections'};
my $numstu = $data->{'num_students'};
my $urlres = $data->{'urlres'};
my $key = $courseid.'&'.$sections.'&'.$numstu.'___'.$urlres.'___stats';
$Store{$key} =
'course='.$courseid.'&'.
'sections='.$sections.'&'.
'timestamp='.time.'&'.
'stdno='.$data->{'num_students'}.'&'.
'avetries='.$data->{'mean_tries'}.'&'.
'difficulty='.$data->{'deg_of_diff'};
if (exists($data->{'deg_of_disc'})) {
$Store{$key} .= '&'.'disc='.$data->{'deg_of_disc'};
}
return %Store;
}
######################################################################
######################################################################
##
## The usual suspects, repeated here to reduce dependency hell
##
######################################################################
######################################################################
sub unescape {
my $str=shift;
$str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
return $str;
}
sub escape {
my $str=shift;
$str =~ s/(\W)/"%".unpack('H2',$1)/eg;
return $str;
}
1;
__END__;
=pod
=back
=cut
FreeBSD-CVSweb <freebsd-cvsweb@FreeBSD.org>