--- loncom/metadata_database/searchcat.pl 2006/01/27 15:53:49 1.66 +++ loncom/metadata_database/searchcat.pl 2007/01/03 03:58:34 1.70.2.1 @@ -2,7 +2,7 @@ # The LearningOnline Network # searchcat.pl "Search Catalog" batch script # -# $Id: searchcat.pl,v 1.66 2006/01/27 15:53:49 albertel Exp $ +# $Id: searchcat.pl,v 1.70.2.1 2007/01/03 03:58:34 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -65,9 +65,6 @@ and correct user experience. =cut use strict; -BEGIN { - eval "use Apache2::compat();"; -}; use DBI; use lib '/home/httpd/lib/perl/'; use LONCAPA::lonmetadata; @@ -121,8 +118,18 @@ if (defined($oneuser)) { ## ## Use variables for table names so we can test this routine a little easier -my $oldname = 'metadata'; -my $newname = 'newmetadata'.$$; # append pid to have unique temporary table +my %oldnames = ( + 'metadata' => 'metadata', + 'portfolio' => 'portfolio_metadata', + 'access' => 'portfolio_access', + 'addedfields' => 'portfolio_addedfields', + ); + +my %newnames; +# new table names - append pid to have unique temporary tables +foreach my $key (keys(%oldnames)) { + $newnames{$key} = 'new'.$oldnames{$key}.$$; +} # # Only run if machine is a library server @@ -158,16 +165,27 @@ if (! ($dbh = DBI->connect("DBI:mysql:lo } # This can return an error and still be okay, so we do not bother checking. # (perhaps it should be more robust and check for specific errors) -$dbh->do('DROP TABLE IF EXISTS '.$newname); +foreach my $key (keys(%newnames)) { + if ($newnames{$key} ne '') { + $dbh->do('DROP TABLE IF EXISTS '.$newnames{$key}); + } +} + # -# Create the new table -my $request = &LONCAPA::lonmetadata::create_metadata_storage($newname); -$dbh->do($request); -if ($dbh->err) { - $dbh->disconnect(); - &log(0,"MySQL Error Create: ".$dbh->errstr); - die $dbh->errstr; +# Create the new metadata and portfolio tables +foreach my $key (keys(%newnames)) { + if ($newnames{$key} ne '') { + my $request = + &LONCAPA::lonmetadata::create_metadata_storage($newnames{$key},$oldnames{$key}); + $dbh->do($request); + if ($dbh->err) { + $dbh->disconnect(); + &log(0,"MySQL Error Create: ".$dbh->errstr); + die $dbh->errstr; + } + } } + # # find out which users we need to examine my @domains = sort(&Apache::lonnet::current_machine_domains()); @@ -203,16 +221,54 @@ foreach my $dom (@domains) { no_chdir => 1, }, join('/',($Apache::lonnet::perlvar{'lonDocRoot'},'res',$dom,$user)) ); } + # Search for public portfolio files + my %portusers; + if ($oneuser) { + %portusers = ( + $oneuser => '', + ); + } else { + my $dir = $Apache::lonnet::perlvar{lonUsersDir}.'/'.$dom; + &descend_tree($dir,0,\%portusers); + } + foreach my $uname (keys(%portusers)) { + my $urlstart = '/uploaded/'.$dom.'/'.$uname; + my $pathstart = &propath($dom,$uname).'/userfiles'; + my $is_course = &Apache::lonnet::is_course($dom,$uname); + my $curr_perm = &Apache::lonnet::get_portfile_permissions($dom,$uname); + my %access = &Apache::lonnet::get_access_controls($curr_perm); + foreach my $file (keys(%access)) { + my ($group,$url,$fullpath); + if ($is_course) { + ($group, my ($path)) = ($file =~ /^(\w+)(\/.+)$/); + $fullpath = $pathstart.'/groups/'.$group.'/portfolio'.$path; + $url = $urlstart.'/groups/'.$group.'/portfolio'.$path; + } else { + $fullpath = $pathstart.'/portfolio'.$file; + $url = $urlstart.'/portfolio'.$file; + } + if (ref($access{$file}) eq 'HASH') { + &process_portfolio_access_data($url,$access{$file}); + } + &process_portfolio_metadata($url,$fullpath,$is_course,$dom, + $uname,$group); + } + } } + # -# Rename the table +# Rename the tables if (! $simulate) { - $dbh->do('DROP TABLE IF EXISTS '.$oldname); - if (! $dbh->do('RENAME TABLE '.$newname.' TO '.$oldname)) { - &log(0,"MySQL Error Rename: ".$dbh->errstr); - die $dbh->errstr; - } else { - &log(1,"MySQL table rename successful."); + foreach my $key (keys(%oldnames)) { + if (($oldnames{$key} ne '') && ($newnames{$key} ne '')) { + $dbh->do('DROP TABLE IF EXISTS '.$oldnames{$key}); + if (! $dbh->do('RENAME TABLE '.$newnames{$key}.' TO '.$oldnames{$key})) { + &log(0,"MySQL Error Rename: ".$dbh->errstr); + die $dbh->errstr; + } else { + &log(1,"MySQL table rename successful for $key."); + } + } } } if (! $dbh->disconnect) { @@ -246,6 +302,108 @@ sub log { } } +sub descend_tree { + my ($dir,$depth,$alldomusers) = @_; + if (-d $dir) { + opendir(DIR,$dir); + my @contents = grep(!/^\./,readdir(DIR)); + closedir(DIR); + $depth ++; + foreach my $item (@contents) { + if ($depth < 4) { + &descend_tree($dir.'/'.$item,$depth,$alldomusers); + } else { + if (-e $dir.'/'.$item.'/file_permissions.db') { + + $$alldomusers{$item} = ''; + } + } + } + } +} + +sub process_portfolio_access_data { + my ($url,$access_hash) = @_; + foreach my $key (keys(%{$access_hash})) { + my $acc_data; + $acc_data->{url} = $url; + $acc_data->{keynum} = $key; + my ($num,$scope,$end,$start) = + ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/); + $acc_data->{scope} = $scope; + if ($end != 0) { + $acc_data->{end} = &sqltime($end); + } + $acc_data->{start} = &sqltime($start); + if (! $simulate) { + my ($count,$err) = + &LONCAPA::lonmetadata::store_metadata($dbh, + $newnames{'access'}, + 'portfolio_access',$acc_data); + if ($err) { + &log(0,"MySQL Error Insert: ".$err); + } + if ($count < 1) { + &log(0,"Unable to insert record into MySQL database for $url"); + } + } + } +} + +sub process_portfolio_metadata { + my ($url,$fullpath,$is_course,$dom,$uname,$group) = @_; + my ($ref,$crs,$addedfields) = &portfolio_metadata($fullpath,$dom,$uname, + $group); + &getfiledates($ref,$fullpath); + if ($is_course) { + $ref->{'groupname'} = $group; + } + my %Data; + if (ref($ref) eq 'HASH') { + %Data = %{$ref}; + } + %Data = ( + %Data, + 'url'=>$url, + 'version'=>'current', + ); + if (! $simulate) { + my ($count,$err) = + &LONCAPA::lonmetadata::store_metadata($dbh, + $newnames{'portfolio'}, + 'portfolio_metadata',\%Data); + if ($err) { + &log(0,"MySQL Error Insert: ".$err); + } + if ($count < 1) { + &log(0,"Unable to insert record into MySQL portfolio_metadata database table for $url"); + } + if (ref($addedfields) eq 'HASH') { + if (keys(%{$addedfields}) > 0) { + foreach my $key (keys(%{$addedfields})) { + my $added_data = { + 'url' => $url, + 'field' => $key, + 'value' => $addedfields->{$key}, + 'courserestricted' => $crs, + }; + ($count,$err) = &LONCAPA::lonmetadata::store_metadata($dbh, + $newnames{'addedfields'}, + 'portfolio_addedfields', + $added_data); + if ($err) { + &log(0,"MySQL Error Insert: ".$err); + } + if ($count < 1) { + &log(0,"Unable to insert record into MySQL portfolio_addedfields database table for url = $url and field = $key"); + } + } + } + } + } + return; +} + ######################################################## ######################################################## ### ### @@ -265,7 +423,7 @@ sub only_meta_files { foreach my $file (@PossibleFiles) { if ( ($file =~ /\.meta$/ && # Ends in meta $file !~ /\.\d+\.[^\.]+\.meta$/ # is not for a prior version - ) || (-d $file )) { # directories are okay + ) || (-d $File::Find::dir."/".$file )) { # directories are okay # but we do not want /. or /.. push(@ChosenFiles,$file); } @@ -296,7 +454,7 @@ sub log_metadata { return if (-d $fullfilename); # No need to do anything here for directories if ($debug) { &log(6,$fullfilename); - my $ref=&metadata($fullfilename); + my $ref = &metadata($fullfilename); if (! defined($ref)) { &log(6," No data"); return; @@ -320,7 +478,7 @@ sub process_meta_file { # &log(3,$filename) if ($debug); # - my $ref=&metadata($filename); + my $ref = &metadata($filename); # # $url is the original file url, not the metadata file my $target = $filename; @@ -349,25 +507,17 @@ sub process_meta_file { %dyn=&get_dynamic_metadata($url); &count_type($url); } + &getfiledates($ref,$target); # - if (! defined($ref->{'creationdate'}) || - $ref->{'creationdate'} =~ /^\s*$/) { - $ref->{'creationdate'} = (stat($target))[9]; - } - if (! defined($ref->{'lastrevisiondate'}) || - $ref->{'lastrevisiondate'} =~ /^\s*$/) { - $ref->{'lastrevisiondate'} = (stat($target))[9]; - } - $ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); - $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); my %Data = ( %$ref, %dyn, 'url'=>$url, 'version'=>'current'); if (! $simulate) { - my ($count,$err) = &LONCAPA::lonmetadata::store_metadata($dbh,$newname, - \%Data); + my ($count,$err) = + &LONCAPA::lonmetadata::store_metadata($dbh,$newnames{'metadata'}, + 'metadata',\%Data); if ($err) { &log(0,"MySQL Error Insert: ".$err); } @@ -389,7 +539,7 @@ sub process_meta_file { ######################################################## ######################################################## sub metadata { - my ($uri)=@_; + my ($uri) = @_; my %metacache=(); $uri=&declutter($uri); my $filename=$uri; @@ -419,7 +569,7 @@ sub metadata { } foreach ( @{$token->[3]}) { $metacache{$uri.''.$unikey.'.'.$_}=$token->[2]->{$_}; - } + } if (! ($metacache{$uri.''.$unikey}=$parser->get_text('/'.$entry))){ $metacache{$uri.''.$unikey} = $metacache{$uri.''.$unikey.'.default'}; @@ -429,6 +579,88 @@ sub metadata { return \%metacache; } +############################################################### +############################################################### +### ### +### &portfolio_metadata($filepath,$dom,$uname,$group) ### +### Retrieve metadata for the given file ### +### Returns array - ### +### contains reference to metadatahash and ### +### optional reference to addedfields hash ### +### ### +############################################################### +############################################################### +sub portfolio_metadata { + my ($fullpath,$dom,$uname,$group)=@_; + my ($mime) = ( $fullpath=~/\.(\w+)$/ ); + my %metacache=(); + if ($fullpath !~ /\.meta$/) { + $fullpath .= '.meta'; + } + my (@standard_fields,%addedfields); + my $colsref = + $LONCAPA::lonmetadata::Portfolio_metadata_table_description; + if (ref($colsref) eq 'ARRAY') { + my @columns = @{$colsref}; + foreach my $coldata (@columns) { + push(@standard_fields,$coldata->{'name'}); + } + } + my $metastring=&getfile($fullpath); + if (! defined($metastring)) { + $metacache{'keys'}= 'owner,domain,mime'; + $metacache{'owner'} = $uname.':'.$dom; + $metacache{'domain'} = $dom; + $metacache{'mime'} = $mime; + if (defined($group)) { + $metacache{'keys'} .= ',courserestricted'; + $metacache{'courserestricted'} = 'course.'.$dom.'_'.$uname; + } + } else { + my $parser=HTML::TokeParser->new(\$metastring); + my $token; + while ($token=$parser->get_token) { + if ($token->[0] eq 'S') { + my $entry=$token->[1]; + if ($metacache{'keys'}) { + $metacache{'keys'}.=','.$entry; + } else { + $metacache{'keys'}=$entry; + } + my $value = $parser->get_text('/'.$entry); + if (!grep(/^\Q$entry\E$/,@standard_fields)) { + my $clean_value = lc($value); + $clean_value =~ s/\s/_/g; + if ($clean_value ne $entry) { + if (defined($addedfields{$entry})) { + $addedfields{$entry} .=','.$value; + } else { + $addedfields{$entry} = $value; + } + } + } else { + $metacache{$entry} = $value; + } + } + } # End of ($token->[0] eq 'S') + } + if (keys(%addedfields) > 0) { + foreach my $key (sort keys(%addedfields)) { + $metacache{'addedfieldnames'} .= $key.','; + $metacache{'addedfieldvalues'} .= $addedfields{$key}.'&&&'; + } + $metacache{'addedfieldnames'} =~ s/,$//; + $metacache{'addedfieldvalues'} =~ s/\&\&\&$//; + if ($metacache{'keys'}) { + $metacache{'keys'}.=',addedfieldnames'; + } else { + $metacache{'keys'}='addedfieldnames'; + } + $metacache{'keys'}.=',addedfieldvalues'; + } + return (\%metacache,$metacache{'courserestricted'},\%addedfields); +} + ## ## &getfile($filename) ## Slurps up an entire file into a scalar. @@ -446,6 +678,24 @@ sub getfile { return $contents; } +## +## &getfiledates() +## Converts creationdate and modifieddates to SQL format +## Applies stat() to file to retrieve dates if missing +sub getfiledates { + my ($ref,$target) = @_; + if (! defined($ref->{'creationdate'}) || + $ref->{'creationdate'} =~ /^\s*$/) { + $ref->{'creationdate'} = (stat($target))[9]; + } + if (! defined($ref->{'lastrevisiondate'}) || + $ref->{'lastrevisiondate'} =~ /^\s*$/) { + $ref->{'lastrevisiondate'} = (stat($target))[9]; + } + $ref->{'creationdate'} = &sqltime($ref->{'creationdate'}); + $ref->{'lastrevisiondate'} = &sqltime($ref->{'lastrevisiondate'}); +} + ######################################################## ######################################################## ### ###