--- loncom/lonnet/perl/lonnet.pm 2004/08/25 16:03:17 1.532 +++ loncom/lonnet/perl/lonnet.pm 2004/09/20 19:54:47 1.543 @@ -1,7 +1,7 @@ # The LearningOnline Network # TCP networking package # -# $Id: lonnet.pm,v 1.532 2004/08/25 16:03:17 albertel Exp $ +# $Id: lonnet.pm,v 1.543 2004/09/20 19:54:47 albertel Exp $ # # Copyright Michigan State University Board of Trustees # @@ -50,7 +50,7 @@ use Fcntl qw(:flock); use Apache::loncoursedata; use Apache::lonlocal; use Storable qw(lock_store lock_nstore lock_retrieve freeze thaw); -use Time::HiRes(); +use Time::HiRes qw( gettimeofday tv_interval ); my $readit; =pod @@ -826,10 +826,12 @@ my $disk_caching_disabled=1; sub devalidate_cache { my ($cache,$id,$name) = @_; delete $$cache{$id.'.time'}; + delete $$cache{$id.'.file'}; delete $$cache{$id}; - if ($disk_caching_disabled) { return; } + if (1 || $disk_caching_disabled) { return; } my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; - open(DB,"$filename.lock"); + if (!-e $filename) { return; } + open(DB,">$filename.lock"); flock(DB,LOCK_EX); my %hash; if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { @@ -856,16 +858,32 @@ sub is_cached { my ($cache,$id,$name,$time) = @_; if (!$time) { $time=300; } if (!exists($$cache{$id.'.time'})) { - &load_cache_item($cache,$name,$id); + &load_cache_item($cache,$name,$id,$time); } if (!exists($$cache{$id.'.time'})) { # &logthis("Didn't find $id"); return (undef,undef); } else { if (time-($$cache{$id.'.time'})>$time) { -# &logthis("Devalidating $id - ".time-($$cache{$id.'.time'})); - &devalidate_cache($cache,$id,$name); - return (undef,undef); + if (exists($$cache{$id.'.file'})) { + foreach my $filename (@{ $$cache{$id.'.file'} }) { + my $mtime=(stat($filename))[9]; + #+1 is to take care of edge effects + if ($mtime && (($mtime+1) < ($$cache{$id.'.time'}))) { +# &logthis("Upping $mtime - ".$$cache{$id.'.time'}. +# "$id because of $filename"); + } else { + &logthis("Devalidating $filename $id - ".(time-($$cache{$id.'.time'}))); + &devalidate_cache($cache,$id,$name); + return (undef,undef); + } + } + $$cache{$id.'.time'}=time; + } else { +# &logthis("Devalidating $id - ".time-($$cache{$id.'.time'})); + &devalidate_cache($cache,$id,$name); + return (undef,undef); + } } } return ($$cache{$id},1); @@ -881,44 +899,66 @@ sub do_cache { $$cache{$id}; } +my %do_save_item; +my %do_save; sub save_cache_item { my ($cache,$name,$id)=@_; if ($disk_caching_disabled) { return; } - my $starttime=&Time::HiRes::time(); -# &logthis("Saving :$name:$id"); - my %hash; - my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; - open(DB,"$filename.lock"); - flock(DB,LOCK_EX); - if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { - eval <<'EVALBLOCK'; - $hash{$id.'.time'}=$$cache{$id.'.time'}; - $hash{$id}=freeze({'item'=>$$cache{$id}}); + $do_save{$name}=$cache; + if (!exists($do_save_item{$name})) { $do_save_item{$name}={} } + $do_save_item{$name}->{$id}=1; + return; +} + +sub save_cache { + if ($disk_caching_disabled) { return; } + my ($cache,$name,$id); + foreach $name (keys(%do_save)) { + $cache=$do_save{$name}; + + my $starttime=&Time::HiRes::time(); + &logthis("Saving :$name:"); + my %hash; + my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; + open(DB,">$filename.lock"); + flock(DB,LOCK_EX); + if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { + foreach $id (keys(%{ $do_save_item{$name} })) { + eval <<'EVALBLOCK'; + $hash{$id.'.time'}=$$cache{$id.'.time'}; + $hash{$id}=freeze({'item'=>$$cache{$id}}); EVALBLOCK - if ($@) { - &logthis("save_cache blew up :$@:$name"); - unlink($filename); - } - } else { - if (-e $filename) { - &logthis("Unable to tie hash (save cache item): $name ($!)"); - unlink($filename); + if ($@) { + &logthis("save_cache blew up :$@:$name"); + unlink($filename); + last; + } + } + } else { + if (-e $filename) { + &logthis("Unable to tie hash (save cache): $name ($!)"); + unlink($filename); + } } + untie(%hash); + flock(DB,LOCK_UN); + close(DB); + &logthis("save_cache $name took ".(&Time::HiRes::time()-$starttime)); } - untie(%hash); - flock(DB,LOCK_UN); - close(DB); -# &logthis("save_cache_item $name took ".(&Time::HiRes::time()-$starttime)); + undef(%do_save); + undef(%do_save_item); + } sub load_cache_item { - my ($cache,$name,$id)=@_; + my ($cache,$name,$id,$time)=@_; if ($disk_caching_disabled) { return; } my $starttime=&Time::HiRes::time(); # &logthis("Before Loading $name for $id size is ".scalar(%$cache)); my %hash; my $filename=$perlvar{'lonDaemons'}.'/tmp/lonnet_internal_cache_'.$name.".db"; - open(DB,"$filename.lock"); + if (!-e $filename) { return; } + open(DB,">$filename.lock"); flock(DB,LOCK_SH); if (tie(%hash,'GDBM_File',$filename,&GDBM_READER(),0640)) { eval <<'EVALBLOCK'; @@ -935,9 +975,11 @@ sub load_cache_item { } # &logthis("Initial load: $count"); } else { - my $hashref=thaw($hash{$id}); - $$cache{$id}=$hashref->{'item'}; - $$cache{$id.'.time'}=$hash{$id.'.time'}; + if (($$cache{$id.'.time'}+$time) < time) { + $$cache{$id.'.time'}=$hash{$id.'.time'}; + my $hashref=thaw($hash{$id}); + $$cache{$id}=$hashref->{'item'}; + } } EVALBLOCK if ($@) { @@ -1067,7 +1109,12 @@ sub subscribe { sub repcopy { my $filename=shift; $filename=~s/\/+/\//g; - if ($filename=~/^\/home\/httpd\/html\/adm\//) { return OK; } + if ($filename=~m|^/home/httpd/html/adm/|) { return OK; } + if ($filename=~m|^/home/httpd/html/lonUsers/|) { return OK; } + if ($filename=~m|^/home/httpd/html/userfiles/| or + $filename=~m|^/*uploaded/|) { + return &repcopy_userfile($filename); + } $filename=~s/[\n\r]//g; my $transname="$filename.in.transfer"; if ((-e $filename) || (-e $transname)) { return OK; } @@ -1279,6 +1326,9 @@ sub clean_filename { $fname=~s/\s+/\_/g; # Replace all other weird characters by nothing $fname=~s/[^\w\.\-]//g; +# Replace all .\d. sequences with _\d. so they no longer look like version +# numbers + $fname=~s/\.(\d+)(?=\.)/_$1/g; return $fname; } @@ -3679,38 +3729,11 @@ sub revokecustomrole { $deleteflag); } - -# ------------------------------------------------------------ Portfolio Director Lister -# returns listing of contents of user's /userfiles/portfolio/ directory -# - -sub portfoliolist { - my ($currentPath, $currentFile) = @_; - my ($udom, $uname, $portfolioRoot); - $uname=$ENV{'user.name'}; - $udom=$ENV{'user.domain'}; - # really should interrogate the system for home directory information, but . . . - $portfolioRoot = '/home/httpd/lonUsers/'.$udom.'/'; - $uname =~ /^(.?)(.?)(.?)/; - $portfolioRoot = $portfolioRoot.$1.'/'.$2.'/'.$3.'/'.$uname.'/userfiles/portfolio'; - my $listing = &reply('ls:'.$portfolioRoot.$currentPath, &homeserver($uname,$udom)); - return $listing; -} - -sub portfoliomanage { - -#FIXME please user the existing remove userfile function instead and -#add a userfilerename functions. -#FIXME uhome should never be an argument to any lonnet functions - - # handles deleting and renaming files in user's userfiles/portfolio/ directory - # - my ($filename, $fileaction, $filenewname) = @_; - my ($udom, $uname, $uhome); - $uname=$ENV{'user.name'}; - $udom=$ENV{'user.domain'}; - $uhome=$ENV{'user.home'}; - my $listing = reply('portfoliomanage:'.$uname.':'.$udom.':'.$filename.':'.$fileaction.':'.$filenewname, $uhome); +# ------------------------------------------------------------ Disk usage +sub diskusage { + my ($udom,$uname,$directoryRoot)=@_; + $directoryRoot =~ s/\/$//; + my $listing=&reply('du:'.$directoryRoot,homeserver($uname,$udom)); return $listing; } @@ -4048,11 +4071,14 @@ sub EXT { my $section; if (defined($courseid) && $courseid eq $ENV{'request.course.id'}) { + if (!$symbparm) { $symbparm=&symbread(); } + } + if ($symbparm && defined($courseid) && + $courseid eq $ENV{'request.course.id'}) { #print '
'.$space.' - '.$qualifier.' - '.$spacequalifierrest; # ----------------------------------------------------- Cascading lookup scheme - if (!$symbparm) { $symbparm=&symbread(); } my $symbp=$symbparm; my $mapp=(&decode_symb($symbp))[0]; @@ -4063,11 +4089,11 @@ sub EXT { ($ENV{'user.domain'} eq $udom)) { $section=$ENV{'request.course.sec'}; } else { - if (! defined($usection)) { - $section=&usection($udom,$uname,$courseid); - } else { - $section = $usection; - } + if (! defined($usection)) { + $section=&usection($udom,$uname,$courseid); + } else { + $section = $usection; + } } my $seclevel=$courseid.'.['.$section.'].'.$spacequalifierrest; @@ -4105,7 +4131,7 @@ sub EXT { $uname." at ".$udom.": ". $tmp.""); } elsif ($tmp=~/error: 2 /) { - &EXT_cache_set($udom,$uname); + &EXT_cache_set($udom,$uname); } elsif ($tmp =~ /^(con_lost|no_such_host)/) { return $tmp; } @@ -4115,10 +4141,10 @@ sub EXT { # -------------------------------------------------------- second, check course my $coursereply=&courseresdata($ENV{'course.'.$courseid.'.num'}, - $ENV{'course.'.$courseid.'.domain'}, - ($seclevelr,$seclevelm,$seclevel, - $courselevelr,$courselevelm, - $courselevel)); + $ENV{'course.'.$courseid.'.domain'}, + ($seclevelr,$seclevelm,$seclevel, + $courselevelr,$courselevelm, + $courselevel)); if (defined($coursereply)) { return $coursereply; } # ------------------------------------------------------ third, check map parms @@ -4253,7 +4279,9 @@ sub metadata { unless ($filename=~/\.meta$/) { $filename.='.meta'; } my $metastring; if ($uri !~ m|^uploaded/|) { - $metastring=&getfile(&filelocation('',&clutter($filename))); + my $file=&filelocation('',&clutter($filename)); + push(@{$metacache{$uri.'.file'}},$file); + $metastring=&getfile($file); } my $parser=HTML::LCParser->new(\$metastring); my $token; @@ -4466,27 +4494,27 @@ sub metadata_generate_part0 { sub gettitle { my $urlsymb=shift; my $symb=&symbread($urlsymb); - unless ($symb) { - unless ($urlsymb) { $urlsymb=$ENV{'request.filename'}; } - return &metadata($urlsymb,'title'); - } - my ($result,$cached)=&is_cached(\%titlecache,$symb,'title',600); - if (defined($cached)) { return $result; } - my ($map,$resid,$url)=&decode_symb($symb); - my $title=''; - my %bighash; - if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', - &GDBM_READER(),0640)) { - my $mapid=$bighash{'map_pc_'.&clutter($map)}; - $title=$bighash{'title_'.$mapid.'.'.$resid}; - untie %bighash; - } - $title=~s/\&colon\;/\:/gs; - if ($title) { - return &do_cache(\%titlecache,$symb,$title,'title'); - } else { - return &metadata($urlsymb,'title'); - } + if ($symb) { + my ($result,$cached)=&is_cached(\%titlecache,$symb,'title',600); + if (defined($cached)) { return $result; } + my ($map,$resid,$url)=&decode_symb($symb); + my $title=''; + my %bighash; + if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', + &GDBM_READER(),0640)) { + my $mapid=$bighash{'map_pc_'.&clutter($map)}; + $title=$bighash{'title_'.$mapid.'.'.$resid}; + untie %bighash; + } + $title=~s/\&colon\;/\:/gs; + if ($title) { + return &do_cache(\%titlecache,$symb,$title,'title'); + } + $urlsymb=$url; + } + my $title=&metadata($urlsymb,'title'); + if (!$title) { $title=(split('/',$urlsymb))[-1]; } + return $title; } # ------------------------------------------------- Update symbolic store links @@ -4618,14 +4646,20 @@ sub deversion { sub symbread { my ($thisfn,$donotrecurse)=@_; + my $cache_str='request.symbread.cached.'.$thisfn; + if (defined($ENV{$cache_str})) { return $ENV{$cache_str}; } # no filename provided? try from environment unless ($thisfn) { - if ($ENV{'request.symb'}) { return &symbclean($ENV{'request.symb'}); } + if ($ENV{'request.symb'}) { + return $ENV{$cache_str}=&symbclean($ENV{'request.symb'}); + } $thisfn=$ENV{'request.filename'}; } # is that filename actually a symb? Verify, clean, and return if ($thisfn=~/\_\_\_\d+\_\_\_(.*)$/) { - if (&symbverify($thisfn,$1)) { return &symbclean($thisfn); } + if (&symbverify($thisfn,$1)) { + return $ENV{$cache_str}=&symbclean($thisfn); + } } $thisfn=declutter($thisfn); my %hash; @@ -4646,7 +4680,7 @@ sub symbread { unless ($syval=~/\_\d+$/) { unless ($ENV{'form.request.prefix'}=~/\.(\d+)\_$/) { &appenv('request.ambiguous' => $thisfn); - return ''; + return $ENV{$cache_str}=''; } $syval.=$1; } @@ -4693,11 +4727,11 @@ sub symbread { } } if ($syval) { - return &symbclean($syval.'___'.$thisfn); + return $ENV{$cache_str}=&symbclean($syval.'___'.$thisfn); } } &appenv('request.ambiguous' => $thisfn); - return ''; + return $ENV{$cache_str}=''; } # ---------------------------------------------------------- Return random seed @@ -4958,30 +4992,32 @@ sub receipt { # the local server. sub getfile { - my ($file,$caller) = @_; + my ($file) = @_; - if ($file !~ m|^/*uploaded/(\w+)/(\w+)/(.+)$|) { - # normal file from res space - &repcopy($file); - return &readfile($file); - } - - my $info; - my $cdom = $1; - my $cnum = $2; - my $filename = $3; - my $path = $Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles'; - my ($lwpresp,$rtncode); - my $localfile = $path.'/'.$cdom.'/'.$cnum.'/'.$filename; - if (-e "$localfile") { - my @fileinfo = stat($localfile); - $lwpresp = &getuploaded('HEAD',$file,$cdom,$cnum,\$info,\$rtncode); + if ($file =~ m|^/*uploaded/|) { $file=&filelocation("",$file); } + &repcopy($file); + return &readfile($file); +} + +sub repcopy_userfile { + my ($file)=@_; + + if ($file =~ m|^/*uploaded/|) { $file=&filelocation("",$file); } + if ($file =~ m|^/home/httpd/html/lonUsers/|) { return OK; } + + my ($cdom,$cnum,$filename) = + ($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+([^/]+)/+([^/]+)/+(.*)|); + my ($info,$rtncode); + my $uri="/uploaded/$cdom/$cnum/$filename"; + if (-e "$file") { + my @fileinfo = stat($file); + my $lwpresp = &getuploaded('HEAD',$uri,$cdom,$cnum,\$info,\$rtncode); if ($lwpresp ne 'ok') { if ($rtncode eq '404') { - unlink($localfile); + unlink($file); } #my $ua=new LWP::UserAgent; - #my $request=new HTTP::Request('GET',&tokenwrapper($file)); + #my $request=new HTTP::Request('GET',&tokenwrapper($uri)); #my $response=$ua->request($request); #if ($response->is_success()) { # return $response->content; @@ -4991,21 +5027,21 @@ sub getfile { return -1; } if ($info < $fileinfo[9]) { - return &readfile($localfile); + return OK; } $info = ''; - $lwpresp = &getuploaded('GET',$file,$cdom,$cnum,\$info,\$rtncode); + $lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode); if ($lwpresp ne 'ok') { return -1; } } else { - $lwpresp = &getuploaded('GET',$file,$cdom,$cnum,\$info,\$rtncode); + my $lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode); if ($lwpresp ne 'ok') { my $ua=new LWP::UserAgent; - my $request=new HTTP::Request('GET',&tokenwrapper($file)); + my $request=new HTTP::Request('GET',&tokenwrapper($uri)); my $response=$ua->request($request); if ($response->is_success()) { - return $response->content; + $info=$response->content; } else { return -1; } @@ -5014,6 +5050,7 @@ sub getfile { if ($filename =~ m|^(.+)/[^/]+$|) { push @parts, split(/\//,$1); } + my $path = $perlvar{'lonDocRoot'}.'/userfiles'; foreach my $part (@parts) { $path .= '/'.$part; if (!-e $path) { @@ -5021,13 +5058,10 @@ sub getfile { } } } - open (FILE,">$localfile"); + open(FILE,">$file"); print FILE $info; close(FILE); - if ($caller eq 'uploadrep') { - return 'ok'; - } - return $info; + return OK; } sub tokenwrapper { @@ -5083,20 +5117,18 @@ sub filelocation { $location = $file; $location =~ s:/~(.*?)/(.*):/home/$1/public_html/$2:; } elsif ($file=~/^\/*uploaded/) { # is an uploaded file - if ($file=~/^\/uploaded\/([^\/]+)\/([^\/]+)\/(\/)?simplepage\/([^\/]+)$/) { - $location=&Apache::loncommon::propath($1,$2).'/userfiles/simplepage/'.$4; - if (not -e $location) { - $file=~/^\/uploaded\/(.*)$/; - $location=$Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles/'.$1; - } - } elsif ($file=~/^\/uploaded\/([^\/]+)\/([^\/]+)\/aboutme\/([^\/]+)$/) { - $location=&Apache::loncommon::propath($1,$2).'/userfiles/aboutme/'.$3; - if (not -e $location) { - $file=~/^\/uploaded\/(.*)$/; - $location=$Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles/'.$1; - } + my ($udom,$uname,$filename)= + ($file=~m|^/+uploaded/+([^/]+)/+([^/]+)/+(.*)$|); + my $home=&homeserver($uname,$udom); + my $is_me=0; + my @ids=¤t_machine_ids(); + foreach my $id (@ids) { if ($id eq $home) { $is_me=1; } } + if ($is_me) { + $location=&Apache::loncommon::propath($udom,$uname). + '/userfiles/'.$filename; } else { - $location=$file; + $location=$Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles/'. + $udom.'/'.$uname.'/'.$filename; } } else { $file=~s/^\Q$perlvar{'lonDocRoot'}\E//; @@ -5293,10 +5325,6 @@ BEGIN { $hostip{$id}=$ip; $iphost{$ip}=$id; if ($role eq 'library') { $libserv{$id}=$name; } - } else { - if ($configline) { - &logthis("Skipping hosts.tab line -$configline-"); - } } } close($config);