--- loncom/lonnet/perl/lonnet.pm 2003/07/29 05:22:56 1.395 +++ loncom/lonnet/perl/lonnet.pm 2003/11/01 18:34:49 1.440 @@ -1,7 +1,7 @@ # The LearningOnline Network # TCP networking package # -# $Id: lonnet.pm,v 1.395 2003/07/29 05:22:56 albertel Exp $ +# $Id: lonnet.pm,v 1.440 2003/11/01 18:34:49 www Exp $ # # Copyright Michigan State University Board of Trustees # @@ -25,44 +25,6 @@ # # http://www.lon-capa.org/ # -# 6/1/99,6/2,6/10,6/11,6/12,6/14,6/26,6/28,6/29,6/30, -# 7/1,7/2,7/9,7/10,7/12,7/14,7/15,7/19, -# 11/8,11/16,11/18,11/22,11/23,12/22, -# 01/06,01/13,02/24,02/28,02/29, -# 03/01,03/02,03/06,03/07,03/13, -# 04/05,05/29,05/31,06/01, -# 06/05,06/26 Gerd Kortemeyer -# 06/26 Ben Tyszka -# 06/30,07/15,07/17,07/18,07/20,07/21,07/22,07/25 Gerd Kortemeyer -# 08/14 Ben Tyszka -# 08/22,08/28,08/31,09/01,09/02,09/04,09/05,09/25,09/28,09/30 Gerd Kortemeyer -# 10/04 Gerd Kortemeyer -# 10/04 Guy Albertelli -# 10/06,10/09,10/10,10/11,10/14,10/20,10/23,10/25,10/26,10/27,10/28,10/29, -# 10/30,10/31, -# 11/2,11/14,11/15,11/16,11/20,11/21,11/22,11/25,11/27, -# 12/02,12/12,12/13,12/14,12/28,12/29 Gerd Kortemeyer -# 05/01/01 Guy Albertelli -# 05/01,06/01,09/01 Gerd Kortemeyer -# 09/01 Guy Albertelli -# 09/01,10/01,11/01 Gerd Kortemeyer -# YEAR=2001 -# 3/2 Gerd Kortemeyer -# 3/19,3/20 Gerd Kortemeyer -# 5/26,5/28 Gerd Kortemeyer -# 5/30 H. K. Ng -# 6/1 Gerd Kortemeyer -# July Guy Albertelli -# 8/4,8/7,8/8,8/9,8/11,8/16,8/17,8/18,8/20,8/23,9/20,9/21,9/26, -# 10/2 Gerd Kortemeyer -# 11/17,11/20,11/22,11/29 Gerd Kortemeyer -# 12/5 Matthew Hall -# 12/5 Guy Albertelli -# 12/6,12/7,12/12 Gerd Kortemeyer -# 12/21,12/22,12/27,12/28 Gerd Kortemeyer -# YEAR=2002 -# 1/4,2/4,2/7 Gerd Kortemeyer -# ### package Apache::lonnet; @@ -73,17 +35,21 @@ use LWP::UserAgent(); use HTTP::Headers; use vars qw(%perlvar %hostname %homecache %badServerCache %hostip %iphost %spareid %hostdom - %libserv %pr %prp %metacache %packagetab %titlecache + %libserv %pr %prp %metacache %packagetab %titlecache %courseresversioncache %resversioncache %courselogs %accesshash %userrolehash $processmarker $dumpcount %coursedombuf %coursenumbuf %coursehombuf %coursedescrbuf %courseresdatacache - %domaindescription %domain_auth_def %domain_auth_arg_def $tmpdir); + %userresdatacache %usectioncache %domaindescription %domain_auth_def %domain_auth_arg_def + %domain_lang_def %domain_city %domain_longi %domain_lati $tmpdir); + use IO::Socket; use GDBM_File; use Apache::Constants qw(:common :http); use HTML::LCParser; use Fcntl qw(:flock); use Apache::loncoursedata; - +use Apache::lonlocal; +use Storable qw(lock_store lock_nstore lock_retrieve freeze thaw); +use Time::HiRes(); my $readit; # --------------------------------------------------------------------- Logging @@ -243,6 +209,20 @@ sub critical { } return $answer; } + +# +# -------------- Remove all key from the env that start witha lowercase letter +# (Which is always a lon-capa value) + +sub cleanenv { +# unless (defined(&Apache::exists_config_define("MODPERL2"))) { return; } +# unless (&Apache::exists_config_define("MODPERL2")) { return; } + foreach my $key (keys(%ENV)) { + if ($key =~ /^[a-z]/) { + delete($ENV{$key}); + } + } +} # ------------------------------------------- Transfer profile into environment @@ -256,10 +236,19 @@ sub transfer_profile_to_env { $idf->close(); } my $envi; + my %Remove; for ($envi=0;$envi<=$#profile;$envi++) { chomp($profile[$envi]); my ($envname,$envvalue)=split(/=/,$profile[$envi]); $ENV{$envname} = $envvalue; + if (my ($key,$time) = ($envname =~ /^(cgi\.(\d+)_\d+\.)/)) { + if ($time < time-300) { + $Remove{$key}++; + } + } + } + foreach my $expired_key (keys(%Remove)) { + &delenv($expired_key); } $ENV{'user.environment'} = "$lonidsdir/$handle.id"; } @@ -377,8 +366,8 @@ sub userload { my $curtime=time; while ($filename=readdir(LONIDS)) { if ($filename eq '.' || $filename eq '..') {next;} - my ($atime)=(stat($perlvar{'lonIDsDir'}.'/'.$filename))[8]; - if ($curtime-$atime < 3600) { $numusers++; } + my ($mtime)=(stat($perlvar{'lonIDsDir'}.'/'.$filename))[9]; + if ($curtime-$mtime < 1800) { $numusers++; } } closedir(LONIDS); } @@ -424,15 +413,27 @@ sub spareserver { my $lowestserver=$loadpercent > $userloadpercent? $loadpercent : $userloadpercent; foreach $tryserver (keys %spareid) { - my $loadans=reply('load',$tryserver); - my $userloadans=reply('userload',$tryserver); - if ($userloadans !~ /\d/) { $userloadans=0; } - my $answer=$loadans > $userloadans? - $loadans : $userloadans; - if (($answer =~ /\d/) && ($answer<$lowestserver)) { - $spareserver="http://$hostname{$tryserver}"; - $lowestserver=$answer; - } + my $loadans=reply('load',$tryserver); + my $userloadans=reply('userload',$tryserver); + if ($loadans !~ /\d/ && $userloadans !~ /\d/) { + next; #didn't get a number from the server + } + my $answer; + if ($loadans =~ /\d/) { + if ($userloadans =~ /\d/) { + #both are numbers, pick the bigger one + $answer=$loadans > $userloadans? + $loadans : $userloadans; + } else { + $answer = $loadans; + } + } else { + $answer = $userloadans; + } + if (($answer =~ /\d/) && ($answer<$lowestserver)) { + $spareserver="http://$hostname{$tryserver}"; + $lowestserver=$answer; + } } return $spareserver; } @@ -556,9 +557,9 @@ sub authenticate { sub homeserver { my ($uname,$udom,$ignoreBadCache)=@_; my $index="$uname:$udom"; - if ($homecache{$index}) { - return "$homecache{$index}"; - } + + my ($result,$cached)=&is_cached(\%homecache,$index,'home',86400); + if (defined($cached)) { return $result; } my $tryserver; foreach $tryserver (keys %libserv) { next if ($ignoreBadCache ne 'true' && @@ -566,8 +567,7 @@ sub homeserver { if ($hostdom{$tryserver} eq $udom) { my $answer=reply("home:$udom:$uname",$tryserver); if ($answer eq 'found') { - $homecache{$index}=$tryserver; - return $tryserver; + return &do_cache(\%homecache,$index,$tryserver,'home'); } elsif ($answer eq 'no_host') { $badServerCache{$tryserver}=1; } @@ -819,8 +819,166 @@ sub getsection { return '-1'; } +sub devalidate_cache { + my ($cache,$id,$name) = @_; + delete $$cache{$id.'.time'}; + delete $$cache{$id}; + my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + open(DB,"$filename.lock"); + flock(DB,LOCK_EX); + my %hash; + if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { + delete($hash{$id}); + delete($hash{$id.'.time'}); + } else { + &logthis("Unable to tie hash (devalidate cache): $name"); + } + untie(%hash); + flock(DB,LOCK_UN); + close(DB); +} + +sub is_cached { + my ($cache,$id,$name,$time) = @_; + if (!$time) { $time=300; } + if (!exists($$cache{$id.'.time'})) { + &load_cache_item($cache,$name,$id); + } + if (!exists($$cache{$id.'.time'})) { +# &logthis("Didn't find $id"); + return (undef,undef); + } else { + if (time-($$cache{$id.'.time'})>$time) { +# &logthis("Devalidating $id - ".time-($$cache{$id.'.time'})); + &devalidate_cache($cache,$id,$name); + return (undef,undef); + } + } + return ($$cache{$id},1); +} + +sub do_cache { + my ($cache,$id,$value,$name) = @_; + $$cache{$id.'.time'}=time; + $$cache{$id}=$value; +# &logthis("Caching $id as :$value:"); + &save_cache_item($cache,$name,$id); + # do_cache implictly return the set value + $$cache{$id}; +} + +sub save_cache { + my ($cache,$name)=@_; + my $starttime=&Time::HiRes::time(); +# &logthis("Saving :$name:"); + eval lock_store($cache,$perlvar{'lonDaemons'}.'/tmp/'.$name.".storable"); + if ($@) { &logthis("lock_store threw a die ".$@); } +# &logthis("save_cache took ".(&Time::HiRes::time()-$starttime)); +} + +sub load_cache { + my ($cache,$name)=@_; + my $starttime=&Time::HiRes::time(); +# &logthis("Before Loading $name size is ".scalar(%$cache)); + my $tmpcache; + eval { + $tmpcache=lock_retrieve($perlvar{'lonDaemons'}.'/tmp/'.$name.".storable"); + }; + if ($@) { &logthis("lock_retreive threw a die ".$@); return; } + if (!%$cache) { + my $count; + while (my ($key,$value)=each(%$tmpcache)) { + $count++; + $$cache{$key}=$value; + } +# &logthis("Initial load: $count"); + } else { + my $key; + my $count; + while ($key=each(%$tmpcache)) { + if ($key !~/^(.*)\.time$/) { next; } + my $name=$1; + if (exists($$cache{$key})) { + if ($$tmpcache{$key} >= $$cache{$key}) { + $$cache{$key}=$$tmpcache{$key}; + $$cache{$name}=$$tmpcache{$name}; + } else { +# &logthis("Would have overwritten $name with is set to expire at ".$$cache{$key}." with ".$$tmpcache{$key}." Whew!"); + } + } else { + $count++; + $$cache{$key}=$$tmpcache{$key}; + $$cache{$name}=$$tmpcache{$name}; + } + } +# &logthis("Additional load: $count"); + } +# &logthis("After Loading $name size is ".scalar(%$cache)); +# &logthis("load_cache took ".(&Time::HiRes::time()-$starttime)); +} + +sub save_cache_item { + my ($cache,$name,$id)=@_; + my $starttime=&Time::HiRes::time(); + # &logthis("Saving :$name:$id"); + my %hash; + my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + open(DB,"$filename.lock"); + flock(DB,LOCK_EX); + if (tie(%hash,'GDBM_File',$filename,&GDBM_WRCREAT(),0640)) { + $hash{$id.'.time'}=$$cache{$id.'.time'}; + $hash{$id}=freeze({'item'=>$$cache{$id}}); + } else { + &logthis("Unable to tie hash (save cache item): $name"); + } + untie(%hash); + flock(DB,LOCK_UN); + close(DB); +# &logthis("save_cache_item $name took ".(&Time::HiRes::time()-$starttime)); +} + +sub load_cache_item { + my ($cache,$name,$id)=@_; + my $starttime=&Time::HiRes::time(); +# &logthis("Before Loading $name for $id size is ".scalar(%$cache)); + my %hash; + my $filename=$perlvar{'lonDaemons'}.'/tmp/'.$name.".db"; + open(DB,"$filename.lock"); + flock(DB,LOCK_SH); + if (tie(%hash,'GDBM_File',$filename,&GDBM_READER(),0640)) { + if (!%$cache) { + my $count; + while (my ($key,$value)=each(%hash)) { + $count++; + if ($key =~ /\.time$/) { + $$cache{$key}=$value; + } else { + my $hashref=thaw($value); + $$cache{$key}=$hashref->{'item'}; + } + } +# &logthis("Initial load: $count"); + } else { + my $hashref=thaw($hash{$id}); + $$cache{$id}=$hashref->{'item'}; + $$cache{$id.'.time'}=$hash{$id.'.time'}; + } + } else { + &logthis("Unable to tie hash (load cache item): $name"); + } + untie(%hash); + flock(DB,LOCK_UN); + close(DB); +# &logthis("After Loading $name size is ".scalar(%$cache)); +# &logthis("load_cache_item $name took ".(&Time::HiRes::time()-$starttime)); +} + sub usection { my ($udom,$unam,$courseid)=@_; + my $hashid="$udom:$unam:$courseid"; + + my ($result,$cached)=&is_cached(\%usectioncache,$hashid,'usection'); + if (defined($cached)) { return $result; } $courseid=~s/\_/\//g; $courseid=~s/^(\w)/\/$1/; foreach (split(/\&/,&reply('dump:'.$udom.':'.$unam.':roles', @@ -839,10 +997,12 @@ sub usection { if ($end) { if ($now>$end) { $notactive=1; } } - unless ($notactive) { return $section; } + unless ($notactive) { + return &do_cache(\%usectioncache,$hashid,$section,'usection'); + } } } - return '-1'; + return &do_cache(\%usectioncache,$hashid,'-1','usection'); } # ------------------------------------- Read an entry from a user's environment @@ -882,6 +1042,8 @@ sub getversion { sub currentversion { my $fname=shift; + my ($result,$cached)=&is_cached(\%resversioncache,$fname,'resversion',600); + if (defined($cached)) { return $result; } my $author=$fname; $author=~s/\/home\/httpd\/html\/res\/([^\/]*)\/([^\/]*).*/$1\/$2/; my ($udom,$uname)=split(/\//,$author); @@ -893,7 +1055,7 @@ sub currentversion { if (($answer eq 'con_lost') || ($answer eq 'rejected')) { return -1; } - return $answer; + return &do_cache(\%resversioncache,$fname,$answer,'resversion'); } # ----------------------------- Subscribe to a resource, return URL if possible @@ -1216,7 +1378,7 @@ sub courseacclog { my $fnsymb=shift; unless ($ENV{'request.course.id'}) { return ''; } my $what=$fnsymb.':'.$ENV{'user.name'}.':'.$ENV{'user.domain'}; - if ($fnsymb=~/(problem|exam|quiz|assess|survey|form)$/) { + if ($fnsymb=~/(problem|exam|quiz|assess|survey|form|page)$/) { $what.=':POST'; foreach (keys %ENV) { if ($_=~/^form\.(.*)/) { @@ -1284,6 +1446,53 @@ sub get_course_adv_roles { return %returnhash; } +sub get_my_roles { + my ($uname,$udom)=@_; + unless (defined($uname)) { $uname=$ENV{'user.name'}; } + unless (defined($udom)) { $udom=$ENV{'user.domain'}; } + my %dumphash= + &dump('nohist_userroles',$udom,$uname); + my %returnhash=(); + my $now=time; + foreach (keys %dumphash) { + my ($tend,$tstart)=split(/\:/,$dumphash{$_}); + if (($tstart) && ($tstart<0)) { next; } + if (($tend) && ($tend<$now)) { next; } + if (($tstart) && ($now<$tstart)) { next; } + my ($role,$username,$domain,$section)=split(/\:/,$_); + $returnhash{$username.':'.$domain.':'.$role}=$tstart.':'.$tend; + } + return %returnhash; +} + +# ----------------------------------------------------- Frontpage Announcements +# +# + +sub postannounce { + my ($server,$text)=@_; + unless (&allowed('psa',$hostdom{$server})) { return 'refused'; } + unless ($text=~/\w/) { $text=''; } + return &reply('setannounce:'.&escape($text),$server); +} + +sub getannounce { + if (my $fh=Apache::File->new($perlvar{'lonDocRoot'}.'/announcement.txt')) { + my $announcement=''; + while (<$fh>) { $announcement .=$_; } + $fh->close(); + if ($announcement=~/\w/) { + return + ''. + '
'.$announcement.'
'; + } else { + return ''; + } + } else { + return ''; + } +} + # ---------------------------------------------------------- Course ID routines # Deal with domain's nohist_courseid.db files # @@ -2077,6 +2286,21 @@ sub dump { return %returnhash; } +# -------------------------------------------------------------- keys interface + +sub getkeys { + my ($namespace,$udomain,$uname)=@_; + if (!$udomain) { $udomain=$ENV{'user.domain'}; } + if (!$uname) { $uname=$ENV{'user.name'}; } + my $uhome=&homeserver($uname,$udomain); + my $rep=reply("keys:$udomain:$uname:$namespace",$uhome); + my @keyarray=(); + foreach (split(/\&/,$rep)) { + push (@keyarray,&unescape($_)); + } + return @keyarray; +} + # --------------------------------------------------------------- currentdump sub currentdump { my ($courseid,$sdom,$sname)=@_; @@ -2096,27 +2320,7 @@ sub currentdump { return if ($tmp[0] =~ /^(error:|no_such_host)/); my %hash = @tmp; @tmp=(); - # Code ripped from lond, essentially. The only difference - # here is the unescaping done by lonnet::dump(). Conceivably - # we might run in to problems with parameter names =~ /^v\./ - while (my ($key,$value) = each(%hash)) { - my ($v,$symb,$param) = split(/:/,$key); - next if ($v eq 'version' || $symb eq 'keys'); - next if (exists($returnhash{$symb}) && - exists($returnhash{$symb}->{$param}) && - $returnhash{$symb}->{'v.'.$param} > $v); - $returnhash{$symb}->{$param}=$value; - $returnhash{$symb}->{'v.'.$param}=$v; - } - # - # Remove all of the keys in the hashes which keep track of - # the version of the parameter. - while (my ($symb,$param_hash) = each(%returnhash)) { - # use a foreach because we are going to delete from the hash. - foreach my $key (keys(%$param_hash)) { - delete($param_hash->{$key}) if ($key =~ /^v\./); - } - } + %returnhash = %{&convert_dump_to_currentdump(\%hash)}; } else { my @pairs=split(/\&/,$rep); foreach (@pairs) { @@ -2129,6 +2333,33 @@ sub currentdump { return %returnhash; } +sub convert_dump_to_currentdump{ + my %hash = %{shift()}; + my %returnhash; + # Code ripped from lond, essentially. The only difference + # here is the unescaping done by lonnet::dump(). Conceivably + # we might run in to problems with parameter names =~ /^v\./ + while (my ($key,$value) = each(%hash)) { + my ($v,$symb,$param) = split(/:/,$key); + next if ($v eq 'version' || $symb eq 'keys'); + next if (exists($returnhash{$symb}) && + exists($returnhash{$symb}->{$param}) && + $returnhash{$symb}->{'v.'.$param} > $v); + $returnhash{$symb}->{$param}=$value; + $returnhash{$symb}->{'v.'.$param}=$v; + } + # + # Remove all of the keys in the hashes which keep track of + # the version of the parameter. + while (my ($symb,$param_hash) = each(%returnhash)) { + # use a foreach because we are going to delete from the hash. + foreach my $key (keys(%$param_hash)) { + delete($param_hash->{$key}) if ($key =~ /^v\./); + } + } + return \%returnhash; +} + # --------------------------------------------------------------- put interface sub put { @@ -2209,6 +2440,9 @@ sub customaccess { $access=($effect eq 'allow'); last; } + if ($realm eq '' && $role eq '') { + $access=($effect eq 'allow'); + } } return $access; } @@ -2217,10 +2451,11 @@ sub customaccess { sub allowed { my ($priv,$uri)=@_; - + $uri=&deversion($uri); my $orguri=$uri; $uri=&declutter($uri); + if (defined($ENV{'allowed.'.$priv})) { return $ENV{'allowed.'.$priv}; } # Free bre access to adm and meta resources if ((($uri=~/^adm\//) || ($uri=~/\.meta$/)) && ($priv eq 'bre')) { @@ -2501,6 +2736,7 @@ sub allowed { sub is_on_map { my $uri=&declutter(shift); + $uri=~s/\.\d+\.(\w+)$/\.$1/; my @uriparts=split(/\//,$uri); my $filename=$uriparts[$#uriparts]; my $pathname=$uri; @@ -2516,6 +2752,29 @@ sub is_on_map { } } +# --------------------------------------------------------- Get symb from alias + +sub get_symb_from_alias { + my $symb=shift; + my ($map,$resid,$url)=&decode_symb($symb); +# Already is a symb + if ($url) { return $symb; } +# Must be an alias + my $aliassymb=''; + my %bighash; + if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', + &GDBM_READER(),0640)) { + my $rid=$bighash{'mapalias_'.$symb}; + if ($rid) { + my ($mapid,$resid)=split(/\./,$rid); + $aliassymb=&encode_symb($bighash{'map_id_'.$mapid}, + $resid,$bighash{'src_'.$rid}); + } + untie %bighash; + } + return $aliassymb; +} + # ----------------------------------------------------------------- Define Role sub definerole { @@ -2642,7 +2901,7 @@ sub userlog_query { sub plaintext { my $short=shift; - return $prp{$short}; + return &mt($prp{$short}); } # ----------------------------------------------------------------- Assign Role @@ -2746,7 +3005,8 @@ sub modifyuser { ' in domain '.$ENV{'request.role.domain'}); my $uhome=&homeserver($uname,$udom,'true'); # ----------------------------------------------------------------- Create User - if (($uhome eq 'no_host') && ($umode) && ($upass)) { + if (($uhome eq 'no_host') && + (($umode && $upass) || ($umode eq 'localauth'))) { my $unhome=''; if (defined($desiredhome) && $hostdom{$desiredhome} eq $udom) { $unhome = $desiredhome; @@ -3070,7 +3330,7 @@ sub dirlist { } my $alldomstr=''; foreach (sort keys %alldom) { - $alldomstr.=$perlvar{'lonDocRoot'}.'/res/'.$_.'&domain:'; + $alldomstr.=$perlvar{'lonDocRoot'}.'/res/'.$_.'/&domain:'; } $alldomstr=~s/:$//; return split(/:/,$alldomstr); @@ -3086,6 +3346,13 @@ sub dirlist { # when it was last modified. It will also return an error of -1 # if an error occurs +## +## FIXME: This subroutine assumes its caller knows something about the +## directory structure of the home server for the student ($root). +## Not a good assumption to make. Since this is for looking up files +## in user directories, the full path should be constructed by lond, not +## whatever machine we request data from. +## sub GetFileTimestamp { my ($studentDomain,$studentName,$filename,$root)=@_; $studentDomain=~s/\W//g; @@ -3162,7 +3429,7 @@ sub condval { sub devalidatecourseresdata { my ($coursenum,$coursedomain)=@_; my $hashid=$coursenum.':'.$coursedomain; - delete $courseresdatacache{$hashid.'.time'}; + &devalidate_cache(\%courseresdatacache,$hashid,'courseres'); } # --------------------------------------------------- Course Resourcedata Query @@ -3171,25 +3438,23 @@ sub courseresdata { my ($coursenum,$coursedomain,@which)=@_; my $coursehom=&homeserver($coursenum,$coursedomain); my $hashid=$coursenum.':'.$coursedomain; - my $dodump=0; - if (!defined($courseresdatacache{$hashid.'.time'})) { - $dodump=1; - } else { - if (time-$courseresdatacache{$hashid.'.time'}>300) { $dodump=1; } - } - if ($dodump) { + my ($result,$cached)=&is_cached(\%courseresdatacache,$hashid,'courseres'); + unless (defined($cached)) { my %dumpreply=&dump('resourcedata',$coursedomain,$coursenum); + $result=\%dumpreply; my ($tmp) = keys(%dumpreply); if ($tmp !~ /^(con_lost|error|no_such_host)/i) { - $courseresdatacache{$hashid.'.time'}=time; - $courseresdatacache{$hashid}=\%dumpreply; + &do_cache(\%courseresdatacache,$hashid,$result,'courseres'); } elsif ($tmp =~ /^(con_lost|no_such_host)/) { return $tmp; + } elsif ($tmp =~ /^(error)/) { + $result=undef; + &do_cache(\%courseresdatacache,$hashid,$result,'courseres'); } } foreach my $item (@which) { - if (defined($courseresdatacache{$hashid}->{$item})) { - return $courseresdatacache{$hashid}->{$item}; + if (defined($result->{$item})) { + return $result->{$item}; } } return undef; @@ -3228,6 +3493,9 @@ sub EXT { #get real user name/domain, courseid and symb my $courseid; my $publicuser; + if ($symbparm) { + $symbparm=&get_symb_from_alias($symbparm); + } if (!($uname && $udom)) { (my $cursymb,$courseid,$udom,$uname,$publicuser)= &Apache::lonxml::whichuser($symbparm); @@ -3313,7 +3581,15 @@ sub EXT { } elsif ($realm eq 'request') { # ------------------------------------------------------------- request.browser if ($space eq 'browser') { - return $ENV{'browser.'.$qualifier}; + if ($qualifier eq 'textremote') { + if (&mt('textual_remote_display') eq 'on') { + return 1; + } else { + return 0; + } + } else { + return $ENV{'browser.'.$qualifier}; + } # ------------------------------------------------------------ request.filename } else { return $ENV{'request.'.$spacequalifierrest}; @@ -3331,7 +3607,7 @@ sub EXT { # ----------------------------------------------------- Cascading lookup scheme if (!$symbparm) { $symbparm=&symbread(); } my $symbp=$symbparm; - my $mapp=(split(/\_\_\_/,$symbp))[0]; + my $mapp=(&decode_symb($symbp))[0]; my $symbparm=$symbp.'.'.$spacequalifierrest; my $mapparm=$mapp.'___(all).'.$spacequalifierrest; @@ -3357,19 +3633,25 @@ sub EXT { # ----------------------------------------------------------- first, check user #most student don\'t have any data set, check if there is some data - #every thirty minutes if (! &EXT_cache_status($udom,$uname)) { - my %resourcedata=&get('resourcedata', - [$courselevelr,$courselevelm,$courselevel], - $udom,$uname); - my ($tmp)=keys(%resourcedata); + my $hashid="$udom:$uname"; + my ($result,$cached)=&is_cached(\%userresdatacache,$hashid, + 'userres'); + if (!defined($cached)) { + my %resourcedata=&get('resourcedata', + [$courselevelr,$courselevelm, + $courselevel],$udom,$uname); + $result=\%resourcedata; + &do_cache(\%userresdatacache,$hashid,$result,'userres'); + } + my ($tmp)=keys(%$result); if (($tmp!~/^error\:/) && ($tmp!~/^con_lost/)) { - if ($resourcedata{$courselevelr}) { - return $resourcedata{$courselevelr}; } - if ($resourcedata{$courselevelm}) { - return $resourcedata{$courselevelm}; } - if ($resourcedata{$courselevel}) { - return $resourcedata{$courselevel}; } + if ($$result{$courselevelr}) { + return $$result{$courselevelr}; } + if ($$result{$courselevelm}) { + return $$result{$courselevelm}; } + if ($$result{$courselevel}) { + return $$result{$courselevel}; } } else { if ($tmp!~/No such file/) { &logthis("WARNING:". @@ -3410,7 +3692,7 @@ sub EXT { my $filename; if (!$symbparm) { $symbparm=&symbread(); } if ($symbparm) { - $filename=(split(/\_\_\_/,$symbparm))[2]; + $filename=(&decode_symb($symbparm))[2]; } else { $filename=$ENV{'request.filename'}; } @@ -3454,7 +3736,6 @@ sub EXT { sub packages_tab_default { my ($uri,$varname)=@_; - &logthis(" $varname"); my (undef,$part,$name)=split(/\./,$varname); my $packages=&metadata($uri,'packages'); foreach my $package (split(/,/,$packages)) { @@ -3486,11 +3767,11 @@ sub add_prefix_and_part { sub metadata { my ($uri,$what,$liburi,$prefix,$depthcount)=@_; - $uri=&declutter($uri); # if it is a non metadata possible uri return quickly if (($uri eq '') || (($uri =~ m|^/*adm/|) && ($uri !~ m|^adm/includes|)) || - ($uri =~ m|/$|) || ($uri =~ m|/.meta$|)) { + ($uri =~ m|/$|) || ($uri =~ m|/.meta$|) || ($uri =~ /^~/) || + ($uri =~ m|home/[^/]+/public_html/|)) { return ''; } my $filename=$uri; @@ -3500,21 +3781,27 @@ sub metadata { # Look at timestamp of caching # Everything is cached by the main uri, libraries are never directly cached # - unless (abs($metacache{$uri.':cachedtimestamp'}-time)<600 && !defined($liburi)) { + if (!defined($liburi)) { + my ($result,$cached)=&is_cached(\%metacache,$uri,'meta'); + if (defined($cached)) { return $result->{':'.$what}; } + } + { # # Is this a recursive call for a library? # + my %lcmetacache; if ($liburi) { $liburi=&declutter($liburi); $filename=$liburi; - } + } else { + &devalidate_cache(\%metacache,$uri,'meta'); + } my %metathesekeys=(); unless ($filename=~/\.meta$/) { $filename.='.meta'; } my $metastring=&getfile(&filelocation('',&clutter($filename))); my $parser=HTML::LCParser->new(\$metastring); my $token; undef %metathesekeys; - delete($metacache{$uri.':packages'}); while ($token=$parser->get_token) { if ($token->[0] eq 'S') { if (defined($token->[2]->{'package'})) { @@ -3526,32 +3813,39 @@ sub metadata { if (defined($token->[2]->{'id'})) { $keyroot.='_'.$token->[2]->{'id'}; } - if ($metacache{$uri.':packages'}) { - $metacache{$uri.':packages'}.=','.$package.$keyroot; + if ($lcmetacache{':packages'}) { + $lcmetacache{':packages'}.=','.$package.$keyroot; } else { - $metacache{$uri.':packages'}=$package.$keyroot; + $lcmetacache{':packages'}=$package.$keyroot; } foreach (keys %packagetab) { - if ($_=~/^$package\&/) { + my $part=$keyroot; + $part=~s/^\_//; + if ($_=~/^\Q$package\E\&/ || + $_=~/^\Q$package\E_0\&/) { my ($pack,$name,$subp)=split(/\&/,$_); # ignore package.tab specified default values # here &package_tab_default() will fetch those if ($subp eq 'default') { next; } my $value=$packagetab{$_}; - my $part=$keyroot; - $part=~s/^\_//; + my $unikey; + if ($pack =~ /_0$/) { + $unikey='parameter_0_'.$name; + $part=0; + } else { + $unikey='parameter'.$keyroot.'_'.$name; + } if ($subp eq 'display') { $value.=' [Part: '.$part.']'; } - my $unikey='parameter'.$keyroot.'_'.$name; - $metacache{$uri.':'.$unikey.'.part'}=$part; + $lcmetacache{':'.$unikey.'.part'}=$part; $metathesekeys{$unikey}=1; - unless (defined($metacache{$uri.':'.$unikey.'.'.$subp})) { - $metacache{$uri.':'.$unikey.'.'.$subp}=$value; + unless (defined($lcmetacache{':'.$unikey.'.'.$subp})) { + $lcmetacache{':'.$unikey.'.'.$subp}=$value; } - if (defined($metacache{$uri.':'.$unikey.'.default'})) { - $metacache{$uri.':'.$unikey}= - $metacache{$uri.':'.$unikey.'.default'}; + if (defined($lcmetacache{':'.$unikey.'.default'})) { + $lcmetacache{':'.$unikey}= + $lcmetacache{':'.$unikey.'.default'}; } } } @@ -3594,18 +3888,18 @@ sub metadata { } $metathesekeys{$unikey}=1; foreach (@{$token->[3]}) { - $metacache{$uri.':'.$unikey.'.'.$_}=$token->[2]->{$_}; + $lcmetacache{':'.$unikey.'.'.$_}=$token->[2]->{$_}; } my $internaltext=&HTML::Entities::decode($parser->get_text('/'.$entry)); - my $default=$metacache{$uri.':'.$unikey.'.default'}; + my $default=$lcmetacache{':'.$unikey.'.default'}; if ( $internaltext =~ /^\s*$/ && $default !~ /^\s*$/) { # only ws inside the tag, and not in default, so use default # as value - $metacache{$uri.':'.$unikey}=$default; + $lcmetacache{':'.$unikey}=$default; } else { # either something interesting inside the tag or default # uninteresting - $metacache{$uri.':'.$unikey}=$internaltext; + $lcmetacache{':'.$unikey}=$internaltext; } # end of not-a-package not-a-library import } @@ -3615,13 +3909,13 @@ sub metadata { } } # are there custom rights to evaluate - if ($metacache{$uri.':copyright'} eq 'custom') { + if ($lcmetacache{':copyright'} eq 'custom') { # # Importing a rights file here # unless ($depthcount) { - my $location=$metacache{$uri.':customdistributionfile'}; + my $location=$lcmetacache{':customdistributionfile'}; my $dir=$filename; $dir=~s|[^/]*$||; $location=&filelocation($dir,$location); @@ -3632,13 +3926,13 @@ sub metadata { } } } - $metacache{$uri.':keys'}=join(',',keys %metathesekeys); - &metadata_generate_part0(\%metathesekeys,\%metacache,$uri); - $metacache{$uri.':allpossiblekeys'}=join(',',keys %metathesekeys); - $metacache{$uri.':cachedtimestamp'}=time; + $lcmetacache{':keys'}=join(',',keys %metathesekeys); + &metadata_generate_part0(\%metathesekeys,\%lcmetacache,$uri); + $lcmetacache{':allpossiblekeys'}=join(',',keys %metathesekeys); + &do_cache(\%metacache,$uri,\%lcmetacache,'meta'); # this is the end of "was not already recently cached } - return $metacache{$uri.':'.$what}; + return $metacache{$uri}->{':'.$what}; } sub metadata_generate_part0 { @@ -3646,8 +3940,8 @@ sub metadata_generate_part0 { my %allnames; foreach my $metakey (sort keys %$metadata) { if ($metakey=~/^parameter\_(.*)/) { - my $part=$$metacache{$uri.':'.$metakey.'.part'}; - my $name=$$metacache{$uri.':'.$metakey.'.name'}; + my $part=$$metacache{':'.$metakey.'.part'}; + my $name=$$metacache{':'.$metakey.'.name'}; if (! exists($$metadata{'parameter_0_'.$name.'.name'})) { $allnames{$name}=$part; } @@ -3655,13 +3949,13 @@ sub metadata_generate_part0 { } foreach my $name (keys(%allnames)) { $$metadata{"parameter_0_$name"}=1; - my $key="$uri:parameter_0_$name"; + my $key=":parameter_0_$name"; $$metacache{"$key.part"}='0'; $$metacache{"$key.name"}=$name; - $$metacache{"$key.type"}=$$metacache{$uri.':parameter_'. + $$metacache{"$key.type"}=$$metacache{':parameter_'. $allnames{$name}.'_'.$name. '.type'}; - my $olddis=$$metacache{$uri.':parameter_'.$allnames{$name}.'_'.$name. + my $olddis=$$metacache{':parameter_'.$allnames{$name}.'_'.$name. '.display'}; my $expr='\\[Part: '.$allnames{$name}.'\\]'; $olddis=~s/$expr/\[Part: 0\]/; @@ -3678,14 +3972,9 @@ sub gettitle { unless ($urlsymb) { $urlsymb=$ENV{'request.filename'}; } return &metadata($urlsymb,'title'); } - if ($titlecache{$symb}) { - if (time < ($titlecache{$symb}[1] + 600)) { - return $titlecache{$symb}[0]; - } else { - delete($titlecache{$symb}); - } - } - my ($map,$resid,$url)=split(/\_\_\_/,$symb); + my ($result,$cached)=&is_cached(\%titlecache,$symb,'title',600); + if (defined($cached)) { return $result; } + my ($map,$resid,$url)=&decode_symb($symb); my $title=''; my %bighash; if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', @@ -3696,8 +3985,7 @@ sub gettitle { } $title=~s/\&colon\;/\:/gs; if ($title) { - $titlecache{$symb}=[$title,time]; - return $title; + return &do_cache(\%titlecache,$symb,$title,'title'); } else { return &metadata($urlsymb,'title'); } @@ -3707,13 +3995,13 @@ sub gettitle { sub symblist { my ($mapname,%newhash)=@_; - $mapname=declutter($mapname); + $mapname=&deversion(&declutter($mapname)); my %hash; if (($ENV{'request.course.fn'}) && (%newhash)) { if (tie(%hash,'GDBM_File',$ENV{'request.course.fn'}.'_symb.db', &GDBM_WRCREAT(),0640)) { foreach (keys %newhash) { - $hash{declutter($_)}=$mapname.'___'.$newhash{$_}; + $hash{declutter($_)}=$mapname.'___'.&deversion($newhash{$_}); } if (untie(%hash)) { return 'ok'; @@ -3731,13 +4019,16 @@ sub symbverify { # direct jump to resource in page or to a sequence - will construct own symbs if ($thisfn=~/\.(page|sequence)$/) { return 1; } # check URL part - my ($map,$resid,$url)=split(/\_\_\_/,$symb); - unless (&symbclean($url) eq &symbclean($thisfn)) { return 0; } + my ($map,$resid,$url)=&decode_symb($symb); + + unless ($url eq $thisfn) { return 0; } $symb=&symbclean($symb); + $thisfn=&deversion($thisfn); my %bighash; my $okay=0; + if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', &GDBM_READER(),0640)) { my $ids=$bighash{'ids_'.&clutter($thisfn)}; @@ -3774,6 +4065,49 @@ sub symbclean { return $symb; } +# ---------------------------------------------- Split symb to find map and url + +sub encode_symb { + my ($map,$resid,$url)=@_; + return &symbclean(&declutter($map).'___'.$resid.'___'.&declutter($url)); +} + +sub decode_symb { + my ($map,$resid,$url)=split(/\_\_\_/,shift); + return (&fixversion($map),$resid,&fixversion($url)); +} + +sub fixversion { + my $fn=shift; + if ($fn=~/^(adm|uploaded|public)/) { return $fn; } + my %bighash; + my $uri=&clutter($fn); + my $key=$ENV{'request.course.id'}.'_'.$uri; +# is this cached? + my ($result,$cached)=&is_cached(\%courseresversioncache,$key, + 'courseresversion',600); + if (defined($cached)) { return $result; } +# unfortunately not cached, or expired + if (tie(%bighash,'GDBM_File',$ENV{'request.course.fn'}.'.db', + &GDBM_READER(),0640)) { + if ($bighash{'version_'.$uri}) { + my $version=$bighash{'version_'.$uri}; + unless ($version eq 'mostrecent') { + $uri=~s/\.(\w+)$/\.$version\.$1/; + } + } + untie %bighash; + } + return &do_cache + (\%courseresversioncache,$key,&declutter($uri),'courseresversion'); +} + +sub deversion { + my $url=shift; + $url=~s/\.\d+\.(\w+)$/\.$1/; + return $url; +} + # ------------------------------------------------------ Return symb list entry sub symbread { @@ -4073,10 +4407,35 @@ sub unescape { return $str; } +sub mod_perl_version { + if (defined($perlvar{'MODPERL2'})) { + return 2; + } + return 1; +} + +sub correct_line_ends { + my ($result)=@_; + &logthis("Wha $result"); + $$result =~s/\r\n/\n/mg; + $$result =~s/\r/\n/mg; +} # ================================================================ Main Program sub goodbye { &logthis("Starting Shut down"); +#not converted to using infrastruture + &logthis(sprintf("%-20s is %s",'%homecache',scalar(%homecache))); + &logthis(sprintf("%-20s is %s",'%badServerCache',scalar(%badServerCache))); + &logthis(sprintf("%-20s is %s",'%metacache',scalar(%metacache))); +#converted + &logthis(sprintf("%-20s is %s",'%titlecache',scalar(%titlecache))); + &logthis(sprintf("%-20s is %s",'%courseresdatacache',scalar(%courseresdatacache))); +#1.1 only + &logthis(sprintf("%-20s is %s",'%userresdatacache',scalar(%userresdatacache))); + &logthis(sprintf("%-20s is %s",'%usectioncache',scalar(%usectioncache))); + &logthis(sprintf("%-20s is %s",'%courseresversioncache',scalar(%courseresversioncache))); + &logthis(sprintf("%-20s is %s",'%resversioncache',scalar(%resversioncache))); &flushcourselogs(); &logthis("Shutting down"); return DONE; @@ -4120,11 +4479,16 @@ BEGIN { next if (/^(\#|\s*$)/); # next if /^\#/; chomp; - my ($domain, $domain_description, $def_auth, $def_auth_arg) - = split(/:/,$_,4); - $domain_auth_def{$domain}=$def_auth; + my ($domain, $domain_description, $def_auth, $def_auth_arg, + $def_lang, $city, $longi, $lati) = split(/:/,$_); + $domain_auth_def{$domain}=$def_auth; $domain_auth_arg_def{$domain}=$def_auth_arg; - $domaindescription{$domain}=$domain_description; + $domaindescription{$domain}=$domain_description; + $domain_lang_def{$domain}=$def_lang; + $domain_city{$domain}=$city; + $domain_longi{$domain}=$longi; + $domain_lati{$domain}=$lati; + # &logthis("Domain.tab: $domain, $domain_auth_def{$domain}, $domain_auth_arg_def{$domain},$domaindescription{$domain}"); # &logthis("Domain.tab: $domain ".$domaindescription{$domain} ); }