--- loncom/lonnet/perl/lonnet.pm 2010/08/17 00:19:24 1.1056.4.3 +++ loncom/lonnet/perl/lonnet.pm 2010/07/06 18:36:56 1.1072 @@ -1,7 +1,7 @@ # The LearningOnline Network # TCP networking package # -# $Id: lonnet.pm,v 1.1056.4.3 2010/08/17 00:19:24 raeburn Exp $ +# $Id: lonnet.pm,v 1.1072 2010/07/06 18:36:56 www Exp $ # # Copyright Michigan State University Board of Trustees # @@ -76,7 +76,7 @@ use HTTP::Date; use Image::Magick; use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir - $_64bit %env %protocol %loncaparevs %serverhomeIDs); + $_64bit %env %protocol); my (%badServerCache, $memcache, %courselogs, %accesshash, %domainrolehash, %userrolehash, $processmarker, $dumpcount, %coursedombuf, @@ -196,7 +196,7 @@ sub get_server_timezone { } sub get_server_loncaparev { - my ($dom,$lonhost,$ignore_cache,$caller) = @_; + my ($dom,$lonhost) = @_; if (defined($lonhost)) { if (!defined(&hostname($lonhost))) { undef($lonhost); @@ -211,74 +211,15 @@ sub get_server_loncaparev { } } if (defined($lonhost)) { - my $cachetime = 12*3600; - if (!$ignore_cache) { - my ($loncaparev,$cached)=&is_cached_new('serverloncaparev',$lonhost); - if (defined($cached)) { - return $loncaparev; - } - } - my ($answer,$loncaparev); - my @ids=¤t_machine_ids(); - if (grep(/^\Q$lonhost\E$/,@ids)) { - $answer = $perlvar{'lonVersion'}; - if ($answer =~ /^[\'\"]?([\d.\-]+)[\'\"]?$/) { - $loncaparev = $1; - } - } else { - $answer = &reply('serverloncaparev',$lonhost); - if (($answer eq 'unknown_cmd') || ($answer eq 'con_lost')) { - if ($caller eq 'loncron') { - my $ua=new LWP::UserAgent; - $ua->timeout(20); - my $protocol = $protocol{$lonhost}; - $protocol = 'http' if ($protocol ne 'https'); - my $url = $protocol.'://'.&hostname($lonhost).'/adm/about.html'; - my $request=new HTTP::Request('GET',$url); - my $response=$ua->request($request); - unless ($response->is_error()) { - my $content = $response->content; - if ($content =~ /

VERSION\:\s*([\d.\-]+)<\/p>/) { - $loncaparev = $1; - } - } - } else { - $loncaparev = $loncaparevs{$lonhost}; - } - } elsif ($answer =~ /^[\'\"]?([\d.\-]+)[\'\"]?$/) { - $loncaparev = $1; - } - } - return &do_cache_new('serverloncaparev',$lonhost,$loncaparev,$cachetime); - } -} - -sub get_server_homeID { - my ($hostname,$ignore_cache,$caller) = @_; - unless ($ignore_cache) { - my ($serverhomeID,$cached)=&is_cached_new('serverhomeID',$hostname); + my $cachetime = 24*3600; + my ($loncaparev,$cached)=&is_cached_new('serverloncaparev',$lonhost); if (defined($cached)) { - return $serverhomeID; - } - } - my $cachetime = 12*3600; - my $serverhomeID; - if ($caller eq 'loncron') { - my @machine_ids = &machine_ids($hostname); - foreach my $id (@machine_ids) { - my $response = &reply('serverhomeID',$id); - unless (($response eq 'unknown_cmd') || ($response eq 'con_lost')) { - $serverhomeID = $response; - last; - } - } - if ($serverhomeID eq '') { - $serverhomeID = $machine_ids[-1]; + return $loncaparev; + } else { + my $loncaparev = &reply('serverloncaparev',$lonhost); + return &do_cache_new('serverloncaparev',$lonhost,$loncaparev,$cachetime); } - } else { - $serverhomeID = $serverhomeIDs{$hostname}; } - return &do_cache_new('serverhomeID',$hostname,$serverhomeID,$cachetime); } # -------------------------------------------------- Non-critical communication @@ -724,30 +665,6 @@ sub userload { return $userloadpercent; } -# ------------------------------------------ Fight off request when overloaded - -sub overloaderror { - my ($r,$checkserver)=@_; - unless ($checkserver) { $checkserver=$perlvar{'lonHostID'}; } - my $loadavg; - if ($checkserver eq $perlvar{'lonHostID'}) { - open(my $loadfile,'/proc/loadavg'); - $loadavg=<$loadfile>; - $loadavg =~ s/\s.*//g; - $loadavg = 100*$loadavg/$perlvar{'lonLoadLim'}; - close($loadfile); - } else { - $loadavg=&reply('load',$checkserver); - } - my $overload=$loadavg-100; - if ($overload>0) { - $r->err_headers_out->{'Retry-After'}=$overload; - $r->log_error('Overload of '.$overload.' on '.$checkserver); - return 413; - } - return ''; -} - # ------------------------------ Find server with least workload from spare.tab sub spareserver { @@ -793,7 +710,7 @@ sub compare_server_load { my $userloadans = &reply('userload',$try_server); if ($loadans !~ /\d/ && $userloadans !~ /\d/) { - return; #didn't get a number from the server + return; #didn't get a number from the server } my $load; @@ -836,27 +753,6 @@ sub has_user_session { return 0; } -# --------- determine least loaded server in a user's domain which allows login - -sub choose_server { - my ($udom) = @_; - my %domconfhash = &Apache::loncommon::get_domainconf($udom); - my %servers = &get_servers($udom); - my $lowest_load = 30000; - my ($login_host,$hostname); - foreach my $lonhost (keys(%servers)) { - my $loginvia = $domconfhash{$udom.'.login.loginvia_'.$lonhost}; - if ($loginvia eq '') { - ($login_host, $lowest_load) = - &compare_server_load($lonhost, $login_host, $lowest_load); - } - } - if ($login_host ne '') { - $hostname = $servers{$login_host}; - } - return ($login_host,$hostname); -} - # --------------------------------------------- Try to change a user's password sub changepass { @@ -915,7 +811,7 @@ sub queryauthenticate { # --------- Try to authenticate user from domain's lib servers (first this one) sub authenticate { - my ($uname,$upass,$udom,$checkdefauth,$clientcancheckhost)=@_; + my ($uname,$upass,$udom,$checkdefauth)=@_; $upass=&escape($upass); $uname= &LONCAPA::clean_username($uname); my $uhome=&homeserver($uname,$udom,1); @@ -938,7 +834,7 @@ sub authenticate { return 'no_host'; } } - my $answer=reply("encrypt:auth:$udom:$uname:$upass:$checkdefauth:$clientcancheckhost",$uhome); + my $answer=reply("encrypt:auth:$udom:$uname:$upass:$checkdefauth",$uhome); if ($answer eq 'authorized') { if ($newhome) { &logthis("User $uname at $udom authorized by $uhome, but needs account"); @@ -956,64 +852,6 @@ sub authenticate { return 'no_host'; } -sub can_host_session { - my ($udom,$lonhost,$remoterev,$remotesessions,$hostedsessions) = @_; - my $canhost = 1; - my $host_idn = &Apache::lonnet::internet_dom($lonhost); - if (ref($remotesessions) eq 'HASH') { - if (ref($remotesessions->{'excludedomain'}) eq 'ARRAY') { - if (grep(/^\Q$host_idn\E$/,@{$remotesessions->{'excludedomain'}})) { - $canhost = 0; - } else { - $canhost = 1; - } - } - if (ref($remotesessions->{'includedomain'}) eq 'ARRAY') { - if (grep(/^\Q$host_idn\E$/,@{$remotesessions->{'includedomain'}})) { - $canhost = 1; - } else { - $canhost = 0; - } - } - if ($canhost) { - if ($remotesessions->{'version'} ne '') { - my ($reqmajor,$reqminor) = ($remotesessions->{'version'} =~ /^(\d+)\.(\d+)$/); - if ($reqmajor ne '' && $reqminor ne '') { - if ($remoterev =~ /^\'?(\d+)\.(\d+)/) { - my $major = $1; - my $minor = $2; - if (($major < $reqmajor ) || - (($major == $reqmajor) && ($minor < $reqminor))) { - $canhost = 0; - } - } else { - $canhost = 0; - } - } - } - } - } - if ($canhost) { - if (ref($hostedsessions) eq 'HASH') { - if (ref($hostedsessions->{'excludedomain'}) eq 'ARRAY') { - if (grep(/^\Q$udom\E$/,@{$hostedsessions->{'excludedomain'}})) { - $canhost = 0; - } else { - $canhost = 1; - } - } - if (ref($hostedsessions->{'includedomain'}) eq 'ARRAY') { - if (grep(/^\Q$udom\E$/,@{$hostedsessions->{'includedomain'}})) { - $canhost = 1; - } else { - $canhost = 0; - } - } - } - } - return $canhost; -} - # ---------------------- Find the homebase for a user from domain's lib servers my %homecache; @@ -1490,7 +1328,7 @@ sub get_domain_defaults { my %domconfig = &Apache::lonnet::get_dom('configuration',['defaults','quotas', 'requestcourses','inststatus', - 'coursedefaults','usersessions'],$domain); + 'coursedefaults'],$domain); if (ref($domconfig{'defaults'}) eq 'HASH') { $domdefaults{'lang_def'} = $domconfig{'defaults'}{'lang_def'}; $domdefaults{'auth_def'} = $domconfig{'defaults'}{'auth_def'}; @@ -1530,14 +1368,6 @@ sub get_domain_defaults { $domdefaults{$item} = $domconfig{'coursedefaults'}{$item}; } } - if (ref($domconfig{'usersessions'}) eq 'HASH') { - if (ref($domconfig{'usersessions'}{'remote'}) eq 'HASH') { - $domdefaults{'remotesessions'} = $domconfig{'usersessions'}{'remote'}; - } - if (ref($domconfig{'usersessions'}{'hosted'}) eq 'HASH') { - $domdefaults{'hostedsessions'} = $domconfig{'usersessions'}{'hosted'}; - } - } &Apache::lonnet::do_cache_new('domdefaults',$domain,\%domdefaults, $cachetime); return %domdefaults; @@ -3354,7 +3184,7 @@ sub get_domain_roles { return %personnel; } -# ----------------------------------------------------------- Check out an item +# ----------------------------------------------------------- Interval timing sub get_first_access { my ($type,$argsymb)=@_; @@ -3390,91 +3220,6 @@ sub set_first_access { return 'already_set'; } -sub checkout { - my ($symb,$tuname,$tudom,$tcrsid)=@_; - my $now=time; - my $lonhost=$perlvar{'lonHostID'}; - my $infostr=&escape( - 'CHECKOUTTOKEN&'. - $tuname.'&'. - $tudom.'&'. - $tcrsid.'&'. - $symb.'&'. - $now.'&'.$ENV{'REMOTE_ADDR'}); - my $token=&reply('tmpput:'.$infostr,$lonhost); - if ($token=~/^error\:/) { - &logthis("WARNING: ". - "Checkout tmpput failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - return ''; - } - - $token=~s/^(\d+)\_.*\_(\d+)$/$1\*$2\*$lonhost/; - $token=~tr/a-z/A-Z/; - - my %infohash=('resource.0.outtoken' => $token, - 'resource.0.checkouttime' => $now, - 'resource.0.outremote' => $ENV{'REMOTE_ADDR'}); - - unless (&cstore(\%infohash,$symb,$tcrsid,$tudom,$tuname) eq 'ok') { - return ''; - } else { - &logthis("WARNING: ". - "Checkout cstore failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - } - - if (&log($tudom,$tuname,&homeserver($tuname,$tudom), - &escape('Checkout '.$infostr.' - '. - $token)) ne 'ok') { - return ''; - } else { - &logthis("WARNING: ". - "Checkout log failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - } - return $token; -} - -# ------------------------------------------------------------ Check in an item - -sub checkin { - my $token=shift; - my $now=time; - my ($ta,$tb,$lonhost)=split(/\*/,$token); - $lonhost=~tr/A-Z/a-z/; - my $dtoken=$ta.'_'.&hostname($lonhost).'_'.$tb; - $dtoken=~s/\W/\_/g; - my ($dummy,$tuname,$tudom,$tcrsid,$symb,$chtim,$rmaddr)= - split(/\&/,&unescape(&reply('tmpget:'.$dtoken,$lonhost))); - - unless (($tuname) && ($tudom)) { - &logthis('Check in '.$token.' ('.$dtoken.') failed'); - return ''; - } - - unless (&allowed('mgr',$tcrsid)) { - &logthis('Check in '.$token.' ('.$dtoken.') unauthorized: '. - $env{'user.name'}.' - '.$env{'user.domain'}); - return ''; - } - - my %infohash=('resource.0.intoken' => $token, - 'resource.0.checkintime' => $now, - 'resource.0.inremote' => $ENV{'REMOTE_ADDR'}); - - unless (&cstore(\%infohash,$symb,$tcrsid,$tudom,$tuname) eq 'ok') { - return ''; - } - - if (&log($tudom,$tuname,&homeserver($tuname,$tudom), - &escape('Checkin - '.$token)) ne 'ok') { - return ''; - } - - return ($symb,$tuname,$tudom,$tcrsid); -} - # --------------------------------------------- Set Expire Date for Spreadsheet sub expirespread { @@ -4052,10 +3797,9 @@ sub rolesinit { my ($domain,$username,$authhost)=@_; my $now=time; my %userroles = ('user.login.time' => $now); - my $extra = &freeze_escape({'clientcheckrole' => 1}); - my $rolesdump=reply("dump:$domain:$username:roles:.::$extra",$authhost); + my $rolesdump=reply("dump:$domain:$username:roles",$authhost); if (($rolesdump eq 'con_lost') || ($rolesdump eq '') || - ($rolesdump =~ /^error:/)) { + ($rolesdump =~ /^error:/)) { return \%userroles; } my %allroles=(); @@ -4181,7 +3925,7 @@ sub set_userprivs { my $adv=0; my %grouproles = (); if (keys(%{$allgroups}) > 0) { - my @groupkeys; + my @groupkeys; foreach my $role (keys(%{$allroles})) { push(@groupkeys,$role); } @@ -4257,7 +4001,7 @@ sub role_status { my %userroles = ( 'user.role.'.$$role.'.'.$$where => $$tstart.'.'.$$tend ); - @rolecodes = ('cm'); + @rolecodes = ('cm'); my $spec=$$role.'.'.$$where; my ($tdummy,$tdomain,$trest)=split(/\//,$$where); if ($$role =~ /^cr\//) { @@ -4268,14 +4012,13 @@ sub role_status { my %rolehash = &get('roles',[$$where.'_'.$$role],$env{'user.domain'}, $env{'user.name'}); my ($trole) = split('_',$rolehash{$$where.'_'.$$role},2); - (undef,my $group_privs) = split(/\//,$trole); $group_privs = &unescape($group_privs); &group_roleprivs(\%allgroups,$$where,$group_privs,$$tend,$$tstart); my %course_roles = &get_my_roles($env{'user.name'},$env{'user.domain'},'userroles',['active'],['cc','co','in','ta','ep','ad','st','cr'],[$tdomain],1); if (keys(%course_roles) > 0) { my ($tnum) = ($trest =~ /^($match_courseid)/); - if ($tdomain ne '' && $tnum ne '') { + if ($tdomain ne '' && $tnum ne '') { foreach my $key (keys(%course_roles)) { if ($key =~ /^\Q$tnum\E:\Q$tdomain\E:([^:]+):?([^:]*)/) { my $crsrole = $1; @@ -6729,7 +6472,7 @@ sub modifyuser { } &logthis('Call to modify user '.$udom.', '.$uname.', '.$uid.', '. $umode.', '.$first.', '.$middle.', '. - $last.', '.$gene.'(forceid: '.$forceid.'; candelete: '.$showcandelete.')'. + $last.', '.$gene.'(forceid: '.$forceid.'; candelete: '.$showcandelete.')'. (defined($desiredhome) ? ' desiredhome = '.$desiredhome : ' desiredhome not specified'). ' by '.$env{'user.name'}.' at '.$env{'user.domain'}. @@ -6793,6 +6536,7 @@ sub modifyuser { } else { %names = @tmp; } +# # If name, email and/or uid are blank (e.g., because an uploaded file # of users did not contain them), do not overwrite existing values # unless field is in $candelete array ref. @@ -8388,6 +8132,7 @@ sub add_prefix_and_part { # ---------------------------------------------------------------- Get metadata my %metaentry; +my %importedpartids; sub metadata { my ($uri,$what,$liburi,$prefix,$depthcount)=@_; $uri=&declutter($uri); @@ -8414,6 +8159,10 @@ sub metadata { if (defined($cached)) { return $result->{':'.$what}; } } { +# Imported parts would go here + my %importedids=(); + my @origfileimportpartids=(); + my $importedparts=0; # # Is this a recursive call for a library? # @@ -8497,27 +8246,55 @@ sub metadata { # This is not a package - some other kind of start tag # my $entry=$token->[1]; - my $unikey; - if ($entry eq 'import') { - $unikey=''; - } else { - $unikey=$entry; - } - $unikey.=&add_prefix_and_part($prefix,$token->[2]->{'part'}); - - if (defined($token->[2]->{'id'})) { - $unikey.='_'.$token->[2]->{'id'}; - } + my $unikey=''; if ($entry eq 'import') { # # Importing a library here # + my $location=$parser->get_text('/import'); + my $dir=$filename; + $dir=~s|[^/]*$||; + $location=&filelocation($dir,$location); + + my $importmode=$token->[2]->{'importmode'}; + if ($importmode eq 'problem') { +# Import as problem/response + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'}); + } elsif ($importmode eq 'part') { +# Import as part(s) + $importedparts=1; +# We need to get the original file and the imported file to get the part order correct +# Good news: we do not need to worry about nested libraries, since parts cannot be nested +# Load and inspect original file + if ($#origfileimportpartids<0) { + undef(%importedpartids); + my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri); + my $origfile=&getfile($origfilelocation); + @origfileimportpartids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs); + } + +# Load and inspect imported file + my $impfile=&getfile($location); + my @impfilepartids=($impfile=~/]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs); + if ($#impfilepartids>=0) { +# This problem had parts + $importedpartids{$token->[2]->{'id'}}=join(',',@impfilepartids); + } else { +# Importing by turning a single problem into a problem part +# It gets the import-tags ID as part-ID + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'id'}); + $importedpartids{$token->[2]->{'id'}}=$token->[2]->{'id'}; + } + } else { +# Normal import + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'}); + if (defined($token->[2]->{'id'})) { + $unikey.='_'.$token->[2]->{'id'}; + } + } + if ($depthcount<20) { - my $location=$parser->get_text('/import'); - my $dir=$filename; - $dir=~s|[^/]*$||; - $location=&filelocation($dir,$location); my $metadata = &metadata($uri,'keys', $location,$unikey, $depthcount+1); @@ -8525,9 +8302,16 @@ sub metadata { $metaentry{':'.$meta}=$metaentry{':'.$meta}; $metathesekeys{$meta}=1; } - } - } else { + } + } else { +# +# Not importing, some other kind of non-package, non-library start tag +# + $unikey=$entry.&add_prefix_and_part($prefix,$token->[2]->{'part'}); + if (defined($token->[2]->{'id'})) { + $unikey.='_'.$token->[2]->{'id'}; + } if (defined($token->[2]->{'name'})) { $unikey.='_'.$token->[2]->{'name'}; } @@ -8601,6 +8385,22 @@ sub metadata { grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'})); $metaentry{':packages'} = join(',',@uniq_packages); + if ($importedparts) { +# We had imported parts and need to rebuild partorder + $metaentry{':partorder'}=''; + $metathesekeys{'partorder'}=1; + for (my $index=0;$index<$#origfileimportpartids;$index+=2) { + if ($origfileimportpartids[$index] eq 'part') { +# original part, part of the problem + $metaentry{':partorder'}.=','.$origfileimportpartids[$index+1]; + } else { +# we have imported parts at this position + $metaentry{':partorder'}.=','.$importedpartids{$origfileimportpartids[$index+1]}; + } + } + $metaentry{':partorder'}=~s/^\,//; + } + $metaentry{':keys'} = join(',',keys(%metathesekeys)); &metadata_generate_part0(\%metathesekeys,\%metaentry,$uri); $metaentry{':allpossiblekeys'}=join(',',keys %metathesekeys); @@ -9962,7 +9762,7 @@ sub get_dns { } sub unique_library { - #2x reverse removes all hostnames that appear more than once + #2x reverse removes all hostnames that appear more than once my %unique = reverse &all_library(); return reverse %unique; } @@ -9992,7 +9792,7 @@ sub get_dns { sub get_unique_servers { my %unique = reverse &get_servers(@_); - return reverse %unique; + return reverse %unique; } sub host_domain { @@ -10009,14 +9809,6 @@ sub get_dns { my @uniq = grep(!$seen{$_}++, values(%hostdom)); return @uniq; } - - sub internet_dom { - &load_hosts_tab() if (!$loaded); - - my ($lonid) = @_; - return $internetdom{$lonid}; - } - } { @@ -10134,40 +9926,6 @@ sub get_dns { return undef; } - sub get_internet_names { - my ($lonid) = @_; - return if ($lonid eq ''); - my ($idnref,$cached)= - &Apache::lonnet::is_cached_new('internetnames',$lonid); - if ($cached) { - return $idnref; - } - my $ip = &get_host_ip($lonid); - my @hosts = &get_hosts_from_ip($ip); - my %iphost = &get_iphost(); - my (@idns,%seen); - foreach my $id (@hosts) { - my $dom = &host_domain($id); - my $prim_id = &domain($dom,'primary'); - my $prim_ip = &get_host_ip($prim_id); - next if ($seen{$prim_ip}); - if (ref($iphost{$prim_ip}) eq 'ARRAY') { - foreach my $id (@{$iphost{$prim_ip}}) { - my $intdom = &internet_dom($id); - unless (grep(/^\Q$intdom\E$/,@idns)) { - push(@idns,$intdom); - } - } - } - $seen{$prim_ip} = 1; - } - return &Apache::lonnet::do_cache_new('internetnames',$lonid,\@idns,12*60*60); - } - -} - -sub all_loncaparevs { - return qw(1.1 1.2 1.3 2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 2.8 2.9 2.10); } BEGIN { @@ -10245,34 +10003,6 @@ BEGIN { close($config); } -# ---------------------------------------------------------- Read loncaparev table -{ - if (-e "$perlvar{'lonTabDir'}/loncaparevs.tab") { - if (open(my $config,"<$perlvar{'lonTabDir'}/loncaparevs.tab")) { - while (my $configline=<$config>) { - chomp($configline); - my ($hostid,$loncaparev)=split(/:/,$configline); - $loncaparevs{$hostid}=$loncaparev; - } - close($config); - } - } -} - -# ---------------------------------------------------------- Read serverhostID table -{ - if (-e "$perlvar{'lonTabDir'}/serverhomeIDs.tab") { - if (open(my $config,"<$perlvar{'lonTabDir'}/serverhomeIDs.tab")) { - while (my $configline=<$config>) { - chomp($configline); - my ($name,$id)=split(/:/,$configline); - $serverhomeIDs{$name}=$id; - } - close($config); - } - } -} - # ------------- set up temporary directory { $tmpdir = $perlvar{'lonDaemons'}.'/tmp/'; @@ -10503,14 +10233,9 @@ authentication scheme =item * X -B: try to +B: try to authenticate user from domain's lib servers (first use the current one). C<$upass> should be the users password. -$checkdefauth is optional (value is 1 if a check should be made to - authenticate user using default authentication method, and allow - account creation if username does not have account in the domain). -$clientcancheckhost is optional (value is 1 if checking whether the - server can host will occur on the client side in lonauth.pm). =item * X