--- loncom/lonnet/perl/lonnet.pm 2010/08/17 01:33:18 1.1056.4.4 +++ loncom/lonnet/perl/lonnet.pm 2010/09/24 13:15:47 1.1085 @@ -1,7 +1,7 @@ # The LearningOnline Network # TCP networking package # -# $Id: lonnet.pm,v 1.1056.4.4 2010/08/17 01:33:18 raeburn Exp $ +# $Id: lonnet.pm,v 1.1085 2010/09/24 13:15:47 raeburn Exp $ # # Copyright Michigan State University Board of Trustees # @@ -76,7 +76,7 @@ use HTTP::Date; use Image::Magick; use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir - $_64bit %env %protocol %loncaparevs %serverhomeIDs); + $_64bit %env %protocol %loncaparevs %serverhomeIDs %needsrelease); my (%badServerCache, $memcache, %courselogs, %accesshash, %domainrolehash, %userrolehash, $processmarker, $dumpcount, %coursedombuf, @@ -222,7 +222,7 @@ sub get_server_loncaparev { my @ids=¤t_machine_ids(); if (grep(/^\Q$lonhost\E$/,@ids)) { $answer = $perlvar{'lonVersion'}; - if ($answer =~ /^[\'\"]?([\d.\-]+)[\'\"]?$/) { + if ($answer =~ /^[\'\"]?([\w.\-]+)[\'\"]?$/) { $loncaparev = $1; } } else { @@ -230,7 +230,7 @@ sub get_server_loncaparev { if (($answer eq 'unknown_cmd') || ($answer eq 'con_lost')) { if ($caller eq 'loncron') { my $ua=new LWP::UserAgent; - $ua->timeout(20); + $ua->timeout(4); my $protocol = $protocol{$lonhost}; $protocol = 'http' if ($protocol ne 'https'); my $url = $protocol.'://'.&hostname($lonhost).'/adm/about.html'; @@ -238,14 +238,14 @@ sub get_server_loncaparev { my $response=$ua->request($request); unless ($response->is_error()) { my $content = $response->content; - if ($content =~ /

VERSION\:\s*([\d.\-]+)<\/p>/) { + if ($content =~ /

VERSION\:\s*([\w.\-]+)<\/p>/) { $loncaparev = $1; } } } else { $loncaparev = $loncaparevs{$lonhost}; } - } elsif ($answer =~ /^[\'\"]?([\d.\-]+)[\'\"]?$/) { + } elsif ($answer =~ /^[\'\"]?([\w.\-]+)[\'\"]?$/) { $loncaparev = $1; } } @@ -263,7 +263,7 @@ sub get_server_homeID { } my $cachetime = 12*3600; my $serverhomeID; - if ($caller eq 'loncron') { + if ($caller eq 'loncron') { my @machine_ids = &machine_ids($hostname); foreach my $id (@machine_ids) { my $response = &reply('serverhomeID',$id); @@ -724,40 +724,26 @@ sub userload { return $userloadpercent; } -# ------------------------------------------ Fight off request when overloaded - -sub overloaderror { - my ($r,$checkserver)=@_; - unless ($checkserver) { $checkserver=$perlvar{'lonHostID'}; } - my $loadavg; - if ($checkserver eq $perlvar{'lonHostID'}) { - open(my $loadfile,'/proc/loadavg'); - $loadavg=<$loadfile>; - $loadavg =~ s/\s.*//g; - $loadavg = 100*$loadavg/$perlvar{'lonLoadLim'}; - close($loadfile); - } else { - $loadavg=&reply('load',$checkserver); - } - my $overload=$loadavg-100; - if ($overload>0) { - $r->err_headers_out->{'Retry-After'}=$overload; - $r->log_error('Overload of '.$overload.' on '.$checkserver); - return 413; - } - return ''; -} - # ------------------------------ Find server with least workload from spare.tab sub spareserver { - my ($loadpercent,$userloadpercent,$want_server_name) = @_; + my ($loadpercent,$userloadpercent,$want_server_name,$udom) = @_; my $spare_server; if ($userloadpercent !~ /\d/) { $userloadpercent=0; } my $lowest_load=($loadpercent > $userloadpercent) ? $loadpercent : $userloadpercent; - + my ($uint_dom,$remotesessions); + if (($udom ne '') && (&domain($udom) ne '')) { + my $uprimary_id = &Apache::lonnet::domain($udom,'primary'); + $uint_dom = &Apache::lonnet::internet_dom($uprimary_id); + my %udomdefaults = &Apache::lonnet::get_domain_defaults($udom); + $remotesessions = $udomdefaults{'remotesessions'}; + } foreach my $try_server (@{ $spareid{'primary'} }) { + if ($uint_dom) { + next unless (&spare_can_host($udom,$uint_dom,$remotesessions, + $try_server)); + } ($spare_server, $lowest_load) = &compare_server_load($try_server, $spare_server, $lowest_load); } @@ -766,6 +752,10 @@ sub spareserver { if (!$found_server) { foreach my $try_server (@{ $spareid{'default'} }) { + if ($uint_dom) { + next unless (&spare_can_host($udom,$uint_dom,$remotesessions, + $try_server)); + } ($spare_server, $lowest_load) = &compare_server_load($try_server, $spare_server, $lowest_load); } @@ -778,7 +768,7 @@ sub spareserver { } if (defined($spare_server)) { my $hostname = &hostname($spare_server); - if (defined($hostname)) { + if (defined($hostname)) { $spare_server = $protocol.'://'.$hostname; } } @@ -793,7 +783,7 @@ sub compare_server_load { my $userloadans = &reply('userload',$try_server); if ($loadans !~ /\d/ && $userloadans !~ /\d/) { - return; #didn't get a number from the server + return; #didn't get a number from the server } my $load; @@ -1014,6 +1004,26 @@ sub can_host_session { return $canhost; } +sub spare_can_host { + my ($udom,$uint_dom,$remotesessions,$try_server)=@_; + my $canhost=1; + my @intdoms; + my $internet_names = &Apache::lonnet::get_internet_names($try_server); + if (ref($internet_names) eq 'ARRAY') { + @intdoms = @{$internet_names}; + } + unless (grep(/^\Q$uint_dom\E$/,@intdoms)) { + my $serverhomeID = &Apache::lonnet::get_server_homeID($try_server); + my $serverhomedom = &Apache::lonnet::host_domain($serverhomeID); + my %defdomdefaults = &Apache::lonnet::get_domain_defaults($serverhomedom); + my $remoterev = &Apache::lonnet::get_server_loncaparev(undef,$try_server); + $canhost = &can_host_session($udom,$try_server,$remoterev, + $remotesessions, + $defdomdefaults{'hostedsessions'}); + } + return $canhost; +} + # ---------------------- Find the homebase for a user from domain's lib servers my %homecache; @@ -3354,7 +3364,7 @@ sub get_domain_roles { return %personnel; } -# ----------------------------------------------------------- Check out an item +# ----------------------------------------------------------- Interval timing sub get_first_access { my ($type,$argsymb)=@_; @@ -3390,91 +3400,6 @@ sub set_first_access { return 'already_set'; } -sub checkout { - my ($symb,$tuname,$tudom,$tcrsid)=@_; - my $now=time; - my $lonhost=$perlvar{'lonHostID'}; - my $infostr=&escape( - 'CHECKOUTTOKEN&'. - $tuname.'&'. - $tudom.'&'. - $tcrsid.'&'. - $symb.'&'. - $now.'&'.$ENV{'REMOTE_ADDR'}); - my $token=&reply('tmpput:'.$infostr,$lonhost); - if ($token=~/^error\:/) { - &logthis("WARNING: ". - "Checkout tmpput failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - return ''; - } - - $token=~s/^(\d+)\_.*\_(\d+)$/$1\*$2\*$lonhost/; - $token=~tr/a-z/A-Z/; - - my %infohash=('resource.0.outtoken' => $token, - 'resource.0.checkouttime' => $now, - 'resource.0.outremote' => $ENV{'REMOTE_ADDR'}); - - unless (&cstore(\%infohash,$symb,$tcrsid,$tudom,$tuname) eq 'ok') { - return ''; - } else { - &logthis("WARNING: ". - "Checkout cstore failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - } - - if (&log($tudom,$tuname,&homeserver($tuname,$tudom), - &escape('Checkout '.$infostr.' - '. - $token)) ne 'ok') { - return ''; - } else { - &logthis("WARNING: ". - "Checkout log failed ".$tudom.' - '.$tuname.' - '.$symb. - ""); - } - return $token; -} - -# ------------------------------------------------------------ Check in an item - -sub checkin { - my $token=shift; - my $now=time; - my ($ta,$tb,$lonhost)=split(/\*/,$token); - $lonhost=~tr/A-Z/a-z/; - my $dtoken=$ta.'_'.&hostname($lonhost).'_'.$tb; - $dtoken=~s/\W/\_/g; - my ($dummy,$tuname,$tudom,$tcrsid,$symb,$chtim,$rmaddr)= - split(/\&/,&unescape(&reply('tmpget:'.$dtoken,$lonhost))); - - unless (($tuname) && ($tudom)) { - &logthis('Check in '.$token.' ('.$dtoken.') failed'); - return ''; - } - - unless (&allowed('mgr',$tcrsid)) { - &logthis('Check in '.$token.' ('.$dtoken.') unauthorized: '. - $env{'user.name'}.' - '.$env{'user.domain'}); - return ''; - } - - my %infohash=('resource.0.intoken' => $token, - 'resource.0.checkintime' => $now, - 'resource.0.inremote' => $ENV{'REMOTE_ADDR'}); - - unless (&cstore(\%infohash,$symb,$tcrsid,$tudom,$tuname) eq 'ok') { - return ''; - } - - if (&log($tudom,$tuname,&homeserver($tuname,$tudom), - &escape('Checkin - '.$token)) ne 'ok') { - return ''; - } - - return ($symb,$tuname,$tudom,$tcrsid); -} - # --------------------------------------------- Set Expire Date for Spreadsheet sub expirespread { @@ -4013,6 +3938,44 @@ sub coursedescription { return %returnhash; } +sub update_released_required { + my ($needsrelease,$cdom,$cnum,$chome,$cid) = @_; + if ($cdom eq '' || $cnum eq '' || $chome eq '' || $cid eq '') { + $cid = $env{'request.course.id'}; + $cdom = $env{'course.'.$cid.'.domain'}; + $cnum = $env{'course.'.$cid.'.num'}; + $chome = $env{'course.'.$cid.'.home'}; + } + if ($needsrelease) { + my %curr_reqd_hash = &userenvironment($cdom,$cnum,'internal.releaserequired'); + my $needsupdate; + if ($curr_reqd_hash{'internal.releaserequired'} eq '') { + $needsupdate = 1; + } else { + my ($currmajor,$currminor) = split(/\./,$curr_reqd_hash{'internal.releaserequired'}); + my ($needsmajor,$needsminor) = split(/\./,$needsrelease); + if (($currmajor < $needsmajor) || ($currmajor == $needsmajor && $currminor < $needsminor)) { + $needsupdate = 1; + } + } + if ($needsupdate) { + my %needshash = ( + 'internal.releaserequired' => $needsrelease, + ); + my $putresult = &put('environment',\%needshash,$cdom,$cnum); + if ($putresult eq 'ok') { + &appenv({'course.'.$cid.'.internal.releaserequired' => $needsrelease}); + my %crsinfo = &courseiddump($cdom,'.',1,'.','.',$cnum,undef,undef,'.'); + if (ref($crsinfo{$cid}) eq 'HASH') { + $crsinfo{$cid}{'releaserequired'} = $needsrelease; + &courseidput($cdom,\%crsinfo,$chome,'notime'); + } + } + } + } + return; +} + # -------------------------------------------------See if a user is privileged sub privileged { @@ -4181,7 +4144,7 @@ sub set_userprivs { my $adv=0; my %grouproles = (); if (keys(%{$allgroups}) > 0) { - my @groupkeys; + my @groupkeys; foreach my $role (keys(%{$allroles})) { push(@groupkeys,$role); } @@ -4257,7 +4220,7 @@ sub role_status { my %userroles = ( 'user.role.'.$$role.'.'.$$where => $$tstart.'.'.$$tend ); - @rolecodes = ('cm'); + @rolecodes = ('cm'); my $spec=$$role.'.'.$$where; my ($tdummy,$tdomain,$trest)=split(/\//,$$where); if ($$role =~ /^cr\//) { @@ -4268,14 +4231,13 @@ sub role_status { my %rolehash = &get('roles',[$$where.'_'.$$role],$env{'user.domain'}, $env{'user.name'}); my ($trole) = split('_',$rolehash{$$where.'_'.$$role},2); - (undef,my $group_privs) = split(/\//,$trole); $group_privs = &unescape($group_privs); &group_roleprivs(\%allgroups,$$where,$group_privs,$$tend,$$tstart); my %course_roles = &get_my_roles($env{'user.name'},$env{'user.domain'},'userroles',['active'],['cc','co','in','ta','ep','ad','st','cr'],[$tdomain],1); if (keys(%course_roles) > 0) { my ($tnum) = ($trest =~ /^($match_courseid)/); - if ($tdomain ne '' && $tnum ne '') { + if ($tdomain ne '' && $tnum ne '') { foreach my $key (keys(%course_roles)) { if ($key =~ /^\Q$tnum\E:\Q$tdomain\E:([^:]+):?([^:]*)/) { my $crsrole = $1; @@ -4958,7 +4920,7 @@ sub is_portfolio_file { } sub usertools_access { - my ($uname,$udom,$tool,$action,$context) = @_; + my ($uname,$udom,$tool,$action,$context,$userenvref,$domdefref,$is_advref)=@_; my ($access,%tools); if ($context eq '') { $context = 'tools'; @@ -5000,9 +4962,14 @@ sub usertools_access { $toolstatus = $env{'environment.'.$context.'.'.$tool}; $inststatus = $env{'environment.inststatus'}; } else { - my %userenv = &userenvironment($udom,$uname,$context.'.'.$tool,'inststatus'); - $toolstatus = $userenv{$context.'.'.$tool}; - $inststatus = $userenv{'inststatus'}; + if (ref($userenvref) eq 'HASH') { + $toolstatus = $userenvref->{$context.'.'.$tool}; + $inststatus = $userenvref->{'inststatus'}; + } else { + my %userenv = &userenvironment($udom,$uname,$context.'.'.$tool,'inststatus'); + $toolstatus = $userenv{$context.'.'.$tool}; + $inststatus = $userenv{'inststatus'}; + } } if ($toolstatus ne '') { @@ -5014,8 +4981,17 @@ sub usertools_access { return $access; } - my $is_adv = &is_advanced_user($udom,$uname); - my %domdef = &get_domain_defaults($udom); + my ($is_adv,%domdef); + if (ref($is_advref) eq 'HASH') { + $is_adv = $is_advref->{'is_adv'}; + } else { + $is_adv = &is_advanced_user($udom,$uname); + } + if (ref($domdefref) eq 'HASH') { + %domdef = %{$domdefref}; + } else { + %domdef = &get_domain_defaults($udom); + } if (ref($domdef{$tool}) eq 'HASH') { if ($is_adv) { if ($domdef{$tool}{'_LC_adv'} ne '') { @@ -5089,6 +5065,11 @@ sub is_course_owner { sub is_advanced_user { my ($udom,$uname) = @_; + if ($udom ne '' && $uname ne '') { + if (($udom eq $env{'user.domain'}) && ($uname eq $env{'user.name'})) { + return $env{'user.adv'}; + } + } my %roleshash = &get_my_roles($uname,$udom,'userroles',undef,undef,undef,1); my %allroles; my $is_adv; @@ -6728,7 +6709,7 @@ sub modifyuser { } &logthis('Call to modify user '.$udom.', '.$uname.', '.$uid.', '. $umode.', '.$first.', '.$middle.', '. - $last.', '.$gene.'(forceid: '.$forceid.'; candelete: '.$showcandelete.')'. + $last.', '.$gene.'(forceid: '.$forceid.'; candelete: '.$showcandelete.')'. (defined($desiredhome) ? ' desiredhome = '.$desiredhome : ' desiredhome not specified'). ' by '.$env{'user.name'}.' at '.$env{'user.domain'}. @@ -6738,7 +6719,6 @@ sub modifyuser { if ($uhome eq 'no_host') { $newuser = 1; } - # ----------------------------------------------------------------- Create User if (($uhome eq 'no_host') && (($umode && $upass) || ($umode eq 'localauth'))) { @@ -6798,6 +6778,7 @@ sub modifyuser { %names = @tmp; %oldnames = %names; } +# # If name, email and/or uid are blank (e.g., because an uploaded file # of users did not contain them), do not overwrite existing values # unless field is in $candelete array ref. @@ -6849,10 +6830,6 @@ sub modifyuser { } } } - my $reply = &put('environment', \%names, $udom,$uname); - if ($reply ne 'ok') { return 'error: '.$reply; } - my $sqlresult = &update_allusers_table($uname,$udom,\%names); - &devalidate_cache_new('namescache',$uname.':'.$udom); my $logmsg = $udom.', '.$uname.', '.$uid.', '. $umode.', '.$first.', '.$middle.', '. $last.', '.$gene.', '.$email.', '.$inststatus; @@ -6878,7 +6855,7 @@ sub modifyuser { return 'ok'; } my $reply = &put('environment', \%names, $udom,$uname); - if ($reply ne 'ok') { + if ($reply ne 'ok') { return 'error: '.$reply; } my $sqlresult = &update_allusers_table($uname,$udom,\%names); @@ -8416,6 +8393,7 @@ sub add_prefix_and_part { # ---------------------------------------------------------------- Get metadata my %metaentry; +my %importedpartids; sub metadata { my ($uri,$what,$liburi,$prefix,$depthcount)=@_; $uri=&declutter($uri); @@ -8442,6 +8420,10 @@ sub metadata { if (defined($cached)) { return $result->{':'.$what}; } } { +# Imported parts would go here + my %importedids=(); + my @origfileimportpartids=(); + my $importedparts=0; # # Is this a recursive call for a library? # @@ -8525,27 +8507,55 @@ sub metadata { # This is not a package - some other kind of start tag # my $entry=$token->[1]; - my $unikey; - if ($entry eq 'import') { - $unikey=''; - } else { - $unikey=$entry; - } - $unikey.=&add_prefix_and_part($prefix,$token->[2]->{'part'}); - - if (defined($token->[2]->{'id'})) { - $unikey.='_'.$token->[2]->{'id'}; - } + my $unikey=''; if ($entry eq 'import') { # # Importing a library here # + my $location=$parser->get_text('/import'); + my $dir=$filename; + $dir=~s|[^/]*$||; + $location=&filelocation($dir,$location); + + my $importmode=$token->[2]->{'importmode'}; + if ($importmode eq 'problem') { +# Import as problem/response + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'}); + } elsif ($importmode eq 'part') { +# Import as part(s) + $importedparts=1; +# We need to get the original file and the imported file to get the part order correct +# Good news: we do not need to worry about nested libraries, since parts cannot be nested +# Load and inspect original file + if ($#origfileimportpartids<0) { + undef(%importedpartids); + my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri); + my $origfile=&getfile($origfilelocation); + @origfileimportpartids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs); + } + +# Load and inspect imported file + my $impfile=&getfile($location); + my @impfilepartids=($impfile=~/]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs); + if ($#impfilepartids>=0) { +# This problem had parts + $importedpartids{$token->[2]->{'id'}}=join(',',@impfilepartids); + } else { +# Importing by turning a single problem into a problem part +# It gets the import-tags ID as part-ID + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'id'}); + $importedpartids{$token->[2]->{'id'}}=$token->[2]->{'id'}; + } + } else { +# Normal import + $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'}); + if (defined($token->[2]->{'id'})) { + $unikey.='_'.$token->[2]->{'id'}; + } + } + if ($depthcount<20) { - my $location=$parser->get_text('/import'); - my $dir=$filename; - $dir=~s|[^/]*$||; - $location=&filelocation($dir,$location); my $metadata = &metadata($uri,'keys', $location,$unikey, $depthcount+1); @@ -8553,9 +8563,16 @@ sub metadata { $metaentry{':'.$meta}=$metaentry{':'.$meta}; $metathesekeys{$meta}=1; } - } - } else { + } + } else { +# +# Not importing, some other kind of non-package, non-library start tag +# + $unikey=$entry.&add_prefix_and_part($prefix,$token->[2]->{'part'}); + if (defined($token->[2]->{'id'})) { + $unikey.='_'.$token->[2]->{'id'}; + } if (defined($token->[2]->{'name'})) { $unikey.='_'.$token->[2]->{'name'}; } @@ -8629,6 +8646,22 @@ sub metadata { grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'})); $metaentry{':packages'} = join(',',@uniq_packages); + if ($importedparts) { +# We had imported parts and need to rebuild partorder + $metaentry{':partorder'}=''; + $metathesekeys{'partorder'}=1; + for (my $index=0;$index<$#origfileimportpartids;$index+=2) { + if ($origfileimportpartids[$index] eq 'part') { +# original part, part of the problem + $metaentry{':partorder'}.=','.$origfileimportpartids[$index+1]; + } else { +# we have imported parts at this position + $metaentry{':partorder'}.=','.$importedpartids{$origfileimportpartids[$index+1]}; + } + } + $metaentry{':partorder'}=~s/^\,//; + } + $metaentry{':keys'} = join(',',keys(%metathesekeys)); &metadata_generate_part0(\%metathesekeys,\%metaentry,$uri); $metaentry{':allpossiblekeys'}=join(',',keys %metathesekeys); @@ -9905,6 +9938,7 @@ sub get_dns { my %libserv; my $loaded; my %name_to_host; + my %internetdom; sub parse_hosts_tab { my ($file) = @_; @@ -9912,7 +9946,7 @@ sub get_dns { next if ($configline =~ /^(\#|\s*$ )/x); next if ($configline =~ /^\^/); chomp($configline); - my ($id,$domain,$role,$name,$protocol)=split(/:/,$configline); + my ($id,$domain,$role,$name,$protocol,$intdom)=split(/:/,$configline); $name=~s/\s//g; if ($id && $domain && $role && $name) { $hostname{$id}=$name; @@ -9928,6 +9962,9 @@ sub get_dns { } else { $protocol{$id} = 'http'; } + if (defined($intdom)) { + $internetdom{$id} = $intdom; + } } } } @@ -9990,7 +10027,7 @@ sub get_dns { } sub unique_library { - #2x reverse removes all hostnames that appear more than once + #2x reverse removes all hostnames that appear more than once my %unique = reverse &all_library(); return reverse %unique; } @@ -10020,7 +10057,7 @@ sub get_dns { sub get_unique_servers { my %unique = reverse &get_servers(@_); - return reverse %unique; + return reverse %unique; } sub host_domain { @@ -10044,7 +10081,6 @@ sub get_dns { my ($lonid) = @_; return $internetdom{$lonid}; } - } { @@ -10301,6 +10337,25 @@ BEGIN { } } +{ + my $file = $Apache::lonnet::perlvar{'lonTabDir'}.'/releaseslist.xml'; + if (-e $file) { + my $parser = HTML::LCParser->new($file); + while (my $token = $parser->get_token()) { + if ($token->[0] eq 'S') { + my $item = $token->[1]; + my $name = $token->[2]{'name'}; + my $value = $token->[2]{'value'}; + if ($item ne '' && $name ne '' && $value ne '') { + my $release = $parser->get_text(); + $release =~ s/(^\s*|\s*$ )//gx; + $needsrelease{$item.':'.$name.':'.$value} = $release; + } + } + } + } +} + # ------------- set up temporary directory { $tmpdir = $perlvar{'lonDaemons'}.'/tmp/'; @@ -10538,7 +10593,7 @@ $checkdefauth is optional (value is 1 if authenticate user using default authentication method, and allow account creation if username does not have account in the domain). $clientcancheckhost is optional (value is 1 if checking whether the - server can host will occur on the client side in lonauth.pm). + server can host will occur on the client side in lonauth.pm). =item * X