--- loncom/lonnet/perl/lonnet.pm	2005/12/19 21:03:53	1.686
+++ loncom/lonnet/perl/lonnet.pm	2006/05/30 12:47:18	1.740
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.686 2005/12/19 21:03:53 albertel Exp $
+# $Id: lonnet.pm,v 1.740 2006/05/30 12:47:18 www Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -45,7 +45,6 @@ qw(%perlvar %hostname %badServerCache %i
 
 use IO::Socket;
 use GDBM_File;
-use Apache::Constants qw(:common :http);
 use HTML::LCParser;
 use HTML::Parser;
 use Fcntl qw(:flock);
@@ -53,6 +52,9 @@ use Storable qw(lock_store lock_nstore l
 use Time::HiRes qw( gettimeofday tv_interval );
 use Cache::Memcached;
 use Digest::MD5;
+use lib '/home/httpd/lib/perl';
+use LONCAPA;
+use LONCAPA::Configuration;
 
 my $readit;
 my $max_connection_retries = 10;     # Or some such value.
@@ -86,6 +88,29 @@ delayed.
 
 
 # --------------------------------------------------------------------- Logging
+{
+    my $logid;
+    sub instructor_log {
+	my ($hash_name,$storehash,$delflag,$uname,$udom)=@_;
+	$logid++;
+	my $id=time().'00000'.$$.'00000'.$logid;
+	return &Apache::lonnet::put('nohist_'.$hash_name,
+				    { $id => {
+					'exe_uname' => $env{'user.name'},
+					'exe_udom'  => $env{'user.domain'},
+					'exe_time'  => time(),
+					'exe_ip'    => $ENV{'REMOTE_ADDR'},
+					'delflag'   => $delflag,
+					'logentry'  => $storehash,
+					'uname'     => $uname,
+					'udom'      => $udom,
+				    }
+				  },
+				    $env{'course.'.$env{'request.course.id'}.'.domain'},
+				    $env{'course.'.$env{'request.course.id'}.'.num'}
+				    );
+    }
+}
 
 sub logtouch {
     my $execdir=$perlvar{'lonDaemons'};
@@ -124,7 +149,7 @@ sub logperm {
 # -------------------------------------------------- Non-critical communication
 sub subreply {
     my ($cmd,$server)=@_;
-    my $peerfile="$perlvar{'lonSockDir'}/$server";
+    my $peerfile="$perlvar{'lonSockDir'}/".$hostname{$server};
     #
     #  With loncnew process trimming, there's a timing hole between lonc server
     #  process exit and the master server picking up the listen on the AF_UNIX
@@ -152,7 +177,7 @@ sub subreply {
     }
     my $answer;
     if ($client) {
-	print $client "$cmd\n";
+	print $client "sethost:$server:$cmd\n";
 	$answer=<$client>;
 	if (!$answer) { $answer="con_lost"; }
 	chomp($answer);
@@ -260,6 +285,13 @@ sub critical {
 
 sub transfer_profile_to_env {
     my ($lonidsdir,$handle)=@_;
+    if (!defined($lonidsdir)) {
+	$lonidsdir = $perlvar{'lonIDsDir'};
+    }
+    if (!defined($handle)) {
+        ($handle) = ($env{'user.environment'} =~m|/([^/]+)\.id$| );
+    }
+
     my @profile;
     {
 	open(my $idf,"$lonidsdir/$handle.id");
@@ -271,7 +303,9 @@ sub transfer_profile_to_env {
     my %Remove;
     for ($envi=0;$envi<=$#profile;$envi++) {
 	chomp($profile[$envi]);
-	my ($envname,$envvalue)=split(/=/,$profile[$envi]);
+	my ($envname,$envvalue)=split(/=/,$profile[$envi],2);
+	$envname=&unescape($envname);
+	$envvalue=&unescape($envvalue);
 	$env{$envname} = $envvalue;
         if (my ($key,$time) = ($envname =~ /^(cgi\.(\d+)_\d+\.)/)) {
             if ($time < time-300) {
@@ -289,14 +323,14 @@ sub transfer_profile_to_env {
 
 sub appenv {
     my %newenv=@_;
-    foreach (keys %newenv) {
-	if (($newenv{$_}=~/^user\.role/) || ($newenv{$_}=~/^user\.priv/)) {
+    foreach my $key (keys(%newenv)) {
+	if (($newenv{$key}=~/^user\.role/) || ($newenv{$key}=~/^user\.priv/)) {
             &logthis("<font color=\"blue\">WARNING: ".
-                "Attempt to modify environment ".$_." to ".$newenv{$_}
+                "Attempt to modify environment ".$key." to ".$newenv{$key}
                 .'</font>');
-	    delete($newenv{$_});
+	    delete($newenv{$key});
         } else {
-            $env{$_}=$newenv{$_};
+            $env{$key}=$newenv{$key};
         }
     }
 
@@ -323,7 +357,9 @@ sub appenv {
     for (my $i=0; $i<=$#oldenv; $i++) {
         chomp($oldenv[$i]);
         if ($oldenv[$i] ne '') {
-	    my ($name,$value)=split(/=/,$oldenv[$i]);
+	    my ($name,$value)=split(/=/,$oldenv[$i],2);
+	    $name=&unescape($name);
+	    $value=&unescape($value);
 	    unless (defined($newenv{$name})) {
 		$newenv{$name}=$value;
 	    }
@@ -336,7 +372,7 @@ sub appenv {
 	}
 	my $newname;
 	foreach $newname (keys %newenv) {
-	    print $fh "$newname=$newenv{$newname}\n";
+	    print $fh &escape($newname).'='.&escape($newenv{$newname})."\n";
 	}
 	close($fh);
     }
@@ -348,7 +384,6 @@ sub appenv {
 
 sub delenv {
     my $delthis=shift;
-    my %newenv=();
     if (($delthis=~/user\.role/) || ($delthis=~/user\.priv/)) {
         &logthis("<font color=\"blue\">WARNING: ".
                 "Attempt to delete from environment ".$delthis);
@@ -380,12 +415,14 @@ sub delenv {
 	    close($fh);
 	    return 'error: '.$!;
 	}
-	foreach (@oldenv) {
-	    if ($_=~/^$delthis/) { 
-                my ($key,undef) = split('=',$_);
+	foreach my $cur_key (@oldenv) {
+	    my $unescaped_cur_key = &unescape($cur_key);
+	    if ($unescaped_cur_key=~/^$delthis/) { 
+                my ($key) = split('=',$cur_key,2);
+		$key = &unescape($key);
                 delete($env{$key});
             } else {
-                print $fh $_; 
+                print $fh $cur_key; 
             }
 	}
 	close($fh);
@@ -840,11 +877,9 @@ sub getsection {
 }
 
 sub save_cache {
-    my ($r)=@_;
-    if (! $r->is_initial_req()) { return DECLINED; }
     &purge_remembered();
+    #&Apache::loncommon::validate_page();
     undef(%env);
-    return OK;
 }
 
 my $to_remember=-1;
@@ -947,25 +982,62 @@ sub userenvironment {
 sub studentphoto {
     my ($udom,$unam,$ext) = @_;
     my $home=&Apache::lonnet::homeserver($unam,$udom);
-    my $ret=&Apache::lonnet::reply("studentphoto:$udom:$unam:$ext",$home);
-    my $url="/uploaded/$udom/$unam/internal/studentphoto.".$ext;
-    if ($ret ne 'ok') {
-	return '/adm/lonKaputt/lonlogo_broken.gif';
+    if (defined($env{'request.course.id'})) {
+        if ($env{'course.'.$env{'request.course.id'}.'.internal.showphoto'}) {
+            if ($udom eq $env{'course.'.$env{'request.course.id'}.'.domain'}) {
+                return(&retrievestudentphoto($udom,$unam,$ext)); 
+            } else {
+                my ($result,$perm_reqd)=
+		    &Apache::lonnet::auto_photo_permission($unam,$udom);
+                if ($result eq 'ok') {
+                    if (!($perm_reqd eq 'yes')) {
+                        return(&retrievestudentphoto($udom,$unam,$ext));
+                    }
+                }
+            }
+        }
+    } else {
+        my ($result,$perm_reqd) = 
+	    &Apache::lonnet::auto_photo_permission($unam,$udom);
+        if ($result eq 'ok') {
+            if (!($perm_reqd eq 'yes')) {
+                return(&retrievestudentphoto($udom,$unam,$ext));
+            }
+        }
+    }
+    return '/adm/lonKaputt/lonlogo_broken.gif';
+}
+
+sub retrievestudentphoto {
+    my ($udom,$unam,$ext,$type) = @_;
+    my $home=&Apache::lonnet::homeserver($unam,$udom);
+    my $ret=&Apache::lonnet::reply("studentphoto:$udom:$unam:$ext:$type",$home);
+    if ($ret eq 'ok') {
+        my $url="/uploaded/$udom/$unam/internal/studentphoto.$ext";
+        if ($type eq 'thumbnail') {
+            $url="/uploaded/$udom/$unam/internal/studentphoto_tn.$ext"; 
+        }
+        my $tokenurl=&Apache::lonnet::tokenwrapper($url);
+        return $tokenurl;
+    } else {
+        if ($type eq 'thumbnail') {
+            return '/adm/lonKaputt/genericstudent_tn.gif';
+        } else { 
+            return '/adm/lonKaputt/lonlogo_broken.gif';
+        }
     }
-    my $tokenurl=&Apache::lonnet::tokenwrapper($url);
-    return $tokenurl;
 }
 
 # -------------------------------------------------------------------- New chat
 
 sub chatsend {
-    my ($newentry,$anon)=@_;
+    my ($newentry,$anon,$group)=@_;
     my $cnum=$env{'course.'.$env{'request.course.id'}.'.num'};
     my $cdom=$env{'course.'.$env{'request.course.id'}.'.domain'};
     my $chome=$env{'course.'.$env{'request.course.id'}.'.home'};
     &reply('chatsend:'.$cdom.':'.$cnum.':'.
 	   &escape($env{'user.domain'}.':'.$env{'user.name'}.':'.$anon.':'.
-		   &escape($newentry)),$chome);
+		   &escape($newentry)).':'.$group,$chome);
 }
 
 # ------------------------------------------ Find current version of a resource
@@ -1108,7 +1180,9 @@ sub ssi {
     my $ua=new LWP::UserAgent;
     
     my $request;
-    
+
+    $form{'no_update_last_known'}=1;
+
     if (%form) {
       $request=new HTTP::Request('POST',"http://".$ENV{'HTTP_HOST'}.$fn);
       $request->content(join('&',map { &escape($_).'='.&escape($form{$_}) } keys %form));
@@ -1281,7 +1355,7 @@ sub clean_filename {
 
 # --------------- Take an uploaded file and put it into the userfiles directory
 # input: $formname - the contents of the file are in $env{"form.$formname"}
-#                    the desired filenam is in $env{"form.$formname"}
+#                    the desired filenam is in $env{"form.$formname.filename"}
 #        $coursedoc - if true up to the current course
 #                     if false
 #        $subdir - directory in userfile to store the file into
@@ -1292,7 +1366,7 @@ sub clean_filename {
 
 
 sub userfileupload {
-    my ($formname,$coursedoc,$subdir,$parser,$allfiles,$codebase)=@_;
+    my ($formname,$coursedoc,$subdir,$parser,$allfiles,$codebase,$destuname,$destudom)=@_;
     if (!defined($subdir)) { $subdir='unknown'; }
     my $fname=$env{'form.'.$formname.'.filename'};
     $fname=&clean_filename($fname);
@@ -1315,6 +1389,7 @@ sub userfileupload {
         close($fh);
         return $fullpath.'/'.$fname; 
     }
+    
 # Create the directory if not present
     $fname="$subdir/$fname";
     if ($coursedoc) {
@@ -1330,9 +1405,19 @@ sub userfileupload {
 				       $fname,$formname,$parser,
 				       $allfiles,$codebase);
         }
+    } elsif (defined($destuname)) {
+        my $docuname=$destuname;
+        my $docudom=$destudom;
+	return &finishuserfileupload($docuname,$docudom,$formname,
+				     $fname,$parser,$allfiles,$codebase);
+        
     } else {
         my $docuname=$env{'user.name'};
         my $docudom=$env{'user.domain'};
+        if (exists($env{'form.group'})) {
+            $docuname=$env{'course.'.$env{'request.course.id'}.'.num'};
+            $docudom=$env{'course.'.$env{'request.course.id'}.'.domain'};
+        }
 	return &finishuserfileupload($docuname,$docudom,$formname,
 				     $fname,$parser,$allfiles,$codebase);
     }
@@ -1358,8 +1443,16 @@ sub finishuserfileupload {
     }
 # Save the file
     {
-	open(FH,'>'.$filepath.'/'.$file);
-	print FH $env{'form.'.$formname};
+	if (!open(FH,'>'.$filepath.'/'.$file)) {
+	    &logthis('Failed to create '.$filepath.'/'.$file);
+	    print STDERR ('Failed to create '.$filepath.'/'.$file."\n");
+	    return '/adm/notfound.html';
+	}
+	if (!print FH ($env{'form.'.$formname})) {
+	    &logthis('Failed to write to '.$filepath.'/'.$file);
+	    print STDERR ('Failed to write to '.$filepath.'/'.$file."\n");
+	    return '/adm/notfound.html';
+	}
 	close(FH);
     }
     if ($parser eq 'parse') {
@@ -1862,8 +1955,8 @@ sub courseiddump {
 sub dcmailput {
     my ($domain,$msgid,$message,$server)=@_;
     my $status = &Apache::lonnet::critical(
-       'dcmailput:'.$domain.':'.&Apache::lonnet::escape($msgid).'='.
-       &Apache::lonnet::escape($message),$server);
+       'dcmailput:'.$domain.':'.&escape($msgid).'='.
+       &escape($message),$server);
     return $status;
 }
 
@@ -2510,7 +2603,7 @@ sub restore {
 # ---------------------------------------------------------- Course Description
 
 sub coursedescription {
-    my $courseid=shift;
+    my ($courseid,$args)=@_;
     $courseid=~s/^\///;
     $courseid=~s/\_/\//g;
     my ($cdomain,$cnum)=split(/\//,$courseid);
@@ -2520,7 +2613,27 @@ sub coursedescription {
     # trying and trying and trying to get the course description.
     my %envhash=();
     my %returnhash=();
-    $envhash{'course.'.$normalid.'.last_cache'}=time;
+    
+    my $expiretime=600;
+    if ($env{'request.course.id'} eq $normalid) {
+	$expiretime=120;
+    }
+
+    my $prefix='course.'.$cdomain.'_'.$cnum.'.';
+    if (!$args->{'freshen_cache'}
+	&& ((time-$env{$prefix.'last_cache'}) < $expiretime) ) {
+	foreach my $key (keys(%env)) {
+	    next if ($key !~ /^\Q$prefix\E(.*)/);
+	    my ($setting) = $1;
+	    $returnhash{$setting} = $env{$key};
+	}
+	return %returnhash;
+    }
+
+    # get the data agin
+    if (!$args->{'one_time'}) {
+	$envhash{'course.'.$normalid.'.last_cache'}=time;
+    }
     if ($chome ne 'no_host') {
        %returnhash=&dump('environment',$cdomain,$cnum);
        if (!exists($returnhash{'con_lost'})) {
@@ -2538,7 +2651,9 @@ sub coursedescription {
            $envhash{'course.'.$normalid.'.num'}=$cnum;
        }
     }
-    &appenv(%envhash);
+    if (!$args->{'one_time'}) {
+	&appenv(%envhash);
+    }
     return %returnhash;
 }
 
@@ -2785,7 +2900,7 @@ sub del {
 # -------------------------------------------------------------- dump interface
 
 sub dump {
-   my ($namespace,$udomain,$uname,$regexp)=@_;
+   my ($namespace,$udomain,$uname,$regexp,$range)=@_;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
    if (!$uname) { $uname=$env{'user.name'}; }
    my $uhome=&homeserver($uname,$udomain);
@@ -2794,16 +2909,23 @@ sub dump {
    } else {
        $regexp='.';
    }
-   my $rep=reply("dump:$udomain:$uname:$namespace:$regexp",$uhome);
+   my $rep=reply("dump:$udomain:$uname:$namespace:$regexp:$range",$uhome);
    my @pairs=split(/\&/,$rep);
    my %returnhash=();
    foreach (@pairs) {
-      my ($key,$value)=split(/=/,$_);
+      my ($key,$value)=split(/=/,$_,2);
       $returnhash{unescape($key)}=&thaw_unescape($value);
    }
    return %returnhash;
 }
 
+# --------------------------------------------------------- dumpstore interface
+
+sub dumpstore {
+   my ($namespace,$udomain,$uname,$regexp,$range)=@_;
+   return &dump($namespace,$udomain,$uname,$regexp,$range);
+}
+
 # -------------------------------------------------------------- keys interface
 
 sub getkeys {
@@ -2945,25 +3067,53 @@ sub newput {
 # ---------------------------------------------------------  putstore interface
 
 sub putstore {
-   my ($namespace,$storehash,$udomain,$uname)=@_;
+   my ($namespace,$symb,$version,$storehash,$udomain,$uname)=@_;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
    if (!$uname) { $uname=$env{'user.name'}; }
    my $uhome=&homeserver($uname,$udomain);
    my $items='';
-   my %allitems = ();
-   foreach (keys %$storehash) {
-       if ($_ =~ m/^([^\:]+):([^\:]+):([^\:]+)$/) {
-           my $key = $1.':keys:'.$2;
-           $allitems{$key} .= $3.':';
-       }
-       $items.=$_.'='.&freeze_escape($$storehash{$_}).'&';
-   }
-   foreach (keys %allitems) {
-       $allitems{$_} =~ s/\:$//;
-       $items.= $_.'='.$allitems{$_}.'&';
+   foreach my $key (keys(%$storehash)) {
+       $items.= &escape($key).'='.&freeze_escape($storehash->{$key}).'&';
    }
    $items=~s/\&$//;
-   return &reply("put:$udomain:$uname:$namespace:$items",$uhome);
+   my $esc_symb=&escape($symb);
+   my $esc_v=&escape($version);
+   my $reply =
+       &reply("putstore:$udomain:$uname:$namespace:$esc_symb:$esc_v:$items",
+	      $uhome);
+   if ($reply eq 'unknown_cmd') {
+       # gfall back to way things use to be done
+       return &old_putstore($namespace,$symb,$version,$storehash,$udomain,
+			    $uname);
+   }
+   return $reply;
+}
+
+sub old_putstore {
+    my ($namespace,$symb,$version,$storehash,$udomain,$uname)=@_;
+    if (!$udomain) { $udomain=$env{'user.domain'}; }
+    if (!$uname) { $uname=$env{'user.name'}; }
+    my $uhome=&homeserver($uname,$udomain);
+    my %newstorehash;
+    foreach (keys %$storehash) {
+	my $key = $version.':'.&escape($symb).':'.$_;
+	$newstorehash{$key} = $storehash->{$_};
+    }
+    my $items='';
+    my %allitems = ();
+    foreach (keys %newstorehash) {
+	if ($_ =~ m/^([^\:]+):([^\:]+):([^\:]+)$/) {
+	    my $key = $1.':keys:'.$2;
+	    $allitems{$key} .= $3.':';
+	}
+	$items.=$_.'='.&freeze_escape($newstorehash{$_}).'&';
+    }
+    foreach (keys %allitems) {
+	$allitems{$_} =~ s/\:$//;
+	$items.= $_.'='.$allitems{$_}.'&';
+    }
+    $items=~s/\&$//;
+    return &reply("put:$udomain:$uname:$namespace:$items",$uhome);
 }
 
 # ------------------------------------------------------ critical put interface
@@ -2975,7 +3125,7 @@ sub cput {
    my $uhome=&homeserver($uname,$udomain);
    my $items='';
    foreach (keys %$storehash) {
-       $items.=escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+       $items.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
    }
    $items=~s/\&$//;
    return &critical("put:$udomain:$uname:$namespace:$items",$uhome);
@@ -3017,8 +3167,9 @@ sub tmpput {
 
 # ------------------------------------------------------------ tmpget interface
 sub tmpget {
-    my ($token)=@_;
-    my $rep=&reply("tmpget:$token",$perlvar{'lonHostID'});
+    my ($token,$server)=@_;
+    if (!defined($server)) { $server = $perlvar{'lonHostID'}; }
+    my $rep=&reply("tmpget:$token",$server);
     my %returnhash;
     foreach my $item (split(/\&/,$rep)) {
 	my ($key,$value)=split(/=/,$item);
@@ -3027,6 +3178,13 @@ sub tmpget {
     return %returnhash;
 }
 
+# ------------------------------------------------------------ tmpget interface
+sub tmpdel {
+    my ($token,$server)=@_;
+    if (!defined($server)) { $server = $perlvar{'lonHostID'}; }
+    return &reply("tmpdel:$token",$server);
+}
+
 # ---------------------------------------------- Custom access rule evaluation
 
 sub customaccess {
@@ -3065,6 +3223,7 @@ sub customaccess {
 
 sub allowed {
     my ($priv,$uri,$symb)=@_;
+    my $ver_orguri=$uri;
     $uri=&deversion($uri);
     my $orguri=$uri;
     $uri=&declutter($uri);
@@ -3077,12 +3236,29 @@ sub allowed {
     }
 
 # Free bre access to user's own portfolio contents
-    my ($space,$domain,$name,$dir)=split('/',$uri);
+    my ($space,$domain,$name,@dir)=split('/',$uri);
     if (($space=~/^(uploaded|editupload)$/) && ($env{'user.name'} eq $name) && 
-	($env{'user.domain'} eq $domain) && ('portfolio' eq $dir)) {
+	($env{'user.domain'} eq $domain) && ('portfolio' eq $dir[0])) {
         return 'F';
     }
 
+# bre access to group if user has rgf priv for this group and course.
+    if (($space=~/^(uploaded|editupload)$/) && ($dir[0] eq 'groups') 
+         && ($dir[2] eq 'portfolio') && ($priv eq 'bre')) {
+        if (exists($env{'request.course.id'})) {
+            my $cdom = $env{'course.'.$env{'request.course.id'}.'.domain'};
+            my $cnum = $env{'course.'.$env{'request.course.id'}.'.num'};
+            if (($domain eq $cdom) && ($name eq $cnum)) {
+                my $courseprivid=$env{'request.course.id'};
+                $courseprivid=~s/\_/\//;
+                if ($env{'user.priv.'.$env{'request.role'}.'./'.$courseprivid
+                    .'/'.$dir[1]} =~/rgf\&([^\:]*)/) {
+                    return $1; 
+                }
+            }
+        }
+    }
+
 # Free bre to public access
 
     if ($priv eq 'bre') {
@@ -3165,7 +3341,7 @@ sub allowed {
                 $thisallowed.=$1;
             }
         } else {
-            my $refuri=$env{'httpref.'.$orguri};
+            my $refuri = $env{'httpref.'.$orguri} || $env{'httpref.'.$ver_orguri};
             if ($refuri) {
                 if ($refuri =~ m|^/adm/|) {
                     $thisallowed='F';
@@ -3297,7 +3473,7 @@ sub allowed {
 	       my ($cdom,$cnum,$csec)=split(/\//,$courseid);
                my $prefix='course.'.$cdom.'_'.$cnum.'.';
                if ((time-$env{$prefix.'last_cache'})>$expiretime) {
-		   &coursedescription($courseid);
+		   &coursedescription($courseid,{'freshen_cache' => 1});
                }
                if (($env{$prefix.'res.'.$uri.'.lock.sections'}=~/\,\Q$csec\E\,/)
                 || ($env{$prefix.'res.'.$uri.'.lock.sections'} eq 'all')) {
@@ -3345,17 +3521,21 @@ sub allowed {
        my $unamedom=$env{'user.name'}.':'.$env{'user.domain'};
        if ($env{'course.'.$env{'request.course.id'}.'.'.$priv.'.roles.denied'}
 	   =~/\Q$rolecode\E/) {
-           &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.':'.
-                'Denied by role: '.$priv.' for '.$uri.' as '.$rolecode.' in '.
-                $env{'request.course.id'});
+	   if ($priv ne 'pch') { 
+	       &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.':'.
+			'Denied by role: '.$priv.' for '.$uri.' as '.$rolecode.' in '.
+			$env{'request.course.id'});
+	   }
            return '';
        }
 
        if ($env{'course.'.$env{'request.course.id'}.'.'.$priv.'.users.denied'}
 	   =~/\Q$unamedom\E/) {
-           &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.
-                'Denied by user: '.$priv.' for '.$uri.' as '.$unamedom.' in '.
-                $env{'request.course.id'});
+	   if ($priv ne 'pch') { 
+	       &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.
+			'Denied by user: '.$priv.' for '.$uri.' as '.$unamedom.' in '.
+			$env{'request.course.id'});
+	   }
            return '';
        }
    }
@@ -3365,9 +3545,11 @@ sub allowed {
    if ($thisallowed=~/R/) {
        my $rolecode=(split(/\./,$env{'request.role'}))[0];
        if (&metadata($uri,'roledeny')=~/\Q$rolecode\E/) {
-	   &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.':'.
-		'Denied by role: '.$priv.' for '.$uri.' as '.$rolecode);
-          return '';
+	   if ($priv ne 'pch') { 
+	       &logthis($env{'user.domain'}.':'.$env{'user.name'}.':'.$env{'user.home'}.':'.
+			'Denied by role: '.$priv.' for '.$uri.' as '.$rolecode);
+	   }
+	   return '';
        }
    }
 
@@ -3390,15 +3572,17 @@ sub allowed {
    return 'F';
 }
 
+sub split_uri_for_cond {
+    my $uri=&deversion(&declutter(shift));
+    my @uriparts=split(/\//,$uri);
+    my $filename=pop(@uriparts);
+    my $pathname=join('/',@uriparts);
+    return ($pathname,$filename);
+}
 # --------------------------------------------------- Is a resource on the map?
 
 sub is_on_map {
-    my $uri=&deversion(&declutter(shift));
-    my @uriparts=split(/\//,$uri);
-    my $filename=$uriparts[$#uriparts];
-    my $pathname=$uri;
-    $pathname=~s|/\Q$filename\E$||;
-    $pathname=~s/^adm\/wrapper\///;    
+    my ($pathname,$filename) = &split_uri_for_cond(shift);
     #Trying to find the conditional for the file
     my $match=($env{'acc.res.'.$env{'request.course.id'}.'.'.$pathname}=~
 	       /\&\Q$filename\E\:([\d\|]+)\&/);
@@ -3671,6 +3855,82 @@ sub auto_create_password {
     return ($authparam,$create_passwd,$authchk);
 }
 
+sub auto_photo_permission {
+    my ($cnum,$cdom,$students) = @_;
+    my $homeserver = &homeserver($cnum,$cdom);
+    my ($outcome,$perm_reqd,$conditions) = 
+	split(/:/,&unescape(&reply('autophotopermission:'.$cdom,$homeserver)),3);
+    if ($outcome =~ /^(con_lost|unknown_cmd|no_such_host)$/) {
+	return (undef,undef);
+    }
+    return ($outcome,$perm_reqd,$conditions);
+}
+
+sub auto_checkphotos {
+    my ($uname,$udom,$pid) = @_;
+    my $homeserver = &homeserver($uname,$udom);
+    my ($result,$resulttype);
+    my $outcome = &unescape(&reply('autophotocheck:'.&escape($udom).':'.
+				   &escape($uname).':'.&escape($pid),
+				   $homeserver));
+    if ($outcome =~ /^(con_lost|unknown_cmd|no_such_host)$/) {
+	return (undef,undef);
+    }
+    if ($outcome) {
+        ($result,$resulttype) = split(/:/,$outcome);
+    } 
+    return ($result,$resulttype);
+}
+
+sub auto_photochoice {
+    my ($cnum,$cdom) = @_;
+    my $homeserver = &homeserver($cnum,$cdom);
+    my ($update,$comment) = split(/:/,&unescape(&reply('autophotochoice:'.
+						       &escape($cdom),
+						       $homeserver)));
+    if ($update =~ /^(con_lost|unknown_cmd|no_such_host)$/) {
+	return (undef,undef);
+    }
+    return ($update,$comment);
+}
+
+sub auto_photoupdate {
+    my ($affiliatesref,$dom,$cnum,$photo) = @_;
+    my $homeserver = &homeserver($cnum,$dom);
+    my $host=$hostname{$homeserver};
+    my $cmd = '';
+    my $maxtries = 1;
+    foreach (keys %{$affiliatesref}) {
+        $cmd .= $_.'='.join(",",@{$$affiliatesref{$_}}).'%%';
+    }
+    $cmd =~ s/%%$//;
+    $cmd = &escape($cmd);
+    my $query = 'institutionalphotos';
+    my $queryid=&reply("querysend:".$query.':'.$dom.':'.$cnum.':'.$cmd,$homeserver);
+    unless ($queryid=~/^\Q$host\E\_/) {
+        &logthis('institutionalphotos: invalid queryid: '.$queryid.' for host: '.$host.' and homeserver: '.$homeserver.' and course: '.$cnum);
+        return 'error: '.$queryid;
+    }
+    my $reply = &get_query_reply($queryid);
+    my $tries = 1;
+    while (($reply=~/^timeout/) && ($tries < $maxtries)) {
+        $reply = &get_query_reply($queryid);
+        $tries ++;
+    }
+    if ( ($reply =~/^timeout/) || ($reply =~/^error/) ) {
+        &logthis('institutionalphotos error: '.$reply.' for '.$dom.' '.$env{'user.name'}.' for '.$queryid.' course: '.$cnum.' maxtries: '.$maxtries.' tries: '.$tries);
+    } else {
+        my @responses = split(/:/,$reply);
+        my $outcome = shift(@responses); 
+        foreach my $item (@responses) {
+            my ($key,$value) = split(/=/,$item);
+            $$photo{$key} = $value;
+        }
+        return $outcome;
+    }
+    return 'error';
+}
+
 sub auto_instcode_format {
     my ($caller,$codedom,$instcodes,$codes,$codetitles,$cat_titles,$cat_order) = @_;
     my $courses = '';
@@ -3725,7 +3985,6 @@ sub modify_group_roles {
     if ($result eq 'ok') {
         &devalidate_getgroups_cache($udom,$uname,$cdom,$cnum);
     }
-
     return $result;
 }
 
@@ -3759,31 +4018,51 @@ sub get_group_membership {
 
 sub get_users_groups {
     my ($udom,$uname,$courseid) = @_;
+    my @usersgroups;
     my $cachetime=1800;
     $courseid=~s/\_/\//g;
     $courseid=~s/^(\w)/\/$1/;
 
     my $hashid="$udom:$uname:$courseid";
-    my ($result,$cached)=&is_cached_new('getgroups',$hashid);
-    if (defined($cached)) { return $result; }
-
-    my %roleshash = &dump('roles',$udom,$uname,$courseid);
-    my ($tmp) = keys(%roleshash);
-    if ($tmp=~/^error:/) {
-        &logthis('Error retrieving roles: '.$tmp.' for '.$uname.':'.$udom);
-        return '';
-    } else {
-        my $grouplist;
-        foreach my $key (keys %roleshash) {
-            if ($key =~ /^\Q$courseid\E\/(\w+)\_gr$/) {
-                unless ($roleshash{$key} =~ /_1_1$/) {   # deleted membership
-                    $grouplist .= $1.':';
+    my ($grouplist,$cached)=&is_cached_new('getgroups',$hashid);
+    if (defined($cached)) {
+        @usersgroups = split(/:/,$grouplist);
+    } else {  
+        $grouplist = '';
+        my %roleshash = &dump('roles',$udom,$uname,$courseid);
+        my ($tmp) = keys(%roleshash);
+        if ($tmp=~/^error:/) {
+            &logthis('Error retrieving roles: '.$tmp.' for '.$uname.':'.$udom);
+        } else {
+            my $access_end = $env{'course.'.$courseid.
+                                  '.default_enrollment_end_date'};
+            my $now = time;
+            foreach my $key (keys(%roleshash)) {
+                if ($key =~ /^\Q$courseid\E\/(\w+)\_gr$/) {
+                    my $group = $1;
+                    if ($roleshash{$key} =~ /_(\d+)_(\d+)$/) {
+                        my $start = $2;
+                        my $end = $1;
+                        if ($start == -1) { next; } # deleted from group
+                        if (($start!=0) && ($start>$now)) { next; }
+                        if (($end!=0) && ($end<$now)) {
+                            if ($access_end && $access_end < $now) {
+                                if ($access_end - $end < 86400) {
+                                    push(@usersgroups,$group);
+                                }
+                            }
+                            next;
+                        }
+                        push(@usersgroups,$group);
+                    }
                 }
             }
+            @usersgroups = &sort_course_groups($courseid,@usersgroups);
+            $grouplist = join(':',@usersgroups);
+            &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
         }
-        $grouplist =~ s/:$//;
-        return &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
     }
+    return @usersgroups;
 }
 
 sub devalidate_getgroups_cache {
@@ -3848,6 +4127,8 @@ sub assignrole {
            $command.='_0_'.$start;
         }
     }
+    my $origstart = $start;
+    my $origend = $end;
 # actually delete
     if ($deleteflag) {
 	if ((&allowed('dro',$udom)) || (&allowed('dro',$url))) {
@@ -3865,6 +4146,11 @@ sub assignrole {
 # log new user role if status is ok
     if ($answer eq 'ok') {
 	&userrolelog($role,$uname,$udom,$url,$start,$end);
+# for course roles, perform group memberships changes triggered by role change.
+        unless ($role =~ /^gr/) {
+            &Apache::longroup::group_changes($udom,$uname,$url,$role,$origend,
+                                             $origstart);
+        }
     }
     return $answer;
 }
@@ -4527,13 +4813,69 @@ sub GetFileTimestamp {
     }
 }
 
+sub stat_file {
+    my ($uri) = @_;
+    $uri = &clutter($uri);
+
+    # we want just the url part without the unneeded accessor url bits
+    if ($uri =~ m-^/adm/-) {
+	$uri=~s-^/adm/wrapper/-/-;
+	$uri=~s-^/adm/coursedocs/showdoc/-/-;
+    }
+    my ($udom,$uname,$file,$dir);
+    if ($uri =~ m-^/(uploaded|editupload)/-) {
+	($udom,$uname,$file) =
+	    ($uri =~ m-/(?:uploaded|editupload)/?([^/]*)/?([^/]*)/?(.*)-);
+	$file = 'userfiles/'.$file;
+	$dir = &propath($udom,$uname);
+    }
+    if ($uri =~ m-^/res/-) {
+	($udom,$uname) = 
+	    ($uri =~ m-/(?:res)/?([^/]*)/?([^/]*)/-);
+	$file = $uri;
+    }
+
+    if (!$udom || !$uname || !$file) {
+	# unable to handle the uri
+	return ();
+    }
+
+    my ($result) = &dirlist($file,$udom,$uname,$dir);
+    my @stats = split('&', $result);
+    
+    if($stats[0] ne 'empty' && $stats[0] ne 'no_such_dir') {
+	shift(@stats); #filename is first
+	return @stats;
+    }
+    return ();
+}
+
 # -------------------------------------------------------- Value of a Condition
 
+# gets the value of a specific preevaluated condition
+#    stored in the string  $env{user.state.<cid>}
+# or looks up a condition reference in the bighash and if if hasn't
+# already been evaluated recurses into docondval to get the value of
+# the condition, then memoizing it to 
+#   $env{user.state.<cid>.<condition>}
 sub directcondval {
     my $number=shift;
     if (!defined($env{'user.state.'.$env{'request.course.id'}})) {
 	&Apache::lonuserstate::evalstate();
     }
+    if (exists($env{'user.state.'.$env{'request.course.id'}.".$number"})) {
+	return $env{'user.state.'.$env{'request.course.id'}.".$number"};
+    } elsif ($number =~ /^_/) {
+	my $sub_condition;
+	if (tie(my %bighash,'GDBM_File',$env{'request.course.fn'}.'.db',
+		&GDBM_READER(),0640)) {
+	    $sub_condition=$bighash{'conditions'.$number};
+	    untie(%bighash);
+	}
+	my $value = &docondval($sub_condition);
+	&appenv('user.state.'.$env{'request.course.id'}.".$number" => $value);
+	return $value;
+    }
     if ($env{'user.state.'.$env{'request.course.id'}}) {
        return substr($env{'user.state.'.$env{'request.course.id'}},$number,1);
     } else {
@@ -4541,43 +4883,49 @@ sub directcondval {
     }
 }
 
+# get the collection of conditions for this resource
 sub condval {
     my $condidx=shift;
-    my $result=0;
     my $allpathcond='';
-    foreach (split(/\|/,$condidx)) {
-       if (defined($env{'acc.cond.'.$env{'request.course.id'}.'.'.$_})) {
-	   $allpathcond.=
-               '('.$env{'acc.cond.'.$env{'request.course.id'}.'.'.$_}.')|';
-       }
+    foreach my $cond (split(/\|/,$condidx)) {
+	if (defined($env{'acc.cond.'.$env{'request.course.id'}.'.'.$cond})) {
+	    $allpathcond.=
+		'('.$env{'acc.cond.'.$env{'request.course.id'}.'.'.$cond}.')|';
+	}
     }
     $allpathcond=~s/\|$//;
-    if ($env{'request.course.id'}) {
-       if ($allpathcond) {
-          my $operand='|';
-	  my @stack;
-           foreach ($allpathcond=~/(\d+|\(|\)|\&|\|)/g) {
-              if ($_ eq '(') {
-                 push @stack,($operand,$result)
-              } elsif ($_ eq ')') {
-                  my $before=pop @stack;
-		  if (pop @stack eq '&') {
-		      $result=$result>$before?$before:$result;
-                  } else {
-                      $result=$result>$before?$result:$before;
-                  }
-              } elsif (($_ eq '&') || ($_ eq '|')) {
-                  $operand=$_;
-              } else {
-                  my $new=directcondval($_);
-                  if ($operand eq '&') {
-                     $result=$result>$new?$new:$result;
-                  } else {
-                     $result=$result>$new?$result:$new;
-                  }
-              }
-          }
-       }
+    return &docondval($allpathcond);
+}
+
+#evaluates an expression of conditions
+sub docondval {
+    my ($allpathcond) = @_;
+    my $result=0;
+    if ($env{'request.course.id'}
+	&& defined($allpathcond)) {
+	my $operand='|';
+	my @stack;
+	foreach my $chunk ($allpathcond=~/(\d+|_\d+\.\d+|\(|\)|\&|\|)/g) {
+	    if ($chunk eq '(') {
+		push @stack,($operand,$result);
+	    } elsif ($chunk eq ')') {
+		my $before=pop @stack;
+		if (pop @stack eq '&') {
+		    $result=$result>$before?$before:$result;
+		} else {
+		    $result=$result>$before?$result:$before;
+		}
+	    } elsif (($chunk eq '&') || ($chunk eq '|')) {
+		$operand=$chunk;
+	    } else {
+		my $new=directcondval($chunk);
+		if ($operand eq '&') {
+		    $result=$result>$new?$new:$result;
+		} else {
+		    $result=$result>$new?$result:$new;
+		}
+	    }
+	}
     }
     return $result;
 }
@@ -4694,8 +5042,8 @@ sub EXT_cache_set {
 
 # --------------------------------------------------------- Value of a Variable
 sub EXT {
-    my ($varname,$symbparm,$udom,$uname,$usection,$recurse)=@_;
 
+    my ($varname,$symbparm,$udom,$uname,$usection,$recurse)=@_;
     unless ($varname) { return ''; }
     #get real user name/domain, courseid and symb
     my $courseid;
@@ -4809,11 +5157,21 @@ sub EXT {
         return $env{'course.'.$courseid.'.'.$spacequalifierrest};
     } elsif ($realm eq 'resource') {
 
-	my ($section,$group);
-        my @groups = ();
 	if (defined($courseid) && $courseid eq $env{'request.course.id'}) {
 	    if (!$symbparm) { $symbparm=&symbread(); }
 	}
+
+	if ($space eq 'title') {
+	    if (!$symbparm) { $symbparm = $env{'request.filename'}; }
+	    return &gettitle($symbparm);
+	}
+	
+	if ($space eq 'map') {
+	    my ($map) = &decode_symb($symbparm);
+	    return &symbread($map);
+	}
+
+	my ($section, $group, @groups);
 	my ($courselevelm,$courselevel);
 	if ($symbparm && defined($courseid) && 
 	    $courseid eq $env{'request.course.id'}) {
@@ -4822,7 +5180,7 @@ sub EXT {
 
 # ----------------------------------------------------- Cascading lookup scheme
 	    my $symbp=$symbparm;
-	    my $mapp=(&decode_symb($symbp))[0];
+	    my $mapp=&deversion((&decode_symb($symbp))[0]);
 
 	    my $symbparm=$symbp.'.'.$spacequalifierrest;
 	    my $mapparm=$mapp.'___(all).'.$spacequalifierrest;
@@ -4830,29 +5188,17 @@ sub EXT {
 	    if (($env{'user.name'} eq $uname) &&
 		($env{'user.domain'} eq $udom)) {
 		$section=$env{'request.course.sec'};
-                @groups=split(/:/,$env{'request.course.groups'});
-                if (@groups > 0) {
-                    @groups = sort(@groups);
-                    $group = $groups[0];
-                }
+                @groups = split(/:/,$env{'request.course.groups'});  
+                @groups=&sort_course_groups($courseid,@groups); 
 	    } else {
 		if (! defined($usection)) {
 		    $section=&getsection($udom,$uname,$courseid);
 		} else {
 		    $section = $usection;
 		}
-                my $grouplist = &get_users_groups($udom,$uname,$courseid);
-                if ($grouplist) {
-                    @groups = split(/:/,$grouplist);
-                    @groups = sort(@groups);
-                    $group = $groups[0];
-                }
+                @groups = &get_users_groups($udom,$uname,$courseid);
 	    }
 
-            my $grplevel=$courseid.'.['.$group.'].'.$spacequalifierrest;
-            my $grplevelr=$courseid.'.['.$group.'].'.$symbparm;
-            my $grplevelm=$courseid.'.['.$group.'].'.$mapparm;
-
 	    my $seclevel=$courseid.'.['.$section.'].'.$spacequalifierrest;
 	    my $seclevelr=$courseid.'.['.$section.'].'.$symbparm;
 	    my $seclevelm=$courseid.'.['.$section.'].'.$mapparm;
@@ -4866,17 +5212,13 @@ sub EXT {
 	    my $userreply=&resdata($uname,$udom,'user',
 				       ($courselevelr,$courselevelm,
 					$courselevel));
-
 	    if (defined($userreply)) { return $userreply; }
 
 # ------------------------------------------------ second, check some of course
             my $coursereply;
-            if (defined($group)) {
-                $coursereply = &resdata($env{'course.'.$courseid.'.num'},
-                                     $env{'course.'.$courseid.'.domain'},
-                                     'course',
-                                     ($grplevelr,$grplevelm,$grplevel,
-                                      $courselevelr));
+            if (@groups > 0) {
+                $coursereply = &check_group_parms($courseid,\@groups,$symbparm,
+                                       $mapparm,$spacequalifierrest);
                 if (defined($coursereply)) { return $coursereply; }
             }
 
@@ -4951,16 +5293,64 @@ sub EXT {
 	if ($space eq 'time') {
 	    return time;
         }
+    } elsif ($realm eq 'server') {
+# ----------------------------------------------------------------- system.time
+	if ($space eq 'name') {
+	    return $ENV{'SERVER_NAME'};
+        }
     }
     return '';
 }
 
+sub check_group_parms {
+    my ($courseid,$groups,$symbparm,$mapparm,$what) = @_;
+    my @groupitems = ();
+    my $resultitem;
+    my @levels = ($symbparm,$mapparm,$what);
+    foreach my $group (@{$groups}) {
+        foreach my $level (@levels) {
+             my $item = $courseid.'.['.$group.'].'.$level;
+             push(@groupitems,$item);
+        }
+    }
+    my $coursereply = &resdata($env{'course.'.$courseid.'.num'},
+                            $env{'course.'.$courseid.'.domain'},
+                                     'course',@groupitems);
+    return $coursereply;
+}
+
+sub sort_course_groups { # Sort groups based on defined rankings. Default is sort().
+    my ($courseid,@groups) = @_;
+    @groups = sort(@groups);
+    return @groups;
+}
+
 sub packages_tab_default {
     my ($uri,$varname)=@_;
     my (undef,$part,$name)=split(/\./,$varname);
-    my $packages=&metadata($uri,'packages');
-    foreach my $package (split(/,/,$packages)) {
+
+    my (@extension,@specifics,$do_default);
+    foreach my $package (split(/,/,&metadata($uri,'packages'))) {
 	my ($pack_type,$pack_part)=split(/_/,$package,2);
+	if ($pack_type eq 'default') {
+	    $do_default=1;
+	} elsif ($pack_type eq 'extension') {
+	    push(@extension,[$package,$pack_type,$pack_part]);
+	} else {
+	    push(@specifics,[$package,$pack_type,$pack_part]);
+	}
+    }
+    # first look for a package that matches the requested part id
+    foreach my $package (@specifics) {
+	my (undef,$pack_type,$pack_part)=@{$package};
+	next if ($pack_part ne $part);
+	if (defined($packagetab{"$pack_type&$name&default"})) {
+	    return $packagetab{"$pack_type&$name&default"};
+	}
+    }
+    # look for any possible matching non extension_ package
+    foreach my $package (@specifics) {
+	my (undef,$pack_type,$pack_part)=@{$package};
 	if (defined($packagetab{"$pack_type&$name&default"})) {
 	    return $packagetab{"$pack_type&$name&default"};
 	}
@@ -4969,6 +5359,20 @@ sub packages_tab_default {
 	    return $packagetab{$pack_type."_".$pack_part."&$name&default"};
 	}
     }
+    # look for any posible extension_ match
+    foreach my $package (@extension) {
+	my ($package,$pack_type)=@{$package};
+	if (defined($packagetab{"$pack_type&$name&default"})) {
+	    return $packagetab{"$pack_type&$name&default"};
+	}
+	if (defined($packagetab{$package."&$name&default"})) {
+	    return $packagetab{$package."&$name&default"};
+	}
+    }
+    # look for a global default setting
+    if ($do_default && defined($packagetab{"default&$name&default"})) {
+	return $packagetab{"default&$name&default"};
+    }
     return undef;
 }
 
@@ -5054,16 +5458,16 @@ sub metadata {
 		    } else {
 			$metaentry{':packages'}=$package.$keyroot;
 		    }
-		    foreach (sort keys %packagetab) {
+		    foreach my $pack_entry (keys(%packagetab)) {
 			my $part=$keyroot;
 			$part=~s/^\_//;
-			if ($_=~/^\Q$package\E\&/ || 
-			    $_=~/^\Q$package\E_0\&/) {
-			    my ($pack,$name,$subp)=split(/\&/,$_);
+			if ($pack_entry=~/^\Q$package\E\&/ || 
+			    $pack_entry=~/^\Q$package\E_0\&/) {
+			    my ($pack,$name,$subp)=split(/\&/,$pack_entry);
 			    # ignore package.tab specified default values
                             # here &package_tab_default() will fetch those
 			    if ($subp eq 'default') { next; }
-			    my $value=$packagetab{$_};
+			    my $value=$packagetab{$pack_entry};
 			    my $unikey;
 			    if ($pack =~ /_0$/) {
 				$unikey='parameter_0_'.$name;
@@ -5111,11 +5515,12 @@ sub metadata {
 			    my $dir=$filename;
 			    $dir=~s|[^/]*$||;
 			    $location=&filelocation($dir,$location);
-			    foreach (sort(split(/\,/,&metadata($uri,'keys',
-							       $location,$unikey,
-							       $depthcount+1)))) {
-				$metaentry{':'.$_}=$metaentry{':'.$_};
-				$metathesekeys{$_}=1;
+			    my $metadata = 
+				&metadata($uri,'keys', $location,$unikey,
+					  $depthcount+1);
+			    foreach my $meta (split(',',$metadata)) {
+				$metaentry{':'.$meta}=$metaentry{':'.$meta};
+				$metathesekeys{$meta}=1;
 			    }
 			}
 		    } else { 
@@ -5124,8 +5529,9 @@ sub metadata {
 			    $unikey.='_'.$token->[2]->{'name'}; 
 			}
 			$metathesekeys{$unikey}=1;
-			foreach (@{$token->[3]}) {
-			    $metaentry{':'.$unikey.'.'.$_}=$token->[2]->{$_};
+			foreach my $param (@{$token->[3]}) {
+			    $metaentry{':'.$unikey.'.'.$param} =
+				$token->[2]->{$param};
 			}
 			my $internaltext=&HTML::Entities::decode($parser->get_text('/'.$entry));
 			my $default=$metaentry{':'.$unikey.'.default'};
@@ -5146,14 +5552,14 @@ sub metadata {
 	    }
 	}
 	my ($extension) = ($uri =~ /\.(\w+)$/);
-	foreach my $key (sort(keys(%packagetab))) {
+	foreach my $key (keys(%packagetab)) {
 	    #no specific packages #how's our extension
 	    if ($key!~/^extension_\Q$extension\E&/) { next; }
 	    &metadata_create_package_def($uri,$key,'extension_'.$extension,
 					 \%metathesekeys);
 	}
 	if (!exists($metaentry{':packages'})) {
-	    foreach my $key (sort(keys(%packagetab))) {
+	    foreach my $key (keys(%packagetab)) {
 		#no specific packages well let's get default then
 		if ($key!~/^default&/) { next; }
 		&metadata_create_package_def($uri,$key,'default',
@@ -5171,18 +5577,25 @@ sub metadata {
 		my $dir=$filename;
 		$dir=~s|[^/]*$||;
 		$location=&filelocation($dir,$location);
-		foreach (sort(split(/\,/,&metadata($uri,'keys',
-						   $location,'_rights',
-						   $depthcount+1)))) {
-		    #$metaentry{':'.$_}=$metacache{$uri}->{':'.$_};
-		    $metathesekeys{$_}=1;
+		my $rights_metadata =
+		    &metadata($uri,'keys',$location,'_rights',
+			      $depthcount+1);
+		foreach my $rights (split(',',$rights_metadata)) {
+		    #$metaentry{':'.$rights}=$metacache{$uri}->{':'.$rights};
+		    $metathesekeys{$rights}=1;
 		}
 	    }
 	}
-	$metaentry{':keys'}=join(',',keys %metathesekeys);
+	# uniqifiy package listing
+	my %seen;
+	my @uniq_packages =
+	    grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'}));
+	$metaentry{':packages'} = join(',',@uniq_packages);
+
+	$metaentry{':keys'} = join(',',keys(%metathesekeys));
 	&metadata_generate_part0(\%metathesekeys,\%metaentry,$uri);
 	$metaentry{':allpossiblekeys'}=join(',',keys %metathesekeys);
-	&do_cache_new('meta',$uri,\%metaentry,60*60*24);
+	&do_cache_new('meta',$uri,\%metaentry,60*60);
 # this is the end of "was not already recently cached
     }
     return $metaentry{':'.$what};
@@ -5215,7 +5628,7 @@ sub metadata_create_package_def {
 sub metadata_generate_part0 {
     my ($metadata,$metacache,$uri) = @_;
     my %allnames;
-    foreach my $metakey (sort keys %$metadata) {
+    foreach my $metakey (keys(%$metadata)) {
 	if ($metakey=~/^parameter\_(.*)/) {
 	  my $part=$$metacache{':'.$metakey.'.part'};
 	  my $name=$$metacache{':'.$metakey.'.name'};
@@ -5278,10 +5691,17 @@ sub get_slot {
 	$cdom=$env{'course.'.$courseid.'.domain'};
 	$cnum=$env{'course.'.$courseid.'.num'};
     }
-    my %slotinfo=&get('slots',[$which],$cdom,$cnum);
-    &Apache::lonhomework::showhash(%slotinfo);
-    my ($tmp)=keys(%slotinfo);
-    if ($tmp=~/^error:/) { return (); }
+    my $key=join("\0",'slots',$cdom,$cnum,$which);
+    my %slotinfo;
+    if (exists($remembered{$key})) {
+	$slotinfo{$which} = $remembered{$key};
+    } else {
+	%slotinfo=&get('slots',[$which],$cdom,$cnum);
+	&Apache::lonhomework::showhash(%slotinfo);
+	my ($tmp)=keys(%slotinfo);
+	if ($tmp=~/^error:/) { return (); }
+	$remembered{$key} = $slotinfo{$which};
+    }
     if (ref($slotinfo{$which}) eq 'HASH') {
 	return %{$slotinfo{$which}};
     }
@@ -5296,9 +5716,12 @@ sub symblist {
     if (($env{'request.course.fn'}) && (%newhash)) {
         if (tie(%hash,'GDBM_File',$env{'request.course.fn'}.'_symb.db',
                       &GDBM_WRCREAT(),0640)) {
-	    foreach (keys %newhash) {
-                $hash{declutter($_)}=&encode_symb($mapname,$newhash{$_}->[1],
-						  $newhash{$_}->[0]);
+	    foreach my $url (keys %newhash) {
+		next if ($url eq 'last_known'
+			 && $env{'form.no_update_last_known'});
+		$hash{declutter($url)}=&encode_symb($mapname,
+						    $newhash{$url}->[1],
+						    $newhash{$url}->[0]);
             }
             if (untie(%hash)) {
 		return 'ok';
@@ -5315,6 +5738,7 @@ sub symbverify {
     my $thisfn=$thisurl;
 # wrapper not part of symbs
     $thisfn=~s/^\/adm\/wrapper//;
+    $thisfn=~s/^\/adm\/coursedocs\/showdoc\///;
     $thisfn=&declutter($thisfn);
 # direct jump to resource in page or to a sequence - will construct own symbs
     if ($thisfn=~/\.(page|sequence)$/) { return 1; }
@@ -5369,6 +5793,7 @@ sub symbclean {
 # remove wrapper
 
     $symb=~s/(\_\_\_\d+\_\_\_)adm\/wrapper\/(res\/)*/$1/;
+    $symb=~s/(\_\_\_\d+\_\_\_)adm\/coursedocs\/showdoc\/(res\/)*/$1/;
     return $symb;
 }
 
@@ -5445,6 +5870,9 @@ sub symbread {
         if ( ($thisfn =~ m/^(uploaded|editupload)\//) && ($thisfn !~ m/\.(page|sequence)$/) ) {
             $targetfn = 'adm/wrapper/'.$thisfn;
         }
+	if ($targetfn =~ m|^adm/wrapper/(ext/.*)|) {
+	    $targetfn=$1;
+	}
         if (tie(%hash,'GDBM_File',$env{'request.course.fn'}.'_symb.db',
                       &GDBM_READER(),0640)) {
 	    $syval=$hash{$targetfn};
@@ -6016,6 +6444,11 @@ sub filelocation {
     my ($dir,$file) = @_;
     my $location;
     $file=~ s/^\s*(\S+)\s*$/$1/; ## strip off leading and trailing spaces
+
+    if ($file =~ m-^/adm/-) {
+	$file=~s-^/adm/wrapper/-/-;
+	$file=~s-^/adm/coursedocs/showdoc/-/-;
+    }
     if ($file=~m:^/~:) { # is a contruction space reference
         $location = $file;
         $location =~ s:/~(.*?)/(.*):/home/$1/public_html/$2:;
@@ -6030,7 +6463,7 @@ sub filelocation {
         my @ids=&current_machine_ids();
         foreach my $id (@ids) { if ($id eq $home) { $is_me=1; } }
         if ($is_me) {
-  	    $location=&Apache::loncommon::propath($udom,$uname).
+  	    $location=&propath($udom,$uname).
   	      '/userfiles/'.$filename;
         } else {
   	  $location=$Apache::lonnet::perlvar{'lonDocRoot'}.'/userfiles/'.
@@ -6055,6 +6488,9 @@ sub hreflocation {
     my ($dir,$file)=@_;
     unless (($file=~m-^http://-i) || ($file=~m-^/-)) {
 	$file=filelocation($dir,$file);
+    } elsif ($file=~m-^/adm/-) {
+	$file=~s-^/adm/wrapper/-/-;
+	$file=~s-^/adm/coursedocs/showdoc/-/-;
     }
     if ($file=~m-^\Q$perlvar{'lonDocRoot'}\E-) {
 	$file=~s-^\Q$perlvar{'lonDocRoot'}\E--;
@@ -6098,6 +6534,8 @@ sub declutter {
     if ($thisfn=~m|^/enc/|) { $thisfn=&Apache::lonenc::unencrypted($thisfn); }
     $thisfn=~s/^\Q$perlvar{'lonDocRoot'}\E//;
     $thisfn=~s/^\///;
+    $thisfn=~s|^adm/wrapper/||;
+    $thisfn=~s|^adm/coursedocs/showdoc/||;
     $thisfn=~s/^res\///;
     $thisfn=~s/\?.+$//;
     return $thisfn;
@@ -6110,6 +6548,30 @@ sub clutter {
     unless ($thisfn=~/^\/(uploaded|editupload|adm|userfiles|ext|raw|priv|public)\//) { 
        $thisfn='/res'.$thisfn; 
     }
+    if ($thisfn !~m|/adm|) {
+	if ($thisfn =~ m|/ext/|) {
+	    $thisfn='/adm/wrapper'.$thisfn;
+	} else {
+	    my ($ext) = ($thisfn =~ /\.(\w+)$/);
+	    my $embstyle=&Apache::loncommon::fileembstyle($ext);
+	    if ($embstyle eq 'ssi'
+		|| ($embstyle eq 'hdn')
+		|| ($embstyle eq 'rat')
+		|| ($embstyle eq 'prv')
+		|| ($embstyle eq 'ign')) {
+		#do nothing with these
+	    } elsif (($embstyle eq 'img') 
+		|| ($embstyle eq 'emb')
+		|| ($embstyle eq 'wrp')) {
+		$thisfn='/adm/wrapper'.$thisfn;
+	    } elsif ($embstyle eq 'unk'
+		     && $thisfn!~/\.(sequence|page)$/) {
+		$thisfn='/adm/coursedocs/showdoc'.$thisfn;
+	    } else {
+#		&logthis("Got a blank emb style");
+	    }
+	}
+    }
     return $thisfn;
 }
 
@@ -6122,21 +6584,6 @@ sub freeze_escape {
     return &escape($value);
 }
 
-# -------------------------------------------------------- Escape Special Chars
-
-sub escape {
-    my $str=shift;
-    $str =~ s/(\W)/"%".unpack('H2',$1)/eg;
-    return $str;
-}
-
-# ----------------------------------------------------- Un-Escape Special Chars
-
-sub unescape {
-    my $str=shift;
-    $str =~ s/%([a-fA-F0-9][a-fA-F0-9])/pack("C",hex($1))/eg;
-    return $str;
-}
 
 sub thaw_unescape {
     my ($value)=@_;
@@ -6174,7 +6621,6 @@ sub goodbye {
    &logthis(sprintf("%-20s is %s",'hits',$hits));
    &flushcourselogs();
    &logthis("Shutting down");
-   return DONE;
 }
 
 BEGIN {
@@ -6927,6 +7373,27 @@ all args are optional
 
 =item *
 
+dumpstore($namespace,$udom,$uname,$regexp,$range) : 
+dumps the complete (or key matching regexp) namespace into a hash
+($udom, $uname, $regexp, $range are optional) for a namespace that is
+normally &store()ed into
+
+$range should be either an integer '100' (give me the first 100
+                                           matching records)
+              or be  two integers sperated by a - with no spaces
+                 '30-50' (give me the 30th through the 50th matching
+                          records)
+
+
+=item *
+
+putstore($namespace,$symb,$version,$storehash,$udomain,$uname) :
+replaces a &store() version of data with a replacement set of data
+for a particular resource in a namespace passed in the $storehash hash 
+reference
+
+=item *
+
 tmpstore($storehash,$symb,$namespace,$udom,$uname) : storage that
 works very similar to store/cstore, but all data is stored in a
 temporary location and can be reset using tmpreset, $storehash should
@@ -6956,10 +7423,15 @@ namesp ($udom and $uname are optional)
 
 =item *
 
-dump($namespace,$udom,$uname,$regexp) : 
+dump($namespace,$udom,$uname,$regexp,$range) : 
 dumps the complete (or key matching regexp) namespace into a hash
-($udom, $uname and $regexp are optional)
+($udom, $uname, $regexp, $range are optional)
 
+$range should be either an integer '100' (give me the first 100
+                                           matching records)
+              or be  two integers sperated by a - with no spaces
+                 '30-50' (give me the 30th through the 50th matching
+                          records)
 =item *
 
 inc($namespace,$store,$udom,$uname) : increments $store in $namespace.
@@ -6975,17 +7447,6 @@ put($namespace,$storehash,$udom,$uname)
 
 =item *
 
-putstore($namespace,$storehash,$udomain,$uname) : stores hash in namesp
-keys used in storehash include version information (e.g., 1:$symb:message etc.) as
-used in records written by &store and retrieved by &restore.  This function 
-was created for use in editing discussion posts, without incrementing the
-version number included in the key for a particular post. The colon 
-separated list of attribute names (e.g., the value associated with the key 
-1:keys:$symb) is also generated and passed in the ampersand separated 
-items sent to lonnet::reply().  
-
-=item *
-
 cput($namespace,$storehash,$udom,$uname) : critical put
 ($udom and $uname are optional)
 
@@ -7115,6 +7576,16 @@ getfile($file,$caller) : two cases - req
    - returns the entire contents of a file or -1; 
    it properly subscribes to and replicates the file if neccessary.
 
+
+=item *
+
+stat_file($url) : $url is expected to be a /res/ or /uploaded/ style file
+                  reference
+
+returns either a stat() list of data about the file or an empty list
+if the file doesn't exist or couldn't find out about it (connection
+problems or user unknown)
+
 =item *
 
 filelocation($dir,$file) : returns file system location of a file