--- loncom/lonnet/perl/lonnet.pm	2006/10/23 21:22:44	1.799
+++ loncom/lonnet/perl/lonnet.pm	2007/02/16 01:04:19	1.832
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.799 2006/10/23 21:22:44 raeburn Exp $
+# $Id: lonnet.pm,v 1.832 2007/02/16 01:04:19 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -53,8 +53,7 @@ use Time::HiRes qw( gettimeofday tv_inte
 use Cache::Memcached;
 use Digest::MD5;
 use Math::Random;
-use lib '/home/httpd/lib/perl';
-use LONCAPA;
+use LONCAPA qw(:DEFAULT :match);
 use LONCAPA::Configuration;
 
 my $readit;
@@ -368,6 +367,26 @@ sub transfer_profile_to_env {
     }
 }
 
+sub timed_flock {
+    my ($file,$lock_type) = @_;
+    my $failed=0;
+    eval {
+	local $SIG{__DIE__}='DEFAULT';
+	local $SIG{ALRM}=sub {
+	    $failed=1;
+	    die("failed lock");
+	};
+	alarm(13);
+	flock($file,$lock_type);
+	alarm(0);
+    };
+    if ($failed) {
+	return undef;
+    } else {
+	return 1;
+    }
+}
+
 # ---------------------------------------------------------- Append Environment
 
 sub appenv {
@@ -382,8 +401,11 @@ sub appenv {
             $env{$key}=$newenv{$key};
         }
     }
-    if (tie(my %disk_env,'GDBM_File',$env{'user.environment'},&GDBM_WRITER(),
-	    0640)) {
+    open(my $env_file,$env{'user.environment'});
+    if (&timed_flock($env_file,LOCK_EX)
+	&&
+	tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	    (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
 	while (my ($key,$value) = each(%newenv)) {
 	    $disk_env{$key} = $value;
 	}
@@ -400,8 +422,11 @@ sub delenv {
                 "Attempt to delete from environment ".$delthis);
         return 'error';
     }
-    if (tie(my %disk_env,'GDBM_File',$env{'user.environment'},&GDBM_WRITER(),
-	    0640)) {
+    open(my $env_file,$env{'user.environment'});
+    if (&timed_flock($env_file,LOCK_EX)
+	&&
+	tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	    (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
 	foreach my $key (keys(%disk_env)) {
 	    if ($key=~/^$delthis/) { 
                 delete($env{$key});
@@ -588,8 +613,8 @@ sub queryauthenticate {
 
 sub authenticate {
     my ($uname,$upass,$udom)=@_;
-    $upass=escape($upass);
-    $uname=~s/\W//g;
+    $upass=&escape($upass);
+    $uname= &LONCAPA::clean_username($uname);
     my $uhome=&homeserver($uname,$udom);
     if (!$uhome) {
 	&logthis("User $uname at $udom is unknown in authenticate");
@@ -664,8 +689,8 @@ sub idget {
 sub idrget {
     my ($udom,@unames)=@_;
     my %returnhash=();
-    foreach (@unames) {
-        $returnhash{$_}=(&userenvironment($udom,$_,'id'))[1];
+    foreach my $uname (@unames) {
+        $returnhash{$uname}=(&userenvironment($udom,$uname,'id'))[1];
     }
     return %returnhash;
 }
@@ -675,22 +700,69 @@ sub idrget {
 sub idput {
     my ($udom,%ids)=@_;
     my %servers=();
-    foreach (keys %ids) {
-	&cput('environment',{'id'=>$ids{$_}},$udom,$_);
-        my $uhom=&homeserver($_,$udom);
+    foreach my $uname (keys(%ids)) {
+	&cput('environment',{'id'=>$ids{$uname}},$udom,$uname);
+        my $uhom=&homeserver($uname,$udom);
         if ($uhom ne 'no_host') {
-            my $id=&escape($ids{$_});
+            my $id=&escape($ids{$uname});
             $id=~tr/A-Z/a-z/;
-            my $unam=&escape($_);
+            my $esc_unam=&escape($uname);
 	    if ($servers{$uhom}) {
-		$servers{$uhom}.='&'.$id.'='.$unam;
+		$servers{$uhom}.='&'.$id.'='.$esc_unam;
             } else {
-                $servers{$uhom}=$id.'='.$unam;
+                $servers{$uhom}=$id.'='.$esc_unam;
             }
         }
     }
-    foreach (keys %servers) {
-        &critical('idput:'.$udom.':'.$servers{$_},$_);
+    foreach my $server (keys(%servers)) {
+        &critical('idput:'.$udom.':'.$servers{$server},$server);
+    }
+}
+
+# ------------------------------------------- get items from domain db files   
+
+sub get_dom {
+    my ($namespace,$storearr,$udom)=@_;
+    my $items='';
+    foreach my $item (@$storearr) {
+        $items.=&escape($item).'&';
+    }
+    $items=~s/\&$//;
+    if (!$udom) { $udom=$env{'user.domain'}; }
+    if (exists($domain_primary{$udom})) {
+        my $uhome=$domain_primary{$udom};
+        my $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        my @pairs=split(/\&/,$rep);
+        if ( $#pairs==0 && $pairs[0] =~ /^(con_lost|error|no_such_host)/i) {
+            return @pairs;
+        }
+        my %returnhash=();
+        my $i=0;
+        foreach my $item (@$storearr) {
+            $returnhash{$item}=&thaw_unescape($pairs[$i]);
+            $i++;
+        }
+        return %returnhash;
+    } else {
+        &logthis("get_dom failed - no primary domain server for $udom");
+    }
+}
+
+# -------------------------------------------- put items in domain db files 
+
+sub put_dom {
+    my ($namespace,$storehash,$udom)=@_;
+    if (!$udom) { $udom=$env{'user.domain'}; }
+    if (exists($domain_primary{$udom})) {
+        my $uhome=$domain_primary{$udom};
+        my $items='';
+        foreach my $item (keys(%$storehash)) {
+            $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
+        }
+        $items=~s/\&$//;
+        return &reply("putdom:$udom:$namespace:$items",$uhome);
+    } else {
+        &logthis("put_dom failed - no primary domain server for $udom");
     }
 }
 
@@ -827,17 +899,32 @@ sub validate_access_key {
 # ------------------------------------- Find the section of student in a course
 sub devalidate_getsection_cache {
     my ($udom,$unam,$courseid)=@_;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
     my $hashid="$udom:$unam:$courseid";
     &devalidate_cache_new('getsection',$hashid);
 }
 
+sub courseid_to_courseurl {
+    my ($courseid) = @_;
+    #already url style courseid
+    return $courseid if ($courseid =~ m{^/});
+
+    if (exists($env{'course.'.$courseid.'.num'})) {
+	my $cnum = $env{'course.'.$courseid.'.num'};
+	my $cdom = $env{'course.'.$courseid.'.domain'};
+	return "/$cdom/$cnum";
+    }
+
+    my %courseinfo=&Apache::lonnet::coursedescription($courseid);
+    if (exists($courseinfo{'num'})) {
+	return "/$courseinfo{'domain'}/$courseinfo{'num'}";
+    }
+
+    return undef;
+}
+
 sub getsection {
     my ($udom,$unam,$courseid)=@_;
     my $cachetime=1800;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
 
     my $hashid="$udom:$unam:$courseid";
     my ($result,$cached)=&is_cached_new('getsection',$hashid);
@@ -858,14 +945,13 @@ sub getsection {
     # If there is more than one expired role, choose the one which ended last.
     # If there is a role which has expired, return it.
     #
-    foreach (split(/\&/,&reply('dump:'.$udom.':'.$unam.':roles',
-                        &homeserver($unam,$udom)))) {
-        my ($key,$value)=split(/\=/,$_);
-        $key=&unescape($key);
+    $courseid = &courseid_to_courseurl($courseid);
+    my %roleshash = &dump('roles',$udom,$unam,$courseid);
+    foreach my $key (keys(%roleshash)) {
         next if ($key !~/^\Q$courseid\E(?:\/)*(\w+)*\_st$/);
         my $section=$1;
         if ($key eq $courseid.'_st') { $section=''; }
-        my ($dummy,$end,$start)=split(/\_/,&unescape($value));
+        my ($dummy,$end,$start)=split(/\_/,&unescape($roleshash{$key}));
         my $now=time;
         if (defined($end) && $end && ($now > $end)) {
             $Expired{$end}=$section;
@@ -1116,6 +1202,7 @@ sub repcopy {
     }
     $filename=~s/[\n\r]//g;
     my $transname="$filename.in.transfer";
+# FIXME: this should flock
     if ((-e $filename) || (-e $transname)) { return 'ok'; }
     my $remoteurl=subscribe($filename);
     if ($remoteurl =~ /^con_lost by/) {
@@ -1364,15 +1451,17 @@ sub store_edited_file {
 }
 
 sub clean_filename {
-    my ($fname)=@_;
+    my ($fname,$args)=@_;
 # Replace Windows backslashes by forward slashes
     $fname=~s/\\/\//g;
-# Get rid of everything but the actual filename
-    $fname=~s/^.*\/([^\/]+)$/$1/;
+    if (!$args->{'keep_path'}) {
+        # Get rid of everything but the actual filename
+	$fname=~s/^.*\/([^\/]+)$/$1/;
+    }
 # Replace spaces by underscores
     $fname=~s/\s+/\_/g;
 # Replace all other weird characters by nothing
-    $fname=~s/[^\w\.\-]//g;
+    $fname=~s{[^/\w\.\-]}{}g;
 # Replace all .\d. sequences with _\d. so they no longer look like version
 # numbers
     $fname=~s/\.(\d+)(?=\.)/_$1/g;
@@ -1641,6 +1730,12 @@ sub removeuserfile {
         if (($fname !~ /\.meta$/) && (&is_portfolio_file($fname))) {
             my $metafile = $fname.'.meta';
             my $metaresult = &removeuserfile($docuname,$docudom,$metafile); 
+	    my $url = "/uploaded/$docudom/$docuname/$fname";
+            my ($file,$group) = (&parse_portfolio_url($url))[3,4];
+            my $sqlresult = 
+                &update_portfolio_table($docuname,$docudom,$file,
+                                        'portfolio_metadata',$group,
+                                        'delete');
         }
     }
     return $result;
@@ -1663,6 +1758,12 @@ sub renameuserfile {
             my $newmeta = $new.'.meta';
             my $metaresult = 
                 &renameuserfile($docuname,$docudom,$oldmeta,$newmeta);
+	    my $url = "/uploaded/$docudom/$docuname/$old";
+            my ($file,$group) = (&parse_portfolio_url($url))[3,4];
+            my $sqlresult = 
+                &update_portfolio_table($docuname,$docudom,$file,
+                                        'portfolio_metadata',$group,
+                                        'delete');
         }
     }
     return $result;
@@ -1691,8 +1792,7 @@ sub flushcourselogs {
 # times and course titles for all courseids
 #
     my %courseidbuffer=();
-    foreach (keys %courselogs) {
-        my $crsid=$_;
+    foreach my $crsid (keys %courselogs) {
         if (&reply('log:'.$coursedombuf{$crsid}.':'.$coursenumbuf{$crsid}.':'.
 		          &escape($courselogs{$crsid}),
 		          $coursehombuf{$crsid}) eq 'ok') {
@@ -1719,8 +1819,8 @@ sub flushcourselogs {
 # Write course id database (reverse lookup) to homeserver of courses 
 # Is used in pickcourse
 #
-    foreach (keys %courseidbuffer) {
-        &courseidput($hostdom{$_},$courseidbuffer{$_},$_);
+    foreach my $crsid (keys(%courseidbuffer)) {
+        &courseidput($hostdom{$crsid},$courseidbuffer{$crsid},$crsid);
     }
 #
 # File accesses
@@ -1729,7 +1829,8 @@ sub flushcourselogs {
     foreach my $entry (keys(%accesshash)) {
         if ($entry =~ /___count$/) {
             my ($dom,$name);
-            ($dom,$name,undef)=($entry=~m:___(\w+)/(\w+)/(.*)___count$:);
+            ($dom,$name,undef)=
+		($entry=~m{___($match_domain)/($match_name)/(.*)___count$});
             if (! defined($dom) || $dom eq '' || 
                 ! defined($name) || $name eq '') {
                 my $cid = $env{'request.course.id'};
@@ -1750,7 +1851,7 @@ sub flushcourselogs {
                 }
             }
         } else {
-            my ($dom,$name) = ($entry=~m:___(\w+)/(\w+)/(.*)___(\w+)$:);
+            my ($dom,$name) = ($entry=~m{___($match_domain)/($match_name)/(.*)___(\w+)$});
             my %temphash=($entry => $accesshash{$entry});
             if (&put('nohist_resevaldata',\%temphash,$dom,$name) eq 'ok') {
                 delete $accesshash{$entry};
@@ -1761,8 +1862,7 @@ sub flushcourselogs {
 # Roles
 # Reverse lookup of user roles for course faculty/staff and co-authorship
 #
-    foreach (keys %userrolehash) {
-        my $entry=$_;
+    foreach my $entry (keys(%userrolehash)) {
         my ($role,$uname,$udom,$runame,$rudom,$rsec)=
 	    split(/\:/,$entry);
         if (&Apache::lonnet::put('nohist_userroles',
@@ -1834,9 +1934,9 @@ sub courseacclog {
     if ($fnsymb=~/(problem|exam|quiz|assess|survey|form|task|page)$/) {
         $what.=':POST';
         # FIXME: Probably ought to escape things....
-	foreach (keys %env) {
-            if ($_=~/^form\.(.*)/) {
-		$what.=':'.$1.'='.$env{$_};
+	foreach my $key (keys(%env)) {
+            if ($key=~/^form\.(.*)/) {
+		$what.=':'.$1.'='.$env{$key};
             }
         }
     } elsif ($fnsymb =~ m:^/adm/searchcat:) {
@@ -1898,19 +1998,19 @@ sub get_course_adv_roles {
     $cid=$env{'request.course.id'} unless (defined($cid));
     my %coursehash=&coursedescription($cid);
     my %nothide=();
-    foreach (split(/\s*\,\s*/,$coursehash{'nothideprivileged'})) {
-	$nothide{join(':',split(/[\@\:]/,$_))}=1;
+    foreach my $user (split(/\s*\,\s*/,$coursehash{'nothideprivileged'})) {
+	$nothide{join(':',split(/[\@\:]/,$user))}=1;
     }
     my %returnhash=();
     my %dumphash=
             &dump('nohist_userroles',$coursehash{'domain'},$coursehash{'num'});
     my $now=time;
-    foreach (keys %dumphash) {
-	my ($tend,$tstart)=split(/\:/,$dumphash{$_});
+    foreach my $entry (keys %dumphash) {
+	my ($tend,$tstart)=split(/\:/,$dumphash{$entry});
         if (($tstart) && ($tstart<0)) { next; }
         if (($tend) && ($tend<$now)) { next; }
         if (($tstart) && ($now<$tstart)) { next; }
-        my ($role,$username,$domain,$section)=split(/\:/,$_);
+        my ($role,$username,$domain,$section)=split(/\:/,$entry);
 	if ($username eq '' || $domain eq '') { next; }
 	if ((&privileged($username,$domain)) && 
 	    (!$nothide{$username.':'.$domain})) { next; }
@@ -1927,21 +2027,45 @@ sub get_course_adv_roles {
 }
 
 sub get_my_roles {
-    my ($uname,$udom)=@_;
+    my ($uname,$udom,$types,$roles,$roledoms)=@_;
     unless (defined($uname)) { $uname=$env{'user.name'}; }
     unless (defined($udom)) { $udom=$env{'user.domain'}; }
     my %dumphash=
             &dump('nohist_userroles',$udom,$uname);
     my %returnhash=();
     my $now=time;
-    foreach (keys %dumphash) {
-	my ($tend,$tstart)=split(/\:/,$dumphash{$_});
+    foreach my $entry (keys(%dumphash)) {
+	my ($tend,$tstart)=split(/\:/,$dumphash{$entry});
         if (($tstart) && ($tstart<0)) { next; }
-        if (($tend) && ($tend<$now)) { next; }
-        if (($tstart) && ($now<$tstart)) { next; }
-        my ($role,$username,$domain,$section)=split(/\:/,$_);
+        my $status = 'active';
+        if (($tend) && ($tend<$now)) {
+            $status = 'previous';
+        } 
+        if (($tstart) && ($now<$tstart)) {
+            $status = 'future';
+        }
+        if (ref($types) eq 'ARRAY') {
+            if (!grep(/^\Q$status\E$/,@{$types})) {
+                next;
+            } 
+        } else {
+            if ($status ne 'active') {
+                next;
+            }
+        }
+        my ($role,$username,$domain,$section)=split(/\:/,$entry);
+        if (ref($roledoms) eq 'ARRAY') {
+            if (!grep(/^\Q$domain\E$/,@{$roledoms})) {
+                next;
+            }
+        }
+        if (ref($roles) eq 'ARRAY') {
+            if (!grep(/^\Q$role\E$/,@{$roles})) {
+                next;
+            }
+        } 
 	$returnhash{$username.':'.$domain.':'.$role}=$tstart.':'.$tend;
-     }
+    }
     return %returnhash;
 }
 
@@ -1960,7 +2084,7 @@ sub getannounce {
 
     if (open(my $fh,$perlvar{'lonDocRoot'}.'/announcement.txt')) {
 	my $announcement='';
-	while (<$fh>) { $announcement .=$_; }
+	while (my $line = <$fh>) { $announcement .= $line; }
 	close($fh);
 	if ($announcement=~/\w/) { 
 	    return 
@@ -1990,12 +2114,12 @@ sub courseiddump {
     foreach my $tryserver (keys %libserv) {
         if ( ($hostidflag == 1 && grep/^$tryserver$/,@{$hostidref}) || (!defined($hostidflag)) ) {
 	    if ((!$domfilter) || ($hostdom{$tryserver} eq $domfilter)) {
-	        foreach (
+	        foreach my $line (
                  split(/\&/,&reply('courseiddump:'.$hostdom{$tryserver}.':'.
 			       $sincefilter.':'.&escape($descfilter).':'.
                                &escape($instcodefilter).':'.&escape($ownerfilter).':'.&escape($coursefilter).':'.&escape($typefilter).':'.&escape($regexp_ok),
                                $tryserver))) {
-		    my ($key,$value)=split(/\=/,$_);
+		    my ($key,$value)=split(/\=/,$line,2);
                     if (($key) && ($value)) {
 		        $returnhash{&unescape($key)}=$value;
                     }
@@ -2024,8 +2148,8 @@ sub dcmaildump {
                                                          &escape($enddate).':';
 	my @esc_senders=map { &escape($_)} @$senders;
 	$cmd.=&escape(join('&',@esc_senders));
-	foreach (split(/\&/,&reply($cmd,$domain_primary{$dom}))) {
-            my ($key,$value) = split(/\=/,$_);
+	foreach my $line (split(/\&/,&reply($cmd,$domain_primary{$dom}))) {
+            my ($key,$value) = split(/\=/,$line,2);
             if (($key) && ($value)) {
                 $returnhash{&unescape($key)} = &unescape($value);
             }
@@ -2048,11 +2172,11 @@ sub get_domain_roles {
     foreach my $tryserver (keys(%libserv)) {
         if ($hostdom{$tryserver} eq $dom) {
             %{$personnel{$tryserver}}=();
-            foreach (
+            foreach my $line (
                 split(/\&/,&reply('domrolesdump:'.$dom.':'.
                    &escape($startdate).':'.&escape($enddate).':'.
                    &escape($rolelist), $tryserver))) {
-                my($key,$value) = split(/\=/,$_);
+                my ($key,$value) = split(/\=/,$line,2);
                 if (($key) && ($value)) {
                     $personnel{$tryserver}{&unescape($key)} = &unescape($value);
                 }
@@ -2274,27 +2398,27 @@ sub hash2str {
 sub hashref2str {
   my ($hashref)=@_;
   my $result='__HASH_REF__';
-  foreach (sort(keys(%$hashref))) {
-    if (ref($_) eq 'ARRAY') {
-      $result.=&arrayref2str($_).'=';
-    } elsif (ref($_) eq 'HASH') {
-      $result.=&hashref2str($_).'=';
-    } elsif (ref($_)) {
+  foreach my $key (sort(keys(%$hashref))) {
+    if (ref($key) eq 'ARRAY') {
+      $result.=&arrayref2str($key).'=';
+    } elsif (ref($key) eq 'HASH') {
+      $result.=&hashref2str($key).'=';
+    } elsif (ref($key)) {
       $result.='=';
-      #print("Got a ref of ".(ref($_))." skipping.");
+      #print("Got a ref of ".(ref($key))." skipping.");
     } else {
-	if ($_) {$result.=&escape($_).'=';} else { last; }
+	if ($key) {$result.=&escape($key).'=';} else { last; }
     }
 
-    if(ref($hashref->{$_}) eq 'ARRAY') {
-      $result.=&arrayref2str($hashref->{$_}).'&';
-    } elsif(ref($hashref->{$_}) eq 'HASH') {
-      $result.=&hashref2str($hashref->{$_}).'&';
-    } elsif(ref($hashref->{$_})) {
+    if(ref($hashref->{$key}) eq 'ARRAY') {
+      $result.=&arrayref2str($hashref->{$key}).'&';
+    } elsif(ref($hashref->{$key}) eq 'HASH') {
+      $result.=&hashref2str($hashref->{$key}).'&';
+    } elsif(ref($hashref->{$key})) {
        $result.='&';
-      #print("Got a ref of ".(ref($hashref->{$_}))." skipping.");
+      #print("Got a ref of ".(ref($hashref->{$key}))." skipping.");
     } else {
-      $result.=&escape($hashref->{$_}).'&';
+      $result.=&escape($hashref->{$key}).'&';
     }
   }
   $result=~s/\&$//;
@@ -2574,8 +2698,8 @@ sub store {
     $$storehash{'host'}=$perlvar{'lonHostID'};
 
     my $namevalue='';
-    foreach (keys %$storehash) {
-        $namevalue.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+    foreach my $key (keys(%$storehash)) {
+        $namevalue.=&escape($key).'='.&freeze_escape($$storehash{$key}).'&';
     }
     $namevalue=~s/\&$//;
     &courselog($symb.':'.$stuname.':'.$domain.':STORE:'.$namevalue);
@@ -2610,8 +2734,8 @@ sub cstore {
     $$storehash{'host'}=$perlvar{'lonHostID'};
 
     my $namevalue='';
-    foreach (keys %$storehash) {
-        $namevalue.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+    foreach my $key (keys(%$storehash)) {
+        $namevalue.=&escape($key).'='.&freeze_escape($$storehash{$key}).'&';
     }
     $namevalue=~s/\&$//;
     &courselog($symb.':'.$stuname.':'.$domain.':CSTORE:'.$namevalue);
@@ -2643,14 +2767,14 @@ sub restore {
     my $answer=&reply("restore:$domain:$stuname:$namespace:$symb","$home");
 
     my %returnhash=();
-    foreach (split(/\&/,$answer)) {
-	my ($name,$value)=split(/\=/,$_);
+    foreach my $line (split(/\&/,$answer)) {
+	my ($name,$value)=split(/\=/,$line);
         $returnhash{&unescape($name)}=&thaw_unescape($value);
     }
     my $version;
     for ($version=1;$version<=$returnhash{'version'};$version++) {
-       foreach (split(/\:/,$returnhash{$version.':keys'})) {
-          $returnhash{$_}=$returnhash{$version.':'.$_};
+       foreach my $item (split(/\:/,$returnhash{$version.':keys'})) {
+          $returnhash{$item}=$returnhash{$version.':'.$item};
        }
     }
     return %returnhash;
@@ -2690,6 +2814,7 @@ sub coursedescription {
     if (!$args->{'one_time'}) {
 	$envhash{'course.'.$normalid.'.last_cache'}=time;
     }
+
     if ($chome ne 'no_host') {
        %returnhash=&dump('environment',$cdomain,$cnum);
        if (!exists($returnhash{'con_lost'})) {
@@ -2725,9 +2850,9 @@ sub privileged {
     if (($rolesdump eq 'con_lost') || ($rolesdump eq '')) { return 0; }
     my $now=time;
     if ($rolesdump ne '') {
-        foreach (split(/&/,$rolesdump)) {
-	    if ($_!~/^rolesdef_/) {
-		my ($area,$role)=split(/=/,$_);
+        foreach my $entry (split(/&/,$rolesdump)) {
+	    if ($entry!~/^rolesdef_/) {
+		my ($area,$role)=split(/=/,$entry);
 		$area=~s/\_\w\w$//;
 		my ($trole,$tend,$tstart)=split(/_/,$role);
 		if (($trole eq 'dc') || ($trole eq 'su')) {
@@ -2759,14 +2884,14 @@ sub rolesinit {
     my $group_privs;
 
     if ($rolesdump ne '') {
-        foreach (split(/&/,$rolesdump)) {
-	  if ($_!~/^rolesdef_/) {
-            my ($area,$role)=split(/=/,$_);
+        foreach my $entry (split(/&/,$rolesdump)) {
+	  if ($entry!~/^rolesdef_/) {
+            my ($area,$role)=split(/=/,$entry);
 	    $area=~s/\_\w\w$//;
             my ($trole,$tend,$tstart,$group_privs);
 	    if ($role=~/^cr/) { 
-		if ($role=~m|^(cr/\w+/\w+/[a-zA-Z0-9]+)_(.*)$|) {
-		    ($trole,my $trest)=($role=~m|^(cr/\w+/\w+/[a-zA-Z0-9]+)_(.*)$|);
+		if ($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|) {
+		    ($trole,my $trest)=($role=~m|^(cr/$match_domain/$match_username/[a-zA-Z0-9]+)_(.*)$|);
 		    ($tend,$tstart)=split('_',$trest);
 		} else {
 		    $trole=$role;
@@ -2845,7 +2970,7 @@ sub group_roleprivs {
     if (($tend!=0) && ($tend<$now)) { $access = 0; }
     if (($tstart!=0) && ($tstart>$now)) { $access=0; }
     if ($access) {
-        my ($course,$group) = ($area =~ m|(/\w+/\w+)/([^/]+)$|);
+        my ($course,$group) = ($area =~ m|(/$match_domain/$match_courseid)/([^/]+)$|);
         $$allgroups{$course}{$group} .=':'.$group_privs;
     }
 }
@@ -2876,7 +3001,7 @@ sub set_userprivs {
     if (keys(%{$allgroups}) > 0) {
         foreach my $role (keys %{$allroles}) {
             my ($trole,$area,$sec,$extendedarea);
-            if ($role =~ m-^(\w+|cr/\w+/\w+/\w+)\.(/\w+/\w+)(/?\w*)-) {
+            if ($role =~ m-^(\w+|cr/$match_domain/$match_username/\w+)\.(/$match_domain/$match_courseid)(/?\w*)-) {
                 $trole = $1;
                 $area = $2;
                 $sec = $3;
@@ -2891,15 +3016,15 @@ sub set_userprivs {
             }
         }
     }
-    foreach (keys(%grouproles)) {
-        $$allroles{$_} = $grouproles{$_};
+    foreach my $group (keys(%grouproles)) {
+        $$allroles{$group} = $grouproles{$group};
     }
-    foreach (keys %{$allroles}) {
-        my %thesepriv=();
-        if (($_=~/^au/) || ($_=~/^ca/)) { $author=1; }
-        foreach (split(/:/,$$allroles{$_})) {
-            if ($_ ne '') {
-                my ($privilege,$restrictions)=split(/&/,$_);
+    foreach my $role (keys(%{$allroles})) {
+        my %thesepriv;
+        if (($role=~/^au/) || ($role=~/^ca/)) { $author=1; }
+        foreach my $item (split(/:/,$$allroles{$role})) {
+            if ($item ne '') {
+                my ($privilege,$restrictions)=split(/&/,$item);
                 if ($restrictions eq '') {
                     $thesepriv{$privilege}='F';
                 } elsif ($thesepriv{$privilege} ne 'F') {
@@ -2909,8 +3034,10 @@ sub set_userprivs {
             }
         }
         my $thesestr='';
-        foreach (keys %thesepriv) { $thesestr.=':'.$_.'&'.$thesepriv{$_}; }
-        $userroles->{'user.priv.'.$_} = $thesestr;
+        foreach my $priv (keys(%thesepriv)) {
+	    $thesestr.=':'.$priv.'&'.$thesepriv{$priv};
+	}
+        $userroles->{'user.priv.'.$role} = $thesestr;
     }
     return ($author,$adv);
 }
@@ -2920,8 +3047,8 @@ sub set_userprivs {
 sub get {
    my ($namespace,$storearr,$udomain,$uname)=@_;
    my $items='';
-   foreach (@$storearr) {
-       $items.=escape($_).'&';
+   foreach my $item (@$storearr) {
+       $items.=&escape($item).'&';
    }
    $items=~s/\&$//;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
@@ -2935,8 +3062,8 @@ sub get {
    }
    my %returnhash=();
    my $i=0;
-   foreach (@$storearr) {
-      $returnhash{$_}=&thaw_unescape($pairs[$i]);
+   foreach my $item (@$storearr) {
+      $returnhash{$item}=&thaw_unescape($pairs[$i]);
       $i++;
    }
    return %returnhash;
@@ -2947,8 +3074,8 @@ sub get {
 sub del {
    my ($namespace,$storearr,$udomain,$uname)=@_;
    my $items='';
-   foreach (@$storearr) {
-       $items.=escape($_).'&';
+   foreach my $item (@$storearr) {
+       $items.=&escape($item).'&';
    }
    $items=~s/\&$//;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
@@ -2986,7 +3113,23 @@ sub dump {
 
 sub dumpstore {
    my ($namespace,$udomain,$uname,$regexp,$range)=@_;
-   return &dump($namespace,$udomain,$uname,$regexp,$range);
+   if (!$udomain) { $udomain=$env{'user.domain'}; }
+   if (!$uname) { $uname=$env{'user.name'}; }
+   my $uhome=&homeserver($uname,$udomain);
+   if ($regexp) {
+       $regexp=&escape($regexp);
+   } else {
+       $regexp='.';
+   }
+   my $rep=&reply("dump:$udomain:$uname:$namespace:$regexp:$range",$uhome);
+   my @pairs=split(/\&/,$rep);
+   my %returnhash=();
+   foreach my $item (@pairs) {
+       my ($key,$value)=split(/=/,$item,2);
+       next if ($key =~ /^error: 2 /);
+       $returnhash{$key}=&thaw_unescape($value);
+   }
+   return %returnhash;
 }
 
 # -------------------------------------------------------------- keys interface
@@ -2998,8 +3141,9 @@ sub getkeys {
    my $uhome=&homeserver($uname,$udomain);
    my $rep=reply("keys:$udomain:$uname:$namespace",$uhome);
    my @keyarray=();
-   foreach (split(/\&/,$rep)) {
-      push (@keyarray,&unescape($_));
+   foreach my $key (split(/\&/,$rep)) {
+      next if ($key =~ /^error: 2 /);
+      push(@keyarray,&unescape($key));
    }
    return @keyarray;
 }
@@ -3019,15 +3163,15 @@ sub currentdump {
    if ($rep eq "unknown_cmd") { 
        # an old lond will not know currentdump
        # Do a dump and make it look like a currentdump
-       my @tmp = &dump($courseid,$sdom,$sname,'.');
+       my @tmp = &dumpstore($courseid,$sdom,$sname,'.');
        return if ($tmp[0] =~ /^(error:|no_such_host)/);
        my %hash = @tmp;
        @tmp=();
        %returnhash = %{&convert_dump_to_currentdump(\%hash)};
    } else {
        my @pairs=split(/\&/,$rep);
-       foreach (@pairs) {
-           my ($key,$value)=split(/=/,$_);
+       foreach my $pair (@pairs) {
+           my ($key,$value)=split(/=/,$pair,2);
            my ($symb,$param) = split(/:/,$key);
            $returnhash{&unescape($symb)}->{&unescape($param)} = 
                                                         &thaw_unescape($value);
@@ -3044,6 +3188,8 @@ sub convert_dump_to_currentdump{
     # we might run in to problems with parameter names =~ /^v\./
     while (my ($key,$value) = each(%hash)) {
         my ($v,$symb,$param) = split(/:/,$key);
+	$symb  = &unescape($symb);
+	$param = &unescape($param);
         next if ($v eq 'version' || $symb eq 'keys');
         next if (exists($returnhash{$symb}) &&
                  exists($returnhash{$symb}->{$param}) &&
@@ -3105,8 +3251,8 @@ sub put {
    if (!$uname) { $uname=$env{'user.name'}; }
    my $uhome=&homeserver($uname,$udomain);
    my $items='';
-   foreach (keys %$storehash) {
-       $items.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+   foreach my $item (keys(%$storehash)) {
+       $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
    }
    $items=~s/\&$//;
    return &reply("put:$udomain:$uname:$namespace:$items",$uhome);
@@ -3158,22 +3304,22 @@ sub old_putstore {
     if (!$uname) { $uname=$env{'user.name'}; }
     my $uhome=&homeserver($uname,$udomain);
     my %newstorehash;
-    foreach (keys %$storehash) {
-	my $key = $version.':'.&escape($symb).':'.$_;
-	$newstorehash{$key} = $storehash->{$_};
+    foreach my $item (keys(%$storehash)) {
+	my $key = $version.':'.&escape($symb).':'.$item;
+	$newstorehash{$key} = $storehash->{$item};
     }
     my $items='';
     my %allitems = ();
-    foreach (keys %newstorehash) {
-	if ($_ =~ m/^([^\:]+):([^\:]+):([^\:]+)$/) {
+    foreach my $item (keys(%newstorehash)) {
+	if ($item =~ m/^([^\:]+):([^\:]+):([^\:]+)$/) {
 	    my $key = $1.':keys:'.$2;
 	    $allitems{$key} .= $3.':';
 	}
-	$items.=$_.'='.&freeze_escape($newstorehash{$_}).'&';
+	$items.=$item.'='.&freeze_escape($newstorehash{$item}).'&';
     }
-    foreach (keys %allitems) {
-	$allitems{$_} =~ s/\:$//;
-	$items.= $_.'='.$allitems{$_}.'&';
+    foreach my $item (keys(%allitems)) {
+	$allitems{$item} =~ s/\:$//;
+	$items.= $item.'='.$allitems{$item}.'&';
     }
     $items=~s/\&$//;
     return &reply("put:$udomain:$uname:$namespace:$items",$uhome);
@@ -3187,8 +3333,8 @@ sub cput {
    if (!$uname) { $uname=$env{'user.name'}; }
    my $uhome=&homeserver($uname,$udomain);
    my $items='';
-   foreach (keys %$storehash) {
-       $items.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+   foreach my $item (keys(%$storehash)) {
+       $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
    }
    $items=~s/\&$//;
    return &critical("put:$udomain:$uname:$namespace:$items",$uhome);
@@ -3199,8 +3345,8 @@ sub cput {
 sub eget {
    my ($namespace,$storearr,$udomain,$uname)=@_;
    my $items='';
-   foreach (@$storearr) {
-       $items.=escape($_).'&';
+   foreach my $item (@$storearr) {
+       $items.=&escape($item).'&';
    }
    $items=~s/\&$//;
    if (!$udomain) { $udomain=$env{'user.domain'}; }
@@ -3210,8 +3356,8 @@ sub eget {
    my @pairs=split(/\&/,$rep);
    my %returnhash=();
    my $i=0;
-   foreach (@$storearr) {
-      $returnhash{$_}=&thaw_unescape($pairs[$i]);
+   foreach my $item (@$storearr) {
+      $returnhash{$item}=&thaw_unescape($pairs[$i]);
       $i++;
    }
    return %returnhash;
@@ -3219,12 +3365,15 @@ sub eget {
 
 # ------------------------------------------------------------ tmpput interface
 sub tmpput {
-    my ($storehash,$server)=@_;
+    my ($storehash,$server,$context)=@_;
     my $items='';
-    foreach (keys(%$storehash)) {
-	$items.=&escape($_).'='.&freeze_escape($$storehash{$_}).'&';
+    foreach my $item (keys(%$storehash)) {
+	$items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
     }
     $items=~s/\&$//;
+    if (defined($context)) {
+        $items .= ':'.&escape($context);
+    }
     return &reply("tmpput:$items",$server);
 }
 
@@ -3254,6 +3403,22 @@ sub portfolio_access {
     my ($requrl) = @_;
     my (undef,$udom,$unum,$file_name,$group) = &parse_portfolio_url($requrl);
     my $result = &get_portfolio_access($udom,$unum,$file_name,$group);
+    if ($result) {
+        my %setters;
+        if ($env{'user.name'} eq 'public' && $env{'user.domain'} eq 'public') {
+            my ($startblock,$endblock) =
+                &Apache::loncommon::blockcheck(\%setters,'port',$unum,$udom);
+            if ($startblock && $endblock) {
+                return 'B';
+            }
+        } else {
+            my ($startblock,$endblock) =
+                &Apache::loncommon::blockcheck(\%setters,'port');
+            if ($startblock && $endblock) {
+                return 'B';
+            }
+        }
+    }
     if ($result eq 'ok') {
        return 'F';
     } elsif ($result =~ /^[^:]+:guest_/) {
@@ -3329,7 +3494,7 @@ sub get_portfolio_access {
                 my (%allgroups,%allroles); 
                 my ($start,$end,$role,$sec,$group);
                 foreach my $envkey (%env) {
-                    if ($envkey =~ m-^user\.role\.(gr|cc|in|ta|ep|st)\./([^/]+)/([^/]+)/?([^/]*)$-) {
+                    if ($envkey =~ m-^user\.role\.(gr|cc|in|ta|ep|st)\./($match_domain)/($match_courseid)/?([^/]*)$-) {
                         my $cid = $2.'_'.$3; 
                         if ($1 eq 'gr') {
                             $group = $4;
@@ -3342,7 +3507,7 @@ sub get_portfolio_access {
                             }
                             $allroles{$cid}{$1}{$sec} = $env{$envkey};
                         }
-                    } elsif ($envkey =~ m-^user\.role\./cr/(\w+/\w+/\w*)./([^/]+)/([^/]+)/?([^/]*)$-) {
+                    } elsif ($envkey =~ m-^user\.role\./cr/($match_domain/$match_username/\w*)./($match_domain)/($match_courseid)/?([^/]*)$-) {
                         my $cid = $2.'_'.$3;
                         if ($4 eq '') {
                             $sec = 'none';
@@ -3437,12 +3602,12 @@ sub parse_portfolio_url {
 
     my ($type,$udom,$unum,$group,$file_name);
     
-    if ($url =~  m-^/*uploaded/([^/]+)/([^/]+)/portfolio(/.+)$-) {
+    if ($url =~  m-^/*(?:uploaded|editupload)/($match_domain)/($match_username)/portfolio(/.+)$-) {
 	$type = 1;
         $udom = $1;
         $unum = $2;
         $file_name = $3;
-    } elsif ($url =~ m-^/*uploaded/([^/]+)/([^/]+)/groups/([^/]+)/portfolio/(.+)$-) {
+    } elsif ($url =~ m-^/*(?:uploaded|editupload)/($match_domain)/($match_courseid)/groups/([^/]+)/portfolio/(.+)$-) {
 	$type = 2;
         $udom = $1;
         $unum = $2;
@@ -3473,17 +3638,18 @@ sub is_portfolio_file {
 
 sub customaccess {
     my ($priv,$uri)=@_;
-    my ($urole,$urealm)=split(/\./,$env{'request.role'});
-    $urealm=~s/^\W//;
-    my ($udom,$ucrs,$usec)=split(/\//,$urealm);
+    my ($urole,$urealm)=split(/\./,$env{'request.role'},2);
+    my (undef,$udom,$ucrs,$usec)=split(/\//,$urealm);
+    $udom = &LONCAPA::clean_domain($udom);
+    $ucrs = &LONCAPA::clean_username($ucrs);
     my $access=0;
-    foreach (split(/\s*\,\s*/,&metadata($uri,'rule_rights'))) {
-	my ($effect,$realm,$role)=split(/\:/,$_);
+    foreach my $right (split(/\s*\,\s*/,&metadata($uri,'rule_rights'))) {
+	my ($effect,$realm,$role)=split(/\:/,$right);
         if ($role) {
 	   if ($role ne $urole) { next; }
         }
-        foreach (split(/\s*\,\s*/,$realm)) {
-            my ($tdom,$tcrs,$tsec)=split(/\_/,$_);
+        foreach my $scope (split(/\s*\,\s*/,$realm)) {
+            my ($tdom,$tcrs,$tsec)=split(/\_/,$scope);
             if ($tdom) {
 		if ($tdom ne $udom) { next; }
             }
@@ -3506,12 +3672,21 @@ sub customaccess {
 # ------------------------------------------------- Check for a user privilege
 
 sub allowed {
-    my ($priv,$uri,$symb)=@_;
+    my ($priv,$uri,$symb,$role)=@_;
     my $ver_orguri=$uri;
     $uri=&deversion($uri);
     my $orguri=$uri;
     $uri=&declutter($uri);
-    
+
+    if ($priv eq 'evb') {
+# Evade communication block restrictions for specified role in a course
+        if ($env{'user.priv.'.$role} =~/evb\&([^\:]*)/) {
+            return $1;
+        } else {
+            return;
+        }
+    }
+
     if (defined($env{'allowed.'.$priv})) { return $env{'allowed.'.$priv}; }
 # Free bre access to adm and meta resources
     if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard)$})) 
@@ -3524,7 +3699,14 @@ sub allowed {
     my ($space,$domain,$name,@dir)=split('/',$uri);
     if (($space=~/^(uploaded|editupload)$/) && ($env{'user.name'} eq $name) && 
 	($env{'user.domain'} eq $domain) && ('portfolio' eq $dir[0])) {
-        return 'F';
+        my %setters;
+        my ($startblock,$endblock) = 
+            &Apache::loncommon::blockcheck(\%setters,'port');
+        if ($startblock && $endblock) {
+            return 'B';
+        } else {
+            return 'F';
+        }
     }
 
 # bre access to group portfolio for rgf priv in group, or mdg or vcg in course.
@@ -3695,14 +3877,14 @@ sub allowed {
        if ($checkreferer) {
 	  my $refuri=$env{'httpref.'.$orguri};
             unless ($refuri) {
-                foreach (keys %env) {
-		    if ($_=~/^httpref\..*\*/) {
-			my $pattern=$_;
+                foreach my $key (keys(%env)) {
+		    if ($key=~/^httpref\..*\*/) {
+			my $pattern=$key;
                         $pattern=~s/^httpref\.\/res\///;
                         $pattern=~s/\*/\[\^\/\]\+/g;
                         $pattern=~s/\//\\\//g;
                         if ($orguri=~/$pattern/) {
-			    $refuri=$env{$_};
+			    $refuri=$env{$key};
                         }
                     }
                 }
@@ -3800,6 +3982,8 @@ sub allowed {
     unless ($env{'request.course.id'}) {
 	if ($thisallowed eq 'A') {
 	    return 'A';
+        } elsif ($thisallowed eq 'B') {
+            return 'B';
 	} else {
 	    return '1';
 	}
@@ -3867,6 +4051,8 @@ sub allowed {
 
     if ($thisallowed eq 'A') {
 	return 'A';
+    } elsif ($thisallowed eq 'B') {
+        return 'B';
     }
    return 'F';
 }
@@ -3920,8 +4106,8 @@ sub get_symb_from_alias {
 sub definerole {
   if (allowed('mcr','/')) {
     my ($rolename,$sysrole,$domrole,$courole)=@_;
-    foreach (split(':',$sysrole)) {
-	my ($crole,$cqual)=split(/\&/,$_);
+    foreach my $role (split(':',$sysrole)) {
+	my ($crole,$cqual)=split(/\&/,$role);
         if ($pr{'cr:s'}!~/\Q$crole\E/) { return "refused:s:$crole"; }
         if ($pr{'cr:s'}=~/\Q$crole\E\&/) {
 	    if ($pr{'cr:s'}!~/\Q$crole\E\&\w*\Q$cqual\E/) { 
@@ -3929,8 +4115,8 @@ sub definerole {
             }
         }
     }
-    foreach (split(':',$domrole)) {
-	my ($crole,$cqual)=split(/\&/,$_);
+    foreach my $role (split(':',$domrole)) {
+	my ($crole,$cqual)=split(/\&/,$role);
         if ($pr{'cr:d'}!~/\Q$crole\E/) { return "refused:d:$crole"; }
         if ($pr{'cr:d'}=~/\Q$crole\E\&/) {
 	    if ($pr{'cr:d'}!~/\Q$crole\W\&\w*\Q$cqual\E/) { 
@@ -3938,8 +4124,8 @@ sub definerole {
             }
         }
     }
-    foreach (split(':',$courole)) {
-	my ($crole,$cqual)=split(/\&/,$_);
+    foreach my $role (split(':',$courole)) {
+	my ($crole,$cqual)=split(/\&/,$role);
         if ($pr{'cr:c'}!~/\Q$crole\E/) { return "refused:c:$crole"; }
         if ($pr{'cr:c'}=~/\Q$crole\E\&/) {
 	    if ($pr{'cr:c'}!~/\Q$crole\E\&\w*\Q$cqual\E/) { 
@@ -3986,13 +4172,25 @@ sub log_query {
     my $uhome=&homeserver($uname,$udom);
     if ($uhome eq 'no_host') { return 'error: no_host'; }
     my $uhost=$hostname{$uhome};
-    my $command=&escape(join(':',map{$_.'='.$filters{$_}} keys %filters));
+    my $command=&escape(join(':',map{$_.'='.$filters{$_}} keys(%filters)));
     my $queryid=&reply("querysend:".$query.':'.$udom.':'.$uname.':'.$command,
                        $uhome);
     unless ($queryid=~/^\Q$uhost\E\_/) { return 'error: '.$queryid; }
     return get_query_reply($queryid);
 }
 
+# -------------------------- Update MySQL table for portfolio file
+
+sub update_portfolio_table {
+    my ($uname,$udom,$file_name,$query,$group,$action) = @_;
+    my $homeserver = &homeserver($uname,$udom);
+    my $queryid=
+        &reply("querysend:".$query.':'.&escape($uname.':'.$udom.':'.$group).
+               ':'.&escape($file_name).':'.$action,$homeserver);
+    my $reply = &get_query_reply($queryid);
+    return $reply;
+}
+
 # ------- Request retrieval of institutional classlists for course(s)
 
 sub fetch_enrollment_query {
@@ -4007,8 +4205,8 @@ sub fetch_enrollment_query {
     }
     my $host=$hostname{$homeserver};
     my $cmd = '';
-    foreach (keys %{$affiliatesref}) {
-        $cmd .= $_.'='.join(",",@{$$affiliatesref{$_}}).'%%';
+    foreach my $affiliate (keys %{$affiliatesref}) {
+        $cmd .= $affiliate.'='.join(",",@{$$affiliatesref{$affiliate}}).'%%';
     }
     $cmd =~ s/%%$//;
     $cmd = &escape($cmd);
@@ -4029,18 +4227,18 @@ sub fetch_enrollment_query {
     } else {
         my @responses = split/:/,$reply;
         if ($homeserver eq $perlvar{'lonHostID'}) {
-            foreach (@responses) {
-                my ($key,$value) = split/=/,$_;
+            foreach my $line (@responses) {
+                my ($key,$value) = split(/=/,$line,2);
                 $$replyref{$key} = $value;
             }
         } else {
             my $pathname = $perlvar{'lonDaemons'}.'/tmp';
-            foreach (@responses) {
-                my ($key,$value) = split/=/,$_;
+            foreach my $line (@responses) {
+                my ($key,$value) = split(/=/,$line);
                 $$replyref{$key} = $value;
                 if ($value > 0) {
-                    foreach (@{$$affiliatesref{$key}}) {
-                        my $filename = $dom.'_'.$key.'_'.$_.'_classlist.xml';
+                    foreach my $item (@{$$affiliatesref{$key}}) {
+                        my $filename = $dom.'_'.$key.'_'.$item.'_classlist.xml';
                         my $destname = $pathname.'/'.$filename;
                         my $xml_classlist = &reply("autoretrieve:".$filename,$homeserver);
                         if ($xml_classlist =~ /^error/) {
@@ -4199,8 +4397,8 @@ sub auto_photoupdate {
     my $host=$hostname{$homeserver};
     my $cmd = '';
     my $maxtries = 1;
-    foreach (keys %{$affiliatesref}) {
-        $cmd .= $_.'='.join(",",@{$$affiliatesref{$_}}).'%%';
+    foreach my $affiliate (keys(%{$affiliatesref})) {
+        $cmd .= $affiliate.'='.join(",",@{$$affiliatesref{$affiliate}}).'%%';
     }
     $cmd =~ s/%%$//;
     $cmd = &escape($cmd);
@@ -4296,8 +4494,8 @@ sub auto_instcode_defaults {
                     $returnhash->{&unescape($name)}=&unescape($value);
                 }
             }
+            $ok_response = 1;
         }
-        $ok_response = 1;
     }
     if ($ok_response) {
         return 'ok';
@@ -4317,8 +4515,8 @@ sub auto_validate_class_sec {
 # ------------------------------------------------------- Course Group routines
 
 sub get_coursegroups {
-    my ($cdom,$cnum,$group) = @_;
-    return(&dump('coursegroups',$cdom,$cnum,$group));
+    my ($cdom,$cnum,$group,$namespace) = @_;
+    return(&dump($namespace,$cdom,$cnum,$group));
 }
 
 sub modify_coursegroup {
@@ -4326,6 +4524,37 @@ sub modify_coursegroup {
     return(&put('coursegroups',$groupsettings,$cdom,$cnum));
 }
 
+sub toggle_coursegroup_status {
+    my ($cdom,$cnum,$group,$action) = @_;
+    my ($from_namespace,$to_namespace);
+    if ($action eq 'delete') {
+        $from_namespace = 'coursegroups';
+        $to_namespace = 'deleted_groups';
+    } else {
+        $from_namespace = 'deleted_groups';
+        $to_namespace = 'coursegroups';
+    }
+    my %curr_group = &get_coursegroups($cdom,$cnum,$group,$from_namespace);
+    if (my $tmp = &error(%curr_group)) {
+        &Apache::lonnet::logthis('Error retrieving group: '.$tmp.' in '.$cnum.':'.$cdom);
+        return ('read error',$tmp);
+    } else {
+        my %savedsettings = %curr_group; 
+        my $result = &put($to_namespace,\%savedsettings,$cdom,$cnum);
+        my $deloutcome;
+        if ($result eq 'ok') {
+            $deloutcome = &del($from_namespace,[$group],$cdom,$cnum);
+        } else {
+            return ('write error',$result);
+        }
+        if ($deloutcome eq 'ok') {
+            return 'ok';
+        } else {
+            return ('delete error',$deloutcome);
+        }
+    }
+}
+
 sub modify_group_roles {
     my ($cdom,$cnum,$group_id,$user,$end,$start,$userprivs) = @_;
     my $url = '/'.$cdom.'/'.$cnum.'/'.$group_id;
@@ -4349,7 +4578,7 @@ sub get_active_groups {
     my $now = time;
     my %groups = ();
     foreach my $key (keys(%env)) {
-        if ($key =~ m-user\.role\.gr\./([^/]+)/([^/]+)/(\w+)$-) {
+        if ($key =~ m-user\.role\.gr\./($match_domain)/($match_courseid)/(\w+)$-) {
             my ($start,$end) = split(/\./,$env{$key});
             if (($end!=0) && ($end<$now)) { next; }
             if (($start!=0) && ($start>$now)) { next; }
@@ -4370,8 +4599,6 @@ sub get_users_groups {
     my ($udom,$uname,$courseid) = @_;
     my @usersgroups;
     my $cachetime=1800;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
 
     my $hashid="$udom:$uname:$courseid";
     my ($grouplist,$cached)=&is_cached_new('getgroups',$hashid);
@@ -4379,38 +4606,34 @@ sub get_users_groups {
         @usersgroups = split(/:/,$grouplist);
     } else {  
         $grouplist = '';
-        my %roleshash = &dump('roles',$udom,$uname,$courseid);
-        my ($tmp) = keys(%roleshash);
-        if ($tmp=~/^error:/) {
-            &logthis('Error retrieving roles: '.$tmp.' for '.$uname.':'.$udom);
-        } else {
-            my $access_end = $env{'course.'.$courseid.
-                                  '.default_enrollment_end_date'};
-            my $now = time;
-            foreach my $key (keys(%roleshash)) {
-                if ($key =~ /^\Q$courseid\E\/(\w+)\_gr$/) {
-                    my $group = $1;
-                    if ($roleshash{$key} =~ /_(\d+)_(\d+)$/) {
-                        my $start = $2;
-                        my $end = $1;
-                        if ($start == -1) { next; } # deleted from group
-                        if (($start!=0) && ($start>$now)) { next; }
-                        if (($end!=0) && ($end<$now)) {
-                            if ($access_end && $access_end < $now) {
-                                if ($access_end - $end < 86400) {
-                                    push(@usersgroups,$group);
-                                }
+        my $courseurl = &courseid_to_courseurl($courseid);
+        my %roleshash = &dump('roles',$udom,$uname,$courseurl);
+        my $access_end = $env{'course.'.$courseid.
+                              '.default_enrollment_end_date'};
+        my $now = time;
+        foreach my $key (keys(%roleshash)) {
+            if ($key =~ /^\Q$courseurl\E\/(\w+)\_gr$/) {
+                my $group = $1;
+                if ($roleshash{$key} =~ /_(\d+)_(\d+)$/) {
+                    my $start = $2;
+                    my $end = $1;
+                    if ($start == -1) { next; } # deleted from group
+                    if (($start!=0) && ($start>$now)) { next; }
+                    if (($end!=0) && ($end<$now)) {
+                        if ($access_end && $access_end < $now) {
+                            if ($access_end - $end < 86400) {
+                                push(@usersgroups,$group);
                             }
-                            next;
                         }
-                        push(@usersgroups,$group);
+                        next;
                     }
+                    push(@usersgroups,$group);
                 }
             }
-            @usersgroups = &sort_course_groups($courseid,@usersgroups);
-            $grouplist = join(':',@usersgroups);
-            &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
         }
+        @usersgroups = &sort_course_groups($courseid,@usersgroups);
+        $grouplist = join(':',@usersgroups);
+        &do_cache_new('getgroups',$hashid,$grouplist,$cachetime);
     }
     return @usersgroups;
 }
@@ -4418,8 +4641,7 @@ sub get_users_groups {
 sub devalidate_getgroups_cache {
     my ($udom,$uname,$cdom,$cnum)=@_;
     my $courseid = $cdom.'_'.$cnum;
-    $courseid=~s/\_/\//g;
-    $courseid=~s/^(\w)/\/$1/;
+
     my $hashid="$udom:$uname:$courseid";
     &devalidate_cache_new('getgroups',$hashid);
 }
@@ -4458,7 +4680,7 @@ sub assignrole {
     my $mrole;
     if ($role =~ /^cr\//) {
         my $cwosec=$url;
-        $cwosec=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwosec=~s/^\/($match_domain)\/($match_courseid)\/.*/$1\/$2/;
 	unless (&allowed('ccr',$cwosec)) {
            &logthis('Refused custom assignrole: '.
              $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4468,7 +4690,7 @@ sub assignrole {
         $mrole='cr';
     } elsif ($role =~ /^gr\//) {
         my $cwogrp=$url;
-        $cwogrp=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwogrp=~s{^/($match_domain)/($match_courseid)/.*}{$1/$2};
         unless (&allowed('mdg',$cwogrp)) {
             &logthis('Refused group assignrole: '.
               $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4478,7 +4700,7 @@ sub assignrole {
         $mrole='gr';
     } else {
         my $cwosec=$url;
-        $cwosec=~s/^\/(\w+)\/(\w+)\/.*/$1\/$2/;
+        $cwosec=~s/^\/($match_domain)\/($match_courseid)\/.*/$1\/$2/;
         unless ((&allowed('c'.$role,$cwosec)) || &allowed('c'.$role,$udom)) { 
            &logthis('Refused assignrole: '.
              $udom.' '.$uname.' '.$url.' '.$role.' '.$end.' '.$start.' by '.
@@ -4558,8 +4780,8 @@ sub modifyuser {
         $umode,   $upass, $first,
         $middle,  $last,  $gene,
         $forceid, $desiredhome, $email)=@_;
-    $udom=~s/\W//g;
-    $uname=~s/\W//g;
+    $udom= &LONCAPA::clean_domain($udom);
+    $uname=&LONCAPA::clean_username($uname);
     &logthis('Call to modify user '.$udom.', '.$uname.', '.$uid.', '.
              $umode.', '.$first.', '.$middle.', '.
 	     $last.', '.$gene.'(forceid: '.$forceid.')'.
@@ -4708,8 +4930,8 @@ sub modify_student_enrollment {
                        ['firstname','middlename','lastname', 'generation','id']
                        ,$udom,$uname);
 
-        #foreach (keys(%tmp)) {
-        #    &logthis("key $_ = ".$tmp{$_});
+        #foreach my $key (keys(%tmp)) {
+        #    &logthis("key $key = ".$tmp{$key});
         #}
         $first  = $tmp{'firstname'}  if (!defined($first)  || $first  eq '');
         $middle = $tmp{'middlename'} if (!defined($middle) || $middle eq '');
@@ -4767,8 +4989,8 @@ sub writecoursepref {
 	return 'error: no such course';
     }
     my $cstring='';
-    foreach (keys %prefs) {
-	$cstring.=escape($_).'='.escape($prefs{$_}).'&';
+    foreach my $pref (keys(%prefs)) {
+	$cstring.=&escape($pref).'='.&escape($prefs{$pref}).'&';
     }
     $cstring=~s/\&$//;
     return reply('put:'.$cdomain.':'.$cnum.':environment:'.$cstring,$chome);
@@ -4844,6 +5066,16 @@ ENDINITMAP
     return '/'.$udom.'/'.$uname;
 }
 
+sub is_course {
+    my ($cdom,$cnum) = @_;
+    my %courses = &courseiddump($cdom,'.',1,'.','.',$cnum,undef,
+				undef,'.');
+    if (exists($courses{$cdom.'_'.$cnum})) {
+        return 1;
+    }
+    return 0;
+}
+
 # ---------------------------------------------------------- Assign Custom Role
 
 sub assigncustomrole {
@@ -4975,20 +5207,20 @@ sub files_not_in_path {
     my $filename = $user."savedfiles";
     my @return_files;
     my $path_part;
-    open (IN, '<'.$Apache::lonnet::perlvar{'lonDaemons'}.'/tmp/'.$filename);
-    while (<IN>) {
+    open(IN, '<'.$Apache::lonnet::perlvar{'lonDaemons'}.'/tmp/'.$filename);
+    while (my $line = <IN>) {
         #ok, I know it's clunky, but I want it to work
-        my @paths_and_file = split m!/!, $_;
-        my $file_part = pop (@paths_and_file);
-        chomp ($file_part);
-        my $path_part = join ('/', @paths_and_file);
+        my @paths_and_file = split(m|/|, $line);
+        my $file_part = pop(@paths_and_file);
+        chomp($file_part);
+        my $path_part = join('/', @paths_and_file);
         $path_part .= '/';
         my $path_and_file = $path_part.$file_part;
         if ($path_part ne $path) {
-            push (@return_files, ($path_and_file));
+            push(@return_files, ($path_and_file));
         }
     }
-    close (OUT);
+    close(OUT);
     return (@return_files);
 }
 
@@ -5122,12 +5354,68 @@ sub modify_access_controls {
         #  remove lock
         my @del_lock = ($file_name."\0".'locked_access_records');
         my $dellockoutcome = &del('file_permissions',\@del_lock,$domain,$user);
+        my ($file,$group);
+        if (&is_course($domain,$user)) {
+            ($group,$file) = split(/\//,$file_name,2);
+        } else {
+            $file = $file_name;
+        }
+        my $sqlresult =
+            &update_portfolio_table($user,$domain,$file,'portfolio_access',
+                                    $group);
     } else {
         $outcome = "error: could not obtain lockfile\n";  
     }
     return ($outcome,$deloutcome,\%new_values,\%translation);
 }
 
+sub make_public_indefinitely {
+    my ($requrl) = @_;
+    my $now = time;
+    my $action = 'activate';
+    my $aclnum = 0;
+    if (&is_portfolio_url($requrl)) {
+        my (undef,$udom,$unum,$file_name,$group) =
+            &parse_portfolio_url($requrl);
+        my $current_perms = &get_portfile_permissions($udom,$unum);
+        my %access_controls = &get_access_controls($current_perms,
+                                                   $group,$file_name);
+        foreach my $key (keys(%{$access_controls{$file_name}})) {
+            my ($num,$scope,$end,$start) = 
+                ($key =~ /^([^:]+):([a-z]+)_(\d*)_?(\d*)$/);
+            if ($scope eq 'public') {
+                if ($start <= $now && $end == 0) {
+                    $action = 'none';
+                } else {
+                    $action = 'update';
+                    $aclnum = $num;
+                }
+                last;
+            }
+        }
+        if ($action eq 'none') {
+             return 'ok';
+        } else {
+            my %changes;
+            my $newend = 0;
+            my $newstart = $now;
+            my $newkey = $aclnum.':public_'.$newend.'_'.$newstart;
+            $changes{$action}{$newkey} = {
+                type => 'public',
+                time => {
+                    start => $newstart,
+                    end   => $newend,
+                },
+            };
+            my ($outcome,$deloutcome,$new_values,$translation) =
+                &modify_access_controls($file_name,\%changes,$udom,$unum);
+            return $outcome;
+        }
+    } else {
+        return 'invalid';
+    }
+}
+
 #------------------------------------------------------Get Marked as Read Only
 
 sub get_marked_as_readonly {
@@ -5257,28 +5545,27 @@ sub dirlist {
 
     if($udom) {
         if($uname) {
-            my $listing=reply('ls2:'.$dirRoot.'/'.$uri,
-                              homeserver($uname,$udom));
+            my $listing = &reply('ls2:'.$dirRoot.'/'.$uri,
+				 &homeserver($uname,$udom));
             my @listing_results;
             if ($listing eq 'unknown_cmd') {
-                $listing=reply('ls:'.$dirRoot.'/'.$uri,
-                               homeserver($uname,$udom));
+                $listing = &reply('ls:'.$dirRoot.'/'.$uri,
+				  &homeserver($uname,$udom));
                 @listing_results = split(/:/,$listing);
             } else {
                 @listing_results = map { &unescape($_); } split(/:/,$listing);
             }
             return @listing_results;
         } elsif(!defined($alternateDirectoryRoot)) {
-            my $tryserver;
-            my %allusers=();
-            foreach $tryserver (keys %libserv) {
+            my %allusers;
+            foreach my $tryserver (keys(%libserv)) {
                 if($hostdom{$tryserver} eq $udom) {
-                    my $listing=reply('ls2:'.$perlvar{'lonDocRoot'}.'/res/'.
-                                      $udom, $tryserver);
+                    my $listing = &reply('ls2:'.$perlvar{'lonDocRoot'}.'/res/'.
+					 $udom, $tryserver);
                     my @listing_results;
                     if ($listing eq 'unknown_cmd') {
-                        $listing=reply('ls:'.$perlvar{'lonDocRoot'}.'/res/'.
-                                       $udom, $tryserver);
+                        $listing = &reply('ls:'.$perlvar{'lonDocRoot'}.'/res/'.
+					  $udom, $tryserver);
                         @listing_results = split(/:/,$listing);
                     } else {
                         @listing_results =
@@ -5287,40 +5574,36 @@ sub dirlist {
                     if ($listing_results[0] ne 'no_such_dir' && 
                         $listing_results[0] ne 'empty'       &&
                         $listing_results[0] ne 'con_lost') {
-                        foreach (@listing_results) {
-                            my ($entry,@stat)=split(/&/,$_);
-                            $allusers{$entry}=1;
+                        foreach my $line (@listing_results) {
+                            my ($entry) = split(/&/,$line,2);
+                            $allusers{$entry} = 1;
                         }
                     }
                 }
             }
             my $alluserstr='';
-            foreach (sort keys %allusers) {
-                $alluserstr.=$_.'&user:';
+            foreach my $user (sort(keys(%allusers))) {
+                $alluserstr.=$user.'&user:';
             }
             $alluserstr=~s/:$//;
             return split(/:/,$alluserstr);
         } else {
-            my @emptyResults = ();
-            push(@emptyResults, 'missing user name');
-            return split(':',@emptyResults);
+            return ('missing user name');
         }
     } elsif(!defined($alternateDirectoryRoot)) {
         my $tryserver;
         my %alldom=();
-        foreach $tryserver (keys %libserv) {
+        foreach $tryserver (keys(%libserv)) {
             $alldom{$hostdom{$tryserver}}=1;
         }
         my $alldomstr='';
-        foreach (sort keys %alldom) {
-            $alldomstr.=$perlvar{'lonDocRoot'}.'/res/'.$_.'/&domain:';
+        foreach my $domain (sort(keys(%alldom))) {
+            $alldomstr.=$perlvar{'lonDocRoot'}.'/res/'.$domain.'/&domain:';
         }
         $alldomstr=~s/:$//;
         return split(/:/,$alldomstr);       
     } else {
-        my @emptyResults = ();
-        push(@emptyResults, 'missing domain');
-        return split(':',@emptyResults);
+        return ('missing domain');
     }
 }
 
@@ -5338,8 +5621,8 @@ sub dirlist {
 ##
 sub GetFileTimestamp {
     my ($studentDomain,$studentName,$filename,$root)=@_;
-    $studentDomain=~s/\W//g;
-    $studentName=~s/\W//g;
+    $studentDomain = &LONCAPA::clean_domain($studentDomain);
+    $studentName   = &LONCAPA::clean_username($studentName);
     my $subdir=$studentName.'__';
     $subdir =~ s/(.)(.)(.).*/$1\/$2\/$3/;
     my $proname="$studentDomain/$subdir/$studentName";
@@ -5362,13 +5645,13 @@ sub stat_file {
     my ($udom,$uname,$file,$dir);
     if ($uri =~ m-^/(uploaded|editupload)/-) {
 	($udom,$uname,$file) =
-	    ($uri =~ m-/(?:uploaded|editupload)/?([^/]*)/?([^/]*)/?(.*)-);
+	    ($uri =~ m-/(?:uploaded|editupload)/?($match_domain)/?($match_name)/?(.*)-);
 	$file = 'userfiles/'.$file;
 	$dir = &propath($udom,$uname);
     }
     if ($uri =~ m-^/res/-) {
 	($udom,$uname) = 
-	    ($uri =~ m-/(?:res)/?([^/]*)/?([^/]*)/-);
+	    ($uri =~ m-/(?:res)/?($match_domain)/?($match_username)/-);
 	$file = $uri;
     }
 
@@ -5949,7 +6232,7 @@ sub metadata {
 	(($uri =~ m|^/*adm/|) && 
 	     ($uri !~ m|^adm/includes|) && ($uri !~ m|/bulletinboard$|)) ||
         ($uri =~ m|/$|) || ($uri =~ m|/.meta$|) || ($uri =~ /^~/) ||
-	($uri =~ m|home/[^/]+/public_html/|)) {
+	($uri =~ m|home/$match_username/public_html/|)) {
 	return undef;
     }
     my $filename=$uri;
@@ -6316,13 +6599,13 @@ sub symbverify {
         }
         if ($ids) {
 # ------------------------------------------------------------------- Has ID(s)
-	    foreach (split(/\,/,$ids)) {
-	       my ($mapid,$resid)=split(/\./,$_);
+	    foreach my $id (split(/\,/,$ids)) {
+	       my ($mapid,$resid)=split(/\./,$id);
                if (
   &symbclean(&declutter($bighash{'map_id_'.$mapid}).'___'.$resid.'___'.$thisfn)
    eq $symb) { 
 		   if (($env{'request.role.adv'}) ||
-		       $bighash{'encrypted_'.$_} eq $env{'request.enc'}) {
+		       $bighash{'encrypted_'.$id} eq $env{'request.enc'}) {
 		       $okay=1; 
 		   }
 	       }
@@ -6465,10 +6748,10 @@ sub symbread {
                  } elsif (!$donotrecurse) {
 # ------------------------------------------ There is more than one possibility
                      my $realpossible=0;
-                     foreach (@possibilities) {
-			 my $file=$bighash{'src_'.$_};
+                     foreach my $id (@possibilities) {
+			 my $file=$bighash{'src_'.$id};
                          if (&allowed('bre',$file)) {
-         		    my ($mapid,$resid)=split(/\./,$_);
+         		    my ($mapid,$resid)=split(/\./,$id);
                             if ($bighash{'map_type_'.$mapid} ne 'page') {
 				$realpossible++;
                                 $syval=&encode_symb($bighash{'map_id_'.$mapid},
@@ -6609,6 +6892,7 @@ sub rndseed {
     if (!$domain) { $domain=$wdomain; }
     if (!$username) { $username=$wusername }
     my $which=&get_rand_alg();
+
     if (defined(&getCODE())) {
 	if ($which eq '64bit5') {
 	    return &rndseed_CODE_64bit5($symb,$courseid,$domain,$username);
@@ -6666,7 +6950,6 @@ sub rndseed_64bit {
 	#&logthis("$symbseed:$nameseed;$domainseed|$courseseed;$namechck:$symbchck");
 	#&logthis("rndseed :$num:$symb");
 	if ($_64bit) { $num1=(($num1<<32)>>32); $num2=(($num2<<32)>>32); }
-	if ($_64bit) { $num1=(($num1<<32)>>32); $num2=(($num2<<32)>>32); }
 	return "$num1,$num2";
     }
 }
@@ -6689,6 +6972,7 @@ sub rndseed_64bit2 {
 	my $num2=$nameseed+$domainseed+$courseseed;
 	#&logthis("$symbseed:$nameseed;$domainseed|$courseseed;$namechck:$symbchck");
 	#&logthis("rndseed :$num:$symb");
+	if ($_64bit) { $num1=(($num1<<32)>>32); $num2=(($num2<<32)>>32); }
 	return "$num1,$num2";
     }
 }
@@ -6926,61 +7210,60 @@ sub repcopy_userfile {
     if ($file =~ m -^/*(uploaded|editupload)/-) { $file=&filelocation("",$file); }
     if ($file =~ m|^/home/httpd/html/lonUsers/|) { return 'ok'; }
     my ($cdom,$cnum,$filename) = 
-	($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+([^/]+)/+([^/]+)/+(.*)|);
-    my ($info,$rtncode);
+	($file=~m|^\Q$perlvar{'lonDocRoot'}\E/+userfiles/+($match_domain)/+($match_name)/+(.*)|);
     my $uri="/uploaded/$cdom/$cnum/$filename";
     if (-e "$file") {
+# we already have a local copy, check it out
 	my @fileinfo = stat($file);
+	my $rtncode;
+	my $info;
 	my $lwpresp = &getuploaded('HEAD',$uri,$cdom,$cnum,\$info,\$rtncode);
 	if ($lwpresp ne 'ok') {
+# there is no such file anymore, even though we had a local copy
 	    if ($rtncode eq '404') {
 		unlink($file);
 	    }
-	    #my $ua=new LWP::UserAgent;
-	    #my $request=new HTTP::Request('GET',&tokenwrapper($uri));
-	    #my $response=$ua->request($request);
-	    #if ($response->is_success()) {
-	#	return $response->content;
-	#    } else {
-	#	return -1;
-	#    }
 	    return -1;
 	}
 	if ($info < $fileinfo[9]) {
+# nice, the file we have is up-to-date, just say okay
 	    return 'ok';
+	} else {
+# the file is outdated, get rid of it
+	    unlink($file);
 	}
-	$info = '';
-	$lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode);
-	if ($lwpresp ne 'ok') {
-	    return -1;
-	}
-    } else {
-	my $lwpresp = &getuploaded('GET',$uri,$cdom,$cnum,\$info,\$rtncode);
-	if ($lwpresp ne 'ok') {
-	    my $ua=new LWP::UserAgent;
-	    my $request=new HTTP::Request('GET',&tokenwrapper($uri));
-	    my $response=$ua->request($request);
-	    if ($response->is_success()) {
-		$info=$response->content;
-	    } else {
-		return -1;
-	    }
-	}
-	my @parts = ($cdom,$cnum); 
-	if ($filename =~ m|^(.+)/[^/]+$|) {
-	    push @parts, split(/\//,$1);
-	}
-	my $path = $perlvar{'lonDocRoot'}.'/userfiles';
-	foreach my $part (@parts) {
-	    $path .= '/'.$part;
-	    if (!-e $path) {
-		mkdir($path,0770);
-	    }
+    }
+# one way or the other, at this point, we don't have the file
+# construct the correct path for the file
+    my @parts = ($cdom,$cnum); 
+    if ($filename =~ m|^(.+)/[^/]+$|) {
+	push @parts, split(/\//,$1);
+    }
+    my $path = $perlvar{'lonDocRoot'}.'/userfiles';
+    foreach my $part (@parts) {
+	$path .= '/'.$part;
+	if (!-e $path) {
+	    mkdir($path,0770);
 	}
     }
-    open(FILE,">$file");
-    print FILE $info;
-    close(FILE);
+# now the path exists for sure
+# get a user agent
+    my $ua=new LWP::UserAgent;
+    my $transferfile=$file.'.in.transfer';
+# FIXME: this should flock
+    if (-e $transferfile) { return 'ok'; }
+    my $request;
+    $uri=~s/^\///;
+    $request=new HTTP::Request('GET','http://'.$hostname{&homeserver($cnum,$cdom)}.'/raw/'.$uri);
+    my $response=$ua->request($request,$transferfile);
+# did it work?
+    if ($response->is_error()) {
+	unlink($transferfile);
+	&logthis("Userfile repcopy failed for $uri");
+	return -1;
+    }
+# worked, rename the transfer file
+    rename($transferfile,$file);
     return 'ok';
 }
 
@@ -7002,6 +7285,10 @@ sub tokenwrapper {
     }
 }
 
+# call with reqtype HEAD: get last modification time
+# call with reqtype GET: get the file contents
+# Do not call this with reqtype GET for large files! It loads everything into memory
+#
 sub getuploaded {
     my ($reqtype,$uri,$cdom,$cnum,$info,$rtncode) = @_;
     $uri=~s/^\///;
@@ -7027,7 +7314,7 @@ sub readfile {
     my $fh;
     open($fh,"<$file");
     my $a='';
-    while (<$fh>) { $a .=$_; }
+    while (my $line = <$fh>) { $a .= $line; }
     return $a;
 }
 
@@ -7043,12 +7330,12 @@ sub filelocation {
     if ($file=~m:^/~:) { # is a contruction space reference
         $location = $file;
         $location =~ s:/~(.*?)/(.*):/home/$1/public_html/$2:;
-    } elsif ($file=~m:^/home/[^/]*/public_html/:) {
+    } elsif ($file=~m{^/home/$match_username/public_html/}) {
 	# is a correct contruction space reference
         $location = $file;
     } elsif ($file=~/^\/*(uploaded|editupload)/) { # is an uploaded file
         my ($udom,$uname,$filename)=
-  	    ($file=~m -^/+(?:uploaded|editupload)/+([^/]+)/+([^/]+)/+(.*)$-);
+  	    ($file=~m -^/+(?:uploaded|editupload)/+($match_domain)/+($match_name)/+(.*)$-);
         my $home=&homeserver($uname,$udom);
         my $is_me=0;
         my @ids=&current_machine_ids();
@@ -7085,10 +7372,10 @@ sub hreflocation {
     }
     if ($file=~m-^\Q$perlvar{'lonDocRoot'}\E-) {
 	$file=~s-^\Q$perlvar{'lonDocRoot'}\E--;
-    } elsif ($file=~m-/home/(\w+)/public_html/-) {
-	$file=~s-^/home/(\w+)/public_html/-/~$1/-;
+    } elsif ($file=~m-/home/($match_username)/public_html/-) {
+	$file=~s-^/home/($match_username)/public_html/-/~$1/-;
     } elsif ($file=~m-^\Q$perlvar{'lonUsersDir'}\E-) {
-	$file=~s-^/home/httpd/lonUsers/([^/]*)/./././([^/]*)/userfiles/
+	$file=~s-^/home/httpd/lonUsers/($match_domain)/./././($match_name)/userfiles/
 	    -/uploaded/$1/$2/-x;
     }
     return $file;
@@ -7118,6 +7405,29 @@ sub current_machine_ids {
     return @ids;
 }
 
+sub additional_machine_domains {
+    my @domains;
+    open(my $fh,"<$perlvar{'lonTabDir'}/expected_domains.tab");
+    while( my $line = <$fh>) {
+        $line =~ s/\s//g;
+        push(@domains,$line);
+    }
+    return @domains;
+}
+
+sub default_login_domain {
+    my $domain = $perlvar{'lonDefDomain'};
+    my $testdomain=(split(/\./,$ENV{'HTTP_HOST'}))[0];
+    foreach my $posdom (&current_machine_domains(),
+                        &additional_machine_domains()) {
+        if (lc($posdom) eq lc($testdomain)) {
+            $domain=$posdom;
+            last;
+        }
+    }
+    return $domain;
+}
+
 # ------------------------------------------------------------- Declutters URLs
 
 sub declutter {
@@ -7238,12 +7548,12 @@ BEGIN {
     %domain_auth_arg_def = ();
     my $fh;
     if (open($fh,"<".$Apache::lonnet::perlvar{'lonTabDir'}.'/domain.tab')) {
-       while (<$fh>) {
-           next if (/^(\#|\s*$)/);
+	while (my $line = <$fh>) {
+           next if ($line =~ /^(\#|\s*$)/);
 #           next if /^\#/;
-           chomp;
+           chomp $line;
            my ($domain, $domain_description, $def_auth, $def_auth_arg,
-	       $def_lang, $city, $longi, $lati, $primary) = split(/:/,$_);
+	       $def_lang, $city, $longi, $lati, $primary) = split(/:/,$line,9);
 	   $domain_auth_def{$domain}=$def_auth;
            $domain_auth_arg_def{$domain}=$def_auth_arg;
 	   $domaindescription{$domain}=$domain_description;
@@ -7290,7 +7600,7 @@ sub get_iphost {
 	if (!exists($name_to_ip{$name})) {
 	    $ip = gethostbyname($name);
 	    if (!$ip || length($ip) ne 4) {
-		&logthis("Skipping host $id name $name no IP found\n");
+		&logthis("Skipping host $id name $name no IP found");
 		next;
 	    }
 	    $ip=inet_ntoa($ip);
@@ -7642,8 +7952,7 @@ passed in @what from the requested user'
 
 =item *
 
-allowed($priv,$uri) : check for a user privilege; returns codes for allowed
-actions
+allowed($priv,$uri,$symb,$role) : check for a user privilege; returns codes for allowed actions
  F: full access
  U,I,K: authentication modes (cxx only)
  '': forbidden
@@ -7662,6 +7971,9 @@ and course level
 plaintext($short) : return value in %prp hash (rolesplain.tab); plain text
 explanation of a user role term
 
+=item *
+
+get_my_roles($uname,$udom,$types,$roles,$roledoms) : All arguments are optional.  Returns a hash of a user's roles, with keys set to colon-sparated $uname,$udom,and $role, and value set to colon-separated start and end times for the role. If no username and domain are specified, will default to current user/domain. Types, roles, and roledoms are references to arrays, of role statuses (active, future or previous), roles (e.g., cc,in, st etc.) and domains of the roles which can be used to restrict the list if roles reported. If no array ref is provided for types, will default to return only active roles.  
 =back
 
 =head2 User Modification
@@ -8083,6 +8395,15 @@ reference filled in from namesp (encrypt
 log($udom,$name,$home,$message) : write to permanent log for user; use
 critical subroutine
 
+=item *
+
+get_dom($namespace,$storearr,$udomain) : returns hash with keys from array
+reference filled in from namespace found in domain level on primary domain server ($udomain is optional)
+
+=item *
+
+put_dom($namespace,$storehash,$udomain) :  stores hash in namespace at domain level on primary domain server ($udomain is optional)
+
 =back
 
 =head2 Network Status Functions