--- loncom/lonnet/perl/lonnet.pm	2018/09/02 02:13:59	1.1172.2.95
+++ loncom/lonnet/perl/lonnet.pm	2019/08/01 19:13:26	1.1172.2.107
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1172.2.95 2018/09/02 02:13:59 raeburn Exp $
+# $Id: lonnet.pm,v 1.1172.2.107 2019/08/01 19:13:26 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -74,8 +74,9 @@ use strict;
 use LWP::UserAgent();
 use HTTP::Date;
 use Image::Magick;
+use CGI::Cookie;
 
-use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir
+use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir $deftex
             $_64bit %env %protocol %loncaparevs %serverhomeIDs %needsrelease
             %managerstab);
 
@@ -142,7 +143,7 @@ our @EXPORT = qw(%env);
 sub logtouch {
     my $execdir=$perlvar{'lonDaemons'};
     unless (-e "$execdir/logs/lonnet.log") {	
-	open(my $fh,">>$execdir/logs/lonnet.log");
+	open(my $fh,">>","$execdir/logs/lonnet.log");
 	close $fh;
     }
     my ($wwwuid,$wwwgid)=(getpwnam('www'))[2,3];
@@ -154,7 +155,7 @@ sub logthis {
     my $execdir=$perlvar{'lonDaemons'};
     my $now=time;
     my $local=localtime($now);
-    if (open(my $fh,">>$execdir/logs/lonnet.log")) {
+    if (open(my $fh,">>","$execdir/logs/lonnet.log")) {
 	my $logstring = $local. " ($$): ".$message."\n"; # Keep any \'s in string.
 	print $fh $logstring;
 	close($fh);
@@ -167,7 +168,7 @@ sub logperm {
     my $execdir=$perlvar{'lonDaemons'};
     my $now=time;
     my $local=localtime($now);
-    if (open(my $fh,">>$execdir/logs/lonnet.perm.log")) {
+    if (open(my $fh,">>","$execdir/logs/lonnet.perm.log")) {
 	print $fh "$now:$message:$local\n";
 	close($fh);
     }
@@ -436,7 +437,7 @@ sub reconlonc {
 
     &logthis("Trying to reconnect lonc");
     my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
-    if (open(my $fh,"<$loncfile")) {
+    if (open(my $fh,"<",$loncfile)) {
 	my $loncpid=<$fh>;
         chomp($loncpid);
         if (kill 0 => $loncpid) {
@@ -476,7 +477,7 @@ sub critical {
             $dumpcount++;
             {
 		my $dfh;
-		if (open($dfh,">$dfilename")) {
+		if (open($dfh,">",$dfilename)) {
 		    print $dfh "$cmd\n"; 
 		    close($dfh);
 		}
@@ -485,7 +486,7 @@ sub critical {
             my $wcmd='';
             {
 		my $dfh;
-		if (open($dfh,"<$dfilename")) {
+		if (open($dfh,"<",$dfilename)) {
 		    $wcmd=<$dfh>; 
 		    close($dfh);
 		}
@@ -601,7 +602,7 @@ sub transfer_profile_to_env {
 
 # ---------------------------------------------------- Check for valid session 
 sub check_for_valid_session {
-    my ($r,$name,$userhashref) = @_;
+    my ($r,$name,$userhashref,$domref) = @_;
     my %cookies=CGI::Cookie->parse($r->header_in('Cookie'));
     if ($name eq '') {
         $name = 'lonID';
@@ -616,7 +617,16 @@ sub check_for_valid_session {
     } else {
         $lonidsdir=$r->dir_config('lonIDsDir');
     }
-    return undef if (!-e "$lonidsdir/$handle.id");
+    if (!-e "$lonidsdir/$handle.id") {
+        if ((ref($domref)) && ($name eq 'lonID') &&
+            ($handle =~ /^($match_username)\_\d+\_($match_domain)\_(.+)$/)) {
+            my ($possuname,$possudom,$possuhome) = ($1,$2,$3);
+            if ((&domain($possudom) ne '') && (&homeserver($possuname,$possudom) eq $possuhome)) {
+                $$domref = $possudom;
+            }
+        }
+        return undef;
+    }
 
     my $opened = open(my $idf,'+<',"$lonidsdir/$handle.id");
     return undef if (!$opened);
@@ -630,6 +640,7 @@ sub check_for_valid_session {
 
     if (!defined($disk_env{'user.name'})
 	|| !defined($disk_env{'user.domain'})) {
+        untie(%disk_env);
 	return undef;
     }
 
@@ -637,6 +648,7 @@ sub check_for_valid_session {
         $userhashref->{'name'} = $disk_env{'user.name'};
         $userhashref->{'domain'} = $disk_env{'user.domain'};
     }
+    untie(%disk_env);
 
     return $handle;
 }
@@ -661,6 +673,37 @@ sub timed_flock {
     }
 }
 
+sub get_sessionfile_vars {
+    my ($handle,$lonidsdir,$storearr) = @_;
+    my %returnhash;
+    unless (ref($storearr) eq 'ARRAY') {
+        return %returnhash;
+    }
+    if (-l "$lonidsdir/$handle.id") {
+        my $link = readlink("$lonidsdir/$handle.id");
+        if ((-e $link) && ($link =~ m{^\Q$lonidsdir\E/(.+)\.id$})) {
+            $handle = $1;
+        }
+    }
+    if ((-e "$lonidsdir/$handle.id") &&
+        ($handle =~ /^($match_username)\_\d+\_($match_domain)\_(.+)$/)) {
+        my ($possuname,$possudom,$possuhome) = ($1,$2,$3);
+        if ((&domain($possudom) ne '') && (&homeserver($possuname,$possudom) eq $possuhome)) {
+            if (open(my $idf,'+<',"$lonidsdir/$handle.id")) {
+                flock($idf,LOCK_SH);
+                if (tie(my %disk_env,'GDBM_File',"$lonidsdir/$handle.id",
+                        &GDBM_READER(),0640)) {
+                    foreach my $item (@{$storearr}) {
+                        $returnhash{$item} = $disk_env{$item};
+                    }
+                    untie(%disk_env);
+                }
+            }
+        }
+    }
+    return %returnhash;
+}
+
 # ---------------------------------------------------------- Append Environment
 
 sub appenv {
@@ -686,16 +729,19 @@ sub appenv {
                 $env{$key}=$newenv->{$key};
             }
         }
-        my $opened = open(my $env_file,'+<',$env{'user.environment'});
-        if ($opened
-	    && &timed_flock($env_file,LOCK_EX)
-	    &&
-	    tie(my %disk_env,'GDBM_File',$env{'user.environment'},
-	        (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
-	    while (my ($key,$value) = each(%{$newenv})) {
-	        $disk_env{$key} = $value;
-	    }
-	    untie(%disk_env);
+        my $lonids = $perlvar{'lonIDsDir'};
+        if ($env{'user.environment'} =~ m{^\Q$lonids/\E$match_username\_\d+\_$match_domain\_[\w\-.]+\.id$}) {
+            my $opened = open(my $env_file,'+<',$env{'user.environment'});
+            if ($opened
+	        && &timed_flock($env_file,LOCK_EX)
+	        &&
+	        tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	            (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
+	        while (my ($key,$value) = each(%{$newenv})) {
+	            $disk_env{$key} = $value;
+	        }
+	        untie(%disk_env);
+            }
         }
     }
     return 'ok';
@@ -939,6 +985,75 @@ sub find_existing_session {
     return;
 }
 
+# check if user's browser sent load balancer cookie and server still has session
+# and is not overloaded.
+sub check_for_balancer_cookie {
+    my ($r,$update_mtime) = @_;
+    my ($otherserver,$cookie);
+    my %cookies=CGI::Cookie->parse($r->header_in('Cookie'));
+    if (exists($cookies{'balanceID'})) {
+        my $balid = $cookies{'balanceID'};
+        $cookie=&LONCAPA::clean_handle($balid->value);
+        my $balancedir=$r->dir_config('lonBalanceDir');
+        if ((-d $balancedir) && (-e "$balancedir/$cookie.id")) {
+            if ($cookie =~ /^($match_domain)_($match_username)_[a-f0-9]+$/) {
+                my ($possudom,$possuname) = ($1,$2);
+                my $has_session = 0;
+                if ((&domain($possudom) ne '') &&
+                    (&homeserver($possuname,$possudom) ne 'no_host')) {
+                    my $try_server;
+                    my $opened = open(my $idf,'+<',"$balancedir/$cookie.id");
+                    if ($opened) {
+                        flock($idf,LOCK_SH);
+                        while (my $line = <$idf>) {
+                            chomp($line);
+                            if (&hostname($line) ne '') {
+                                $try_server = $line;
+                                last;
+                            }
+                        }
+                        close($idf);
+                        if (($try_server) &&
+                            (&has_user_session($try_server,$possudom,$possuname))) {
+                            my $lowest_load = 30000;
+                            ($otherserver,$lowest_load) =
+                                &compare_server_load($try_server,undef,$lowest_load);
+                            if ($otherserver ne '' && $lowest_load < 100) {
+                                $has_session = 1;
+                            } else {
+                                undef($otherserver);
+                            }
+                        }
+                    }
+                }
+                if ($has_session) {
+                    if ($update_mtime) {
+                        my $atime = my $mtime = time;
+                        utime($atime,$mtime,"$balancedir/$cookie.id");
+                    }
+                } else {
+                    unlink("$balancedir/$cookie.id");
+                }
+            }
+        }
+    }
+    return ($otherserver,$cookie);
+}
+
+sub delbalcookie {
+    my ($cookie,$balancer) =@_;
+    if ($cookie =~ /^($match_domain)\_($match_username)\_[a-f0-9]{32}$/) {
+        my ($udom,$uname) = ($1,$2);
+        my $uprimary_id = &domain($udom,'primary');
+        my $uintdom = &internet_dom($uprimary_id);
+        my $intdom = &internet_dom($balancer);
+        my $serverhomedom = &host_domain($balancer);
+        if (($uintdom ne '') && ($uintdom eq $intdom)) {
+            return &reply("delbalcookie:$cookie",$balancer);
+        }
+    }
+}
+
 # -------------------------------- ask if server already has a session for user
 sub has_user_session {
     my ($lonid,$udom,$uname) = @_;
@@ -1276,7 +1391,7 @@ sub get_lonbalancer_config {
 sub check_loadbalancing {
     my ($uname,$udom,$caller) = @_;
     my ($is_balancer,$currtargets,$currrules,$dom_in_use,$homeintdom,
-        $rule_in_effect,$offloadto,$otherserver);
+        $rule_in_effect,$offloadto,$otherserver,$setcookie);
     my $lonhost = $perlvar{'lonHostID'};
     my @hosts = &current_machine_ids();
     my $uprimary_id = &Apache::lonnet::domain($udom,'primary');
@@ -1303,7 +1418,7 @@ sub check_loadbalancing {
         }
     }
     if (ref($result) eq 'HASH') {
-        ($is_balancer,$currtargets,$currrules) =
+        ($is_balancer,$currtargets,$currrules,$setcookie) =
             &check_balancer_result($result,@hosts);
         if ($is_balancer) {
             if (ref($currrules) eq 'HASH') {
@@ -1364,7 +1479,7 @@ sub check_loadbalancing {
             }
         }
         if (ref($result) eq 'HASH') {
-            ($is_balancer,$currtargets,$currrules) =
+            ($is_balancer,$currtargets,$currrules,$setcookie) =
                 &check_balancer_result($result,@hosts);
             if ($is_balancer) {
                 if (ref($currrules) eq 'HASH') {
@@ -1430,20 +1545,22 @@ sub check_loadbalancing {
                 $is_balancer = 0;
                 if ($uname ne '' && $udom ne '') {
                     if (($env{'user.name'} eq $uname) && ($env{'user.domain'} eq $udom)) {
-
                         &appenv({'user.loadbalexempt'     => $lonhost,
                                  'user.loadbalcheck.time' => time});
                     }
                 }
             }
         }
+        unless ($homeintdom) {
+            undef($setcookie);
+        }
     }
-    return ($is_balancer,$otherserver);
+    return ($is_balancer,$otherserver,$setcookie);
 }
 
 sub check_balancer_result {
     my ($result,@hosts) = @_;
-    my ($is_balancer,$currtargets,$currrules);
+    my ($is_balancer,$currtargets,$currrules,$setcookie);
     if (ref($result) eq 'HASH') {
         if ($result->{'lonhost'} ne '') {
             my $currbalancer = $result->{'lonhost'};
@@ -1459,12 +1576,13 @@ sub check_balancer_result {
                     $is_balancer = 1;
                     $currrules = $result->{$key}{'rules'};
                     $currtargets = $result->{$key}{'targets'};
+                    $setcookie = $result->{$key}{'cookie'};
                     last;
                 }
             }
         }
     }
-    return ($is_balancer,$currtargets,$currrules);
+    return ($is_balancer,$currtargets,$currrules,$setcookie);
 }
 
 sub get_loadbalancer_targets {
@@ -1829,7 +1947,7 @@ sub inst_directory_query {
 			   &escape($srch->{'srchtype'}),$homeserver);
 	my $host=&hostname($homeserver);
 	if ($queryid !~/^\Q$host\E\_/) {
-	    &logthis('institutional directory search invalid queryid: '.$queryid.' for host: '.$homeserver.'in domain '.$udom);
+	    &logthis('institutional directory search invalid queryid: '.$queryid.' for host: '.$homeserver.' in domain '.$udom);
 	    return;
 	}
 	my $response = &get_query_reply($queryid);
@@ -2181,6 +2299,9 @@ sub get_domain_defaults {
         } elsif ($domconfig{'coursedefaults'}{'canclone'}) {
             $domdefaults{'canclone'}=$domconfig{'coursedefaults'}{'canclone'};
         }
+        if ($domconfig{'coursedefaults'}{'texengine'}) {
+            $domdefaults{'texengine'} = $domconfig{'coursedefaults'}{'texengine'};
+        }
     }
     if (ref($domconfig{'usersessions'}) eq 'HASH') {
         if (ref($domconfig{'usersessions'}{'remote'}) eq 'HASH') {
@@ -2244,6 +2365,65 @@ sub get_domain_defaults {
     return %domdefaults;
 }
 
+sub get_dom_cats {
+    my ($dom) = @_;
+    return unless (&domain($dom));
+    my ($cats,$cached)=&is_cached_new('cats',$dom);
+    unless (defined($cached)) {
+        my %domconfig = &get_dom('configuration',['coursecategories'],$dom);
+        if (ref($domconfig{'coursecategories'}) eq 'HASH') {
+            if (ref($domconfig{'coursecategories'}{'cats'}) eq 'HASH') {
+                %{$cats} = %{$domconfig{'coursecategories'}{'cats'}};
+            } else {
+                $cats = {};
+            }
+        } else {
+            $cats = {};
+        }
+        &Apache::lonnet::do_cache_new('cats',$dom,$cats,3600);
+    }
+    return $cats;
+}
+
+sub get_dom_instcats {
+    my ($dom) = @_;
+    return unless (&domain($dom));
+    my ($instcats,$cached)=&is_cached_new('instcats',$dom);
+    unless (defined($cached)) {
+        my (%coursecodes,%codes,@codetitles,%cat_titles,%cat_order);
+        my $totcodes = &retrieve_instcodes(\%coursecodes,$dom);
+        if ($totcodes > 0) {
+            my $caller = 'global';
+            if (&auto_instcode_format($caller,$dom,\%coursecodes,\%codes,
+                                      \@codetitles,\%cat_titles,\%cat_order) eq 'ok') {
+                $instcats = {
+                                codes => \%codes,
+                                codetitles => \@codetitles,
+                                cat_titles => \%cat_titles,
+                                cat_order => \%cat_order,
+                            };
+                &do_cache_new('instcats',$dom,$instcats,3600);
+            }
+        }
+    }
+    return $instcats;
+}
+
+sub retrieve_instcodes {
+    my ($coursecodes,$dom) = @_;
+    my $totcodes;
+    my %courses = &courseiddump($dom,'.',1,'.','.','.',undef,undef,'Course');
+    foreach my $course (keys(%courses)) {
+        if (ref($courses{$course}) eq 'HASH') {
+            if ($courses{$course}{'inst_code'} ne '') {
+                $$coursecodes{$course} = $courses{$course}{'inst_code'};
+                $totcodes ++;
+            }
+        }
+    }
+    return $totcodes;
+}
+
 # --------------------------------------------------- Assign a key to a student
 
 sub assign_access_key {
@@ -2828,8 +3008,7 @@ sub absolute_url {
 sub ssi {
 
     my ($fn,%form)=@_;
-    my $ua=new LWP::UserAgent;
-    my $request;
+    my ($request,$response);
 
     $form{'no_update_last_known'}=1;
     &Apache::lonenc::check_encrypt(\$fn);
@@ -2846,7 +3025,30 @@ sub ssi {
     }
 
     $request->header(Cookie => $ENV{'HTTP_COOKIE'});
-    my $response= $ua->request($request);
+
+    if (($env{'request.course.id'}) &&
+        ($form{'grade_courseid'} eq $env{'request.course.id'}) &&
+        ($form{'grade_username'} ne '') && ($form{'grade_domain'} ne '') &&
+        ($form{'grade_symb'} ne '') &&
+        (&Apache::lonnet::allowed('mgr',$env{'request.course.id'}.
+                                 ($env{'request.course.sec'}?'/'.$env{'request.course.sec'}:'')))) {
+        if (LWP::UserAgent->VERSION >= 5.834) {
+            my $ua=new LWP::UserAgent;
+            $ua->local_address('127.0.0.1');
+            $response = $ua->request($request);
+        } else {
+            {
+                require LWP::Protocol::http;
+                local @LWP::Protocol::http::EXTRA_SOCK_OPTS = (LocalAddr => '127.0.0.1');
+                my $ua=new LWP::UserAgent;
+                $response = $ua->request($request);
+                @LWP::Protocol::http::EXTRA_SOCK_OPTS = ();
+            }
+        }
+    } else {
+        my $ua=new LWP::UserAgent;
+        $response = $ua->request($request);
+    }
     if (wantarray) {
 	return ($response->content, $response);
     } else {
@@ -2866,6 +3068,72 @@ sub externalssi {
     }
 }
 
+# If the local copy of a replicated resource is outdated, trigger a
+# connection from the homeserver to flush the delayed queue. If no update
+# happens, remove local copies of outdated resource (and corresponding
+# metadata file).
+
+sub remove_stale_resfile {
+    my ($url) = @_;
+    my $removed;
+    if ($url=~m{^/res/($match_domain)/($match_username)/}) {
+        my $audom = $1;
+        my $auname = $2;
+        unless (($url =~ /\.\d+\.\w+$/) || ($url =~ m{^/res/lib/templates/})) {
+            my $homeserver = &homeserver($auname,$audom);
+            unless (($homeserver eq 'no_host') ||
+                    (grep { $_ eq $homeserver } &current_machine_ids())) {
+                my $fname = &filelocation('',$url);
+                if (-e $fname) {
+                    my $ua=new LWP::UserAgent;
+                    $ua->timeout(5);
+                    my $protocol = $protocol{$homeserver};
+                    $protocol = 'http' if ($protocol ne 'https');
+                    my $hostname = &hostname($homeserver);
+                    if ($hostname) {
+                        my $uri = $protocol.'://'.$hostname.'/raw/'.&declutter($url);
+                        my $request=new HTTP::Request('HEAD',$uri);
+                        my $response=$ua->request($request);
+                        if ($response->is_success()) {
+                            my $remmodtime = &HTTP::Date::str2time( $response->header('Last-modified') );
+                            my $locmodtime = (stat($fname))[9];
+                            if ($locmodtime < $remmodtime) {
+                                my $stale;
+                                my $answer = &reply('pong',$homeserver);
+                                if ($answer eq $homeserver.':'.$perlvar{'lonHostID'}) {
+                                    sleep(0.2);
+                                    $locmodtime = (stat($fname))[9];
+                                    if ($locmodtime < $remmodtime) {
+                                        my $posstransfer = $fname.'.in.transfer';
+                                        if ((-e $posstransfer) && ($remmodtime < (stat($posstransfer))[9])) {
+                                            $removed = 1;
+                                        } else {
+                                            $stale = 1;
+                                        }
+                                    } else {
+                                        $removed = 1;
+                                    }
+                                } else {
+                                    $stale = 1;
+                                }
+                                if ($stale) {
+                                    unlink($fname);
+                                    if ($uri!~/\.meta$/) {
+                                        unlink($fname.'.meta');
+                                    }
+                                    &reply("unsub:$fname",$homeserver);
+                                    $removed = 1;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+    return $removed;
+}
+
 # -------------------------------- Allow a /uploaded/ URI to be vouched for
 
 sub allowuploaded {
@@ -3194,7 +3462,7 @@ sub process_coursefile {
                                  $home);
             }
         } elsif ($action eq 'uploaddoc') {
-            open(my $fh,'>'.$filepath.'/'.$fname);
+            open(my $fh,'>',$filepath.'/'.$fname);
             print $fh $env{'form.'.$source};
             close($fh);
             if ($parser eq 'parse') {
@@ -3252,7 +3520,7 @@ sub store_edited_file {
     ($fpath,$fname) = ($file =~ m|^(.*)/([^/]+)$|);
     $fpath=$docudom.'/'.$docuname.'/'.$fpath;
     my $filepath = &build_filepath($fpath);
-    open(my $fh,'>'.$filepath.'/'.$fname);
+    open(my $fh,'>',$filepath.'/'.$fname);
     print $fh $content;
     close($fh);
     my $home=&homeserver($docuname,$docudom);
@@ -3368,12 +3636,12 @@ sub userfileupload {
                          '_'.$env{'user.domain'}.'/pending';
         } elsif (($context eq 'existingfile') || ($context eq 'canceloverwrite')) {
             my ($docuname,$docudom);
-            if ($destudom) {
+            if ($destudom =~ /^$match_domain$/) {
                 $docudom = $destudom;
             } else {
                 $docudom = $env{'user.domain'};
             }
-            if ($destuname) {
+            if ($destuname =~ /^$match_username$/) { 
                 $docuname = $destuname;
             } else {
                 $docuname = $env{'user.name'};
@@ -3403,7 +3671,7 @@ sub userfileupload {
                 mkdir($fullpath,0777);
             }
         }
-        open(my $fh,'>'.$fullpath.'/'.$fname);
+        open(my $fh,'>',$fullpath.'/'.$fname);
         print $fh $env{'form.'.$formname};
         close($fh);
         if ($context eq 'existingfile') {
@@ -3478,7 +3746,7 @@ sub finishuserfileupload {
 
 # Save the file
     {
-	if (!open(FH,'>'.$filepath.'/'.$file)) {
+	if (!open(FH,'>',$filepath.'/'.$file)) {
 	    &logthis('Failed to create '.$filepath.'/'.$file);
 	    print STDERR ('Failed to create '.$filepath.'/'.$file."\n");
 	    return '/adm/notfound.html';
@@ -3536,7 +3804,8 @@ sub finishuserfileupload {
         my $input = $filepath.'/'.$file;
         my $output = $filepath.'/'.'tn-'.$file;
         my $thumbsize = $thumbwidth.'x'.$thumbheight;
-        system("convert -sample $thumbsize $input $output");
+        my @args = ('convert','-sample',$thumbsize,$input,$output);
+        system({$args[0]} @args);
         if (-e $filepath.'/'.'tn-'.$file) {
             $fetchthumb  = 1; 
         }
@@ -4496,7 +4765,7 @@ sub postannounce {
 
 sub getannounce {
 
-    if (open(my $fh,$perlvar{'lonDocRoot'}.'/announcement.txt')) {
+    if (open(my $fh,"<",$perlvar{'lonDocRoot'}.'/announcement.txt')) {
 	my $announcement='';
 	while (my $line = <$fh>) { $announcement .= $line; }
 	close($fh);
@@ -4792,7 +5061,12 @@ sub set_first_access {
     }
     $cachedkey='';
     my $firstaccess=&get_first_access($type,$symb,$map);
-    if (!$firstaccess) {
+    if ($firstaccess) {
+        &logthis("First access time already set ($firstaccess) when attempting ".
+                 "to set new value (type: $type, extent: $res) for $uname:$udom ".
+                 "in $courseid");
+        return 'already_set';
+    } else {
         my $start = time;
 	my $putres = &put('firstaccesstimes',{"$courseid\0$res"=>$start},
                           $udom,$uname);
@@ -4805,6 +5079,12 @@ sub set_first_access {
                         'course.'.$courseid.'.timerinterval.'.$res => $interval,
                      }
                   );
+            if (($cachedtime) && (abs($start-$cachedtime) < 5)) {
+                $cachedtimes{"$courseid\0$res"} = $start;
+            }
+        } elsif ($putres ne 'refused') {
+            &logthis("Result: $putres when attempting to set first access time ".
+                     "(type: $type, extent: $res) for $uname:$udom in $courseid");
         }
         return $putres;
     }
@@ -8113,7 +8393,7 @@ sub fetch_enrollment_query {
                         if ($xml_classlist =~ /^error/) {
                             &logthis('fetch_enrollment_query - autoretrieve error: '.$xml_classlist.' for '.$filename.' from server: '.$homeserver.' '.$context.' '.$cnum);
                         } else {
-                            if ( open(FILE,">$destname") ) {
+                            if ( open(FILE,">",$destname) ) {
                                 print FILE &unescape($xml_classlist);
                                 close(FILE);
                             } else {
@@ -8142,7 +8422,7 @@ sub get_query_reply {
     for (1..$loopmax) {
 	sleep($sleep);
         if (-e $replyfile.'.end') {
-	    if (open(my $fh,$replyfile)) {
+	    if (open(my $fh,"<",$replyfile)) {
 		$reply = join('',<$fh>);
 		close($fh);
 	   } else { return 'error: reply_file_error'; }
@@ -9806,7 +10086,7 @@ sub save_selected_files {
     my ($user, $path, @files) = @_;
     my $filename = $user."savedfiles";
     my @other_files = &files_not_in_path($user, $path);
-    open (OUT, '>'.$tmpdir.$filename);
+    open (OUT,'>',LONCAPA::tempdir().$filename);
     foreach my $file (@files) {
         print (OUT $env{'form.currentpath'}.$file."\n");
     }
@@ -9820,7 +10100,7 @@ sub save_selected_files {
 sub clear_selected_files {
     my ($user) = @_;
     my $filename = $user."savedfiles";
-    open (OUT, '>'.LONCAPA::tempdir().$filename);
+    open (OUT,'>',LONCAPA::tempdir().$filename);
     print (OUT undef);
     close (OUT);
     return ("ok");    
@@ -9830,7 +10110,7 @@ sub files_in_path {
     my ($user, $path) = @_;
     my $filename = $user."savedfiles";
     my %return_files;
-    open (IN, '<'.LONCAPA::tempdir().$filename);
+    open (IN,'<',LONCAPA::tempdir().$filename);
     while (my $line_in = <IN>) {
         chomp ($line_in);
         my @paths_and_file = split (m!/!, $line_in);
@@ -9852,7 +10132,7 @@ sub files_not_in_path {
     my $filename = $user."savedfiles";
     my @return_files;
     my $path_part;
-    open(IN, '<'.LONCAPA::.$filename);
+    open(IN, '<',LONCAPA::tempdir().$filename);
     while (my $line = <IN>) {
         #ok, I know it's clunky, but I want it to work
         my @paths_and_file = split(m|/|, $line);
@@ -10986,6 +11266,7 @@ sub add_prefix_and_part {
 
 my %metaentry;
 my %importedpartids;
+my %importedrespids;
 sub metadata {
     my ($uri,$what,$liburi,$prefix,$depthcount)=@_;
     $uri=&declutter($uri);
@@ -11013,9 +11294,11 @@ sub metadata {
     }
     {
 # Imported parts would go here
-        my %importedids=();
-        my @origfileimportpartids=();
+        my @origfiletagids=();
         my $importedparts=0;
+
+# Imported responseids would go here
+        my $importedresponses=0;
 #
 # Is this a recursive call for a library?
 #
@@ -11110,8 +11393,37 @@ sub metadata {
                         my $dir=$filename;
                         $dir=~s|[^/]*$||;
                         $location=&filelocation($dir,$location);
-                       
+
+                        my $importid=$token->[2]->{'id'};
                         my $importmode=$token->[2]->{'importmode'};
+#
+# Check metadata for imported file to
+# see if it contained response items
+#
+                        my %currmetaentry = %metaentry;
+                        my $libresponseorder = &metadata($location,'responseorder');
+                        my $origfile;
+                        if ($libresponseorder ne '') {
+                            if ($#origfiletagids<0) {
+                                undef(%importedrespids);
+                                undef(%importedpartids);
+                            }
+                            @{$importedrespids{$importid}} = split(/\s*,\s*/,$libresponseorder);
+                            if (@{$importedrespids{$importid}} > 0) {
+                                $importedresponses = 1;
+# We need to get the original file and the imported file to get the response order correct
+# Load and inspect original file
+                                if ($#origfiletagids<0) {
+                                    my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                    $origfile=&getfile($origfilelocation);
+                                    @origfiletagids=($origfile=~/<((?:\w+)response|import|part)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                                }
+                            }
+                        }
+# Do not overwrite contents of %metaentry hash for resource itself with 
+# hash populated for imported library file
+                        %metaentry = %currmetaentry;
+                        undef(%currmetaentry);
                         if ($importmode eq 'problem') {
 # Import as problem/response
                            $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
@@ -11120,12 +11432,15 @@ sub metadata {
                            $importedparts=1;
 # We need to get the original file and the imported file to get the part order correct
 # Good news: we do not need to worry about nested libraries, since parts cannot be nested
-# Load and inspect original file
-                           if ($#origfileimportpartids<0) {
-                              undef(%importedpartids);
-                              my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
-                              my $origfile=&getfile($origfilelocation);
-                              @origfileimportpartids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+# Load and inspect original file if we didn't do that already
+                           if ($#origfiletagids<0) {
+                               undef(%importedrespids);
+                               undef(%importedpartids);
+                               if ($origfile eq '') {
+                                   my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                   $origfile=&getfile($origfilelocation);
+                                   @origfiletagids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                               }
                            }
 
 # Load and inspect imported file
@@ -11239,20 +11554,48 @@ sub metadata {
 	    grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'}));
 	$metaentry{':packages'} = join(',',@uniq_packages);
 
-        if ($importedparts) {
+        if (($importedresponses) || ($importedparts)) {
+            if ($importedparts) {
 # We had imported parts and need to rebuild partorder
-           $metaentry{':partorder'}='';
-           $metathesekeys{'partorder'}=1;
-           for (my $index=0;$index<$#origfileimportpartids;$index+=2) {
-               if ($origfileimportpartids[$index] eq 'part') {
-# original part, part of the problem
-                  $metaentry{':partorder'}.=','.$origfileimportpartids[$index+1];
-               } else {
-# we have imported parts at this position
-                  $metaentry{':partorder'}.=','.$importedpartids{$origfileimportpartids[$index+1]};
-               }
-           }
-           $metaentry{':partorder'}=~s/^\,//;
+                $metaentry{':partorder'}='';
+                $metathesekeys{'partorder'}=1;
+            }
+            if ($importedresponses) {
+# We had imported responses and need to rebuild responseorder
+                $metaentry{':responseorder'}='';
+                $metathesekeys{'responseorder'}=1;
+            }
+            for (my $index=0;$index<$#origfiletagids;$index+=2) {
+                my $origid = $origfiletagids[$index+1];
+                if ($origfiletagids[$index] eq 'part') {
+# Original part, part of the problem
+                    if ($importedparts) {
+                        $metaentry{':partorder'}.=','.$origid;
+                    }
+                } elsif ($origfiletagids[$index] eq 'import') {
+                    if ($importedparts) {
+# We have imported parts at this position
+                        $metaentry{':partorder'}.=','.$importedpartids{$origid};
+                    }
+                    if ($importedresponses) {
+# We have imported responses at this position
+                        if (ref($importedrespids{$origid}) eq 'ARRAY') {
+                            $metaentry{':responseorder'}.=','.join(',',map { $origid.'_'.$_ } @{$importedrespids{$origid}});
+                        }
+                    }
+                } else {
+# Original response item, part of the problem
+                    if ($importedresponses) {
+                        $metaentry{':responseorder'}.=','.$origid;
+                    }
+                }
+            }
+            if ($importedparts) {
+                $metaentry{':partorder'}=~s/^\,//;
+            }
+            if ($importedresponses) {
+                $metaentry{':responseorder'}=~s/^\,//;
+            }
         }
 
 	$metaentry{':keys'} = join(',',keys(%metathesekeys));
@@ -12340,7 +12683,7 @@ sub readfile {
     my $file = shift;
     if ( (! -e $file ) || ($file eq '') ) { return -1; };
     my $fh;
-    open($fh,"<$file");
+    open($fh,"<",$file);
     my $a='';
     while (my $line = <$fh>) { $a .= $line; }
     return $a;
@@ -12453,7 +12796,7 @@ sub machine_ids {
 
 sub additional_machine_domains {
     my @domains;
-    open(my $fh,"<$perlvar{'lonTabDir'}/expected_domains.tab");
+    open(my $fh,"<","$perlvar{'lonTabDir'}/expected_domains.tab");
     while( my $line = <$fh>) {
         $line =~ s/\s//g;
         push(@domains,$line);
@@ -12474,6 +12817,27 @@ sub default_login_domain {
     return $domain;
 }
 
+sub shared_institution {
+    my ($dom) = @_;
+    my $same_intdom;
+    my $hostintdom = &internet_dom($perlvar{'lonHostID'});
+    if ($hostintdom ne '') {
+        my %iphost = &get_iphost();
+        my $primary_id = &domain($dom,'primary');
+        my $primary_ip = &get_host_ip($primary_id);
+        if (ref($iphost{$primary_ip}) eq 'ARRAY') {
+            foreach my $id (@{$iphost{$primary_ip}}) {
+                my $intdom = &internet_dom($id);
+                if ($intdom eq $hostintdom) {
+                    $same_intdom = 1;
+                    last;
+                }
+            }
+        }
+    }
+    return $same_intdom;
+}
+
 # ------------------------------------------------------------- Declutters URLs
 
 sub declutter {
@@ -12597,15 +12961,17 @@ sub get_dns {
     }
 
     my %alldns;
-    open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
-    foreach my $dns (<$config>) {
-	next if ($dns !~ /^\^(\S*)/x);
-        my $line = $1;
-        my ($host,$protocol) = split(/:/,$line);
-        if ($protocol ne 'https') {
-            $protocol = 'http';
+    if (open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab")) {
+        foreach my $dns (<$config>) {
+	    next if ($dns !~ /^\^(\S*)/x);
+            my $line = $1;
+            my ($host,$protocol) = split(/:/,$line);
+            if ($protocol ne 'https') {
+                $protocol = 'http';
+            }
+	    $alldns{$host} = $protocol;
         }
-	$alldns{$host} = $protocol;
+        close($config);
     }
     while (%alldns) {
 	my ($dns) = sort { $b cmp $a } keys(%alldns);
@@ -12622,12 +12988,12 @@ sub get_dns {
 	&$func(\@content,$hashref);
 	return;
     }
-    close($config);
     my $which = (split('/',$url))[3];
     &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
-    open($config,"<$perlvar{'lonTabDir'}/dns_$which.tab");
-    my @content = <$config>;
-    &$func(\@content,$hashref);
+    if (open(my $config,"<","$perlvar{'lonTabDir'}/dns_$which.tab")) {
+        my @content = <$config>;
+        &$func(\@content,$hashref);
+    }
     return;
 }
 
@@ -12718,7 +13084,7 @@ sub fetch_dns_checksums {
 	my ($ignore_cache,$nocache) = @_;
 	&get_dns('/adm/dns/domain',\&parse_domain_tab,$ignore_cache,$nocache);
 	my $fh;
-	if (open($fh,"<".$perlvar{'lonTabDir'}.'/domain.tab')) {
+	if (open($fh,"<",$perlvar{'lonTabDir'}.'/domain.tab')) {
 	    my @lines = <$fh>;
 	    &parse_domain_tab(\@lines);
 	}
@@ -12770,8 +13136,23 @@ sub fetch_dns_checksums {
 	    my ($id,$domain,$role,$name,$protocol,$intdom)=split(/:/,$configline);
 	    $name=~s/\s//g;
 	    if ($id && $domain && $role && $name) {
+                if ((exists($hostname{$id})) && ($hostname{$id} ne '')) {
+                    my $curr = $hostname{$id};
+                    my $skip;
+                    if (ref($name_to_host{$curr}) eq 'ARRAY') {
+                        if (($curr eq $name) && (@{$name_to_host{$curr}} == 1)) {
+                            $skip = 1;
+                        } else {
+                            @{$name_to_host{$curr}} = grep { $_ ne $id } @{$name_to_host{$curr}};
+                        }
+                    }
+                    unless ($skip) {
+                        push(@{$name_to_host{$name}},$id);
+                    }
+                } else {
+                    push(@{$name_to_host{$name}},$id);
+                }
 		$hostname{$id}=$name;
-		push(@{$name_to_host{$name}}, $id);
 		$hostdom{$id}=$domain;
 		if ($role eq 'library') { $libserv{$id}=$name; }
                 if (defined($protocol)) {
@@ -12804,7 +13185,7 @@ sub fetch_dns_checksums {
     sub load_hosts_tab {
 	my ($ignore_cache,$nocache) = @_;
 	&get_dns('/adm/dns/hosts',\&parse_hosts_tab,$ignore_cache,$nocache);
-	open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+	open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab");
 	my @config = <$config>;
 	&parse_hosts_tab(\@config);
 	close($config);
@@ -13070,7 +13451,7 @@ sub all_loncaparevs {
 {
     sub load_loncaparevs {
         if (-e "$perlvar{'lonTabDir'}/loncaparevs.tab") {
-            if (open(my $config,"<$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+            if (open(my $config,"<","$perlvar{'lonTabDir'}/loncaparevs.tab")) {
                 while (my $configline=<$config>) {
                     chomp($configline);
                     my ($hostid,$loncaparev)=split(/:/,$configline);
@@ -13086,7 +13467,7 @@ sub all_loncaparevs {
 {
     sub load_serverhomeIDs {
         if (-e "$perlvar{'lonTabDir'}/serverhomeIDs.tab") {
-            if (open(my $config,"<$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+            if (open(my $config,"<","$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
                 while (my $configline=<$config>) {
                     chomp($configline);
                     my ($name,$id)=split(/:/,$configline);
@@ -13111,7 +13492,7 @@ BEGIN {
 
 # ------------------------------------------------------ Read spare server file
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/spare.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/spare.tab");
 
     while (my $configline=<$config>) {
        chomp($configline);
@@ -13125,7 +13506,7 @@ BEGIN {
 }
 # ------------------------------------------------------------ Read permissions
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/roles.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/roles.tab");
 
     while (my $configline=<$config>) {
 	chomp($configline);
@@ -13139,7 +13520,7 @@ BEGIN {
 
 # -------------------------------------------- Read plain texts for permissions
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/rolesplain.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/rolesplain.tab");
 
     while (my $configline=<$config>) {
 	chomp($configline);
@@ -13159,7 +13540,7 @@ BEGIN {
 
 # ---------------------------------------------------------- Read package table
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/packages.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/packages.tab");
 
     while (my $configline=<$config>) {
 	if ($configline !~ /\S/ || $configline=~/^#/) { next; }
@@ -13205,7 +13586,7 @@ BEGIN {
 # ---------------------------------------------------------- Read managers table
 {
     if (-e "$perlvar{'lonTabDir'}/managers.tab") {
-        if (open(my $config,"<$perlvar{'lonTabDir'}/managers.tab")) {
+        if (open(my $config,"<","$perlvar{'lonTabDir'}/managers.tab")) {
             while (my $configline=<$config>) {
                 chomp($configline);
                 next if ($configline =~ /^\#/);
@@ -13224,6 +13605,11 @@ BEGIN {
 
 }
 
+# ------------- set default texengine (domain default overrides this)
+{
+    $deftex = LONCAPA::texengine();
+}
+
 $memcache=new Cache::Memcached({'servers'           => ['127.0.0.1:11211'],
 				'compress_threshold'=> 20_000,
  			        });