--- loncom/lonnet/perl/lonnet.pm	2018/03/30 21:30:00	1.1371
+++ loncom/lonnet/perl/lonnet.pm	2018/11/01 04:33:11	1.1387
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1371 2018/03/30 21:30:00 raeburn Exp $
+# $Id: lonnet.pm,v 1.1387 2018/11/01 04:33:11 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -230,12 +230,19 @@ sub get_server_distarch {
 }
 
 sub get_servercerts_info {
-    my ($lonhost,$context) = @_;
+    my ($lonhost,$hostname,$context) = @_;
+    return if ($lonhost eq '');
+    if ($hostname eq '') {
+        $hostname = &hostname($lonhost);
+    }
+    return if ($hostname eq '');
     my ($rep,$uselocal);
-    if (grep { $_ eq $lonhost } &current_machine_ids()) {
+    if (context eq 'install') {
+        $uselocal = 1;
+    } elsif (grep { $_ eq $lonhost } &current_machine_ids()) {
         $uselocal = 1;
     }
-    if (($context ne 'cgi') && ($uselocal)) {
+    if (($context ne 'cgi') && ($context ne 'install') && ($uselocal)) {
         my $distro = (split(/\:/,&get_server_distarch($lonhost)))[0];
         if ($distro eq '') {
             $uselocal = 0;
@@ -250,16 +257,11 @@ sub get_servercerts_info {
         }
     }
     if ($uselocal) {
-        $rep = LONCAPA::Lond::server_certs(\%perlvar);
+        $rep = LONCAPA::Lond::server_certs(\%perlvar,$lonhost,$hostname);
     } else {
         $rep=&reply('servercerts',$lonhost);
     }
     my ($result,%returnhash);
-    if (defined($lonhost)) {
-        if (!defined(&hostname($lonhost))) {
-            return;
-        }
-    }
     if (($rep=~/^(refused|rejected|error)/) || ($rep eq 'con_lost') ||
         ($rep eq 'unknown_cmd')) {
         $result = $rep;
@@ -652,31 +654,39 @@ sub transfer_profile_to_env {
 sub check_for_valid_session {
     my ($r,$name,$userhashref,$domref) = @_;
     my %cookies=CGI::Cookie->parse($r->header_in('Cookie'));
-    my ($linkname,$pubname);
-    if ($name eq '') {
-        $name = 'lonID';
+    my ($lonidsdir,$linkname,$pubname,$secure,$lonid);
+    if ($name eq 'lonDAV') {
+        $lonidsdir=$r->dir_config('lonDAVsessDir');
+    } else {
+        $lonidsdir=$r->dir_config('lonIDsDir');
+        if ($name eq '') {
+            $name = 'lonID';
+        }
+    }
+    if ($name eq 'lonID') {
+        $secure = 'lonSID';
         $linkname = 'lonLinkID';
         $pubname = 'lonPubID';
-    }
-    my $lonid=$cookies{$name};
-    if (!$lonid) {
-        if (($name eq 'lonID') && ($ENV{'SERVER_PORT'} != 443) && ($linkname)) {
+        if (exists($cookies{$secure})) {
+            $lonid=$cookies{$secure};
+        } elsif (exists($cookies{$name})) {
+            $lonid=$cookies{$name};
+        } elsif (exists($cookies{$linkname})) {
             $lonid=$cookies{$linkname};
+        } elsif (exists($cookies{$pubname})) {
+            $lonid=$cookies{$pubname};
         }
-        if (!$lonid) {
-            if (($name eq 'lonID') && ($pubname)) {
-                $lonid=$cookies{$pubname};
-            }
-        }
+    } else {
+        $lonid=$cookies{$name};
     }
     return undef if (!$lonid);
 
     my $handle=&LONCAPA::clean_handle($lonid->value);
-    my $lonidsdir;
-    if ($name eq 'lonDAV') {
-        $lonidsdir=$r->dir_config('lonDAVsessDir');
-    } else {
-        $lonidsdir=$r->dir_config('lonIDsDir');
+    if (-l "$lonidsdir/$handle.id") {
+        my $link = readlink("$lonidsdir/$handle.id");
+        if ((-e $link) && ($link =~ m{^\Q$lonidsdir\E/(.+)\.id$})) {
+            $handle = $1;
+        }
     }
     if (!-e "$lonidsdir/$handle.id") {
         if ((ref($domref)) && ($name eq 'lonID') && 
@@ -708,6 +718,10 @@ sub check_for_valid_session {
         $userhashref->{'name'} = $disk_env{'user.name'};
         $userhashref->{'domain'} = $disk_env{'user.domain'};
         $userhashref->{'lti'} = $disk_env{'request.lti.login'};
+        if ($userhashref->{'lti'}) {
+            $userhashref->{'ltitarget'} = $disk_env{'request.lti.target'};
+            $userhashref->{'ltiuri'} = $disk_env{'request.lti.uri'};
+        }
     }
 
     return $handle;
@@ -758,16 +772,19 @@ sub appenv {
                 $env{$key}=$newenv->{$key};
             }
         }
-        my $opened = open(my $env_file,'+<',$env{'user.environment'});
-        if ($opened
-	    && &timed_flock($env_file,LOCK_EX)
-	    &&
-	    tie(my %disk_env,'GDBM_File',$env{'user.environment'},
-	        (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
-	    while (my ($key,$value) = each(%{$newenv})) {
-	        $disk_env{$key} = $value;
-	    }
-	    untie(%disk_env);
+        my $lonids = $perlvar{'lonIDsDir'};
+        if ($env{'user.environment'} =~ m{^\Q$lonids/\E$match_username\_\d+\_$match_domain\_[\w\-.]+\.id$}) {
+            my $opened = open(my $env_file,'+<',$env{'user.environment'});
+            if ($opened
+	        && &timed_flock($env_file,LOCK_EX)
+	        &&
+	        tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	            (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
+	        while (my ($key,$value) = each(%{$newenv})) {
+	            $disk_env{$key} = $value;
+	        }
+	        untie(%disk_env);
+            }
         }
     }
     return 'ok';
@@ -3182,7 +3199,17 @@ sub ssi {
 
     $request->header(Cookie => $ENV{'HTTP_COOKIE'});
     my $lonhost = $perlvar{'lonHostID'};
-    my $response= &LONCAPA::LWPReq::makerequest($lonhost,$request,'',\%perlvar);
+    my $islocal;
+    if (($env{'request.course.id'}) &&
+        ($form{'grade_courseid'} eq $env{'request.course.id'}) &&
+        ($form{'grade_username'} ne '') && ($form{'grade_domain'} ne '') &&
+        ($form{'grade_symb'} ne '') &&
+        (&Apache::lonnet::allowed('mgr',$env{'request.course.id'}.
+                                 ($env{'request.course.sec'}?'/'.$env{'request.course.sec'}:'')))) {
+        $islocal = 1;
+    }
+    my $response= &LONCAPA::LWPReq::makerequest($lonhost,$request,'',\%perlvar,
+                                                '','','',$islocal);
 
     if (wantarray) {
 	return ($response->content, $response);
@@ -5216,7 +5243,12 @@ sub set_first_access {
     }
     $cachedkey='';
     my $firstaccess=&get_first_access($type,$symb,$map);
-    if (!$firstaccess) {
+    if ($firstaccess) {
+        &logthis("First access time already set ($firstaccess) when attempting ".
+                 "to set new value (type: $type, extent: $res) for $uname:$udom ". 
+                 "in $courseid"); 
+        return 'already_set';
+    } else {
         my $start = time;
 	my $putres = &put('firstaccesstimes',{"$courseid\0$res"=>$start},
                           $udom,$uname);
@@ -5232,6 +5264,9 @@ sub set_first_access {
             if (($cachedtime) && (abs($start-$cachedtime) < 5)) {
                 $cachedtimes{"$courseid\0$res"} = $start;
             }
+        } elsif ($putres ne 'refused') {
+            &logthis("Result: $putres when attempting to set first access time ".
+                     "(type: $type, extent: $res) for $uname:$udom in $courseid");
         }
         return $putres;
     }
@@ -9283,7 +9318,7 @@ sub assignrole {
             }
             if ($refused) {
                 my ($cdom,$cnum) = ($cwosec =~ m{^/?($match_domain)/($match_courseid)$});
-                if (!$selfenroll && $context eq 'course') {
+                if (!$selfenroll && (($context eq 'course') || ($context eq 'ltienroll' && $env{'request.lti.login'}))) {
                     my %crsenv;
                     if ($role eq 'cc' || $role eq 'co') {
                         %crsenv = &userenvironment($cdom,$cnum,('internal.courseowner'));
@@ -9306,7 +9341,7 @@ sub assignrole {
                 } elsif (($selfenroll == 1) && ($udom eq $env{'user.domain'}) && ($uname eq $env{'user.name'})) {
                     if ($role eq 'st') {
                         $refused = '';
-                    } elsif (($context eq 'ltienroll') && ($env{'request.lti'})) {
+                    } elsif (($context eq 'ltienroll') && ($env{'request.lti.login'})) {
                         $refused = '';
                     }
                 } elsif ($context eq 'requestcourses') {
@@ -10026,12 +10061,25 @@ sub is_course {
     my ($cdom, $cnum) = scalar(@_) == 1 ? 
          ($_[0] =~ /^($match_domain)_($match_courseid)$/)  :  @_;
 
-    return unless $cdom and $cnum;
-
-    my %courses = &courseiddump($cdom, '.', 1, '.', '.', $cnum, undef, undef,
-        '.');
-
-    return unless(exists($courses{$cdom.'_'.$cnum}));
+    return unless (($cdom =~ /^$match_domain$/) && ($cnum =~ /^$match_courseid$/));
+    my $uhome=&homeserver($cnum,$cdom);
+    my $iscourse;
+    if (grep { $_ eq $uhome } current_machine_ids()) {
+        $iscourse = &LONCAPA::Lond::is_course($cdom,$cnum);
+    } else {
+        my $hashid = $cdom.':'.$cnum;
+        ($iscourse,my $cached) = &is_cached_new('iscourse',$hashid);
+        unless (defined($cached)) {
+            my %courses = &courseiddump($cdom, '.', 1, '.', '.',
+                                        $cnum,undef,undef,'.');
+            $iscourse = 0;
+            if (exists($courses{$cdom.'_'.$cnum})) {
+                $iscourse = 1;
+            }
+            &do_cache_new('iscourse',$hashid,$iscourse,3600);
+        }
+    }
+    return unless ($iscourse);
     return wantarray ? ($cdom, $cnum) : $cdom.'_'.$cnum;
 }
 
@@ -11928,23 +11976,31 @@ sub metadata {
 # Check metadata for imported file to
 # see if it contained response items
 #
+                        my ($origfile,@libfilekeys);
                         my %currmetaentry = %metaentry;
-                        my $libresponseorder = &metadata($location,'responseorder');
-                        my $origfile;
-                        if ($libresponseorder ne '') {
-                            if ($#origfiletagids<0) {
-                                undef(%importedrespids);
-                                undef(%importedpartids);
-                            }
-                            @{$importedrespids{$importid}} = split(/\s*,\s*/,$libresponseorder);
-                            if (@{$importedrespids{$importid}} > 0) {
-                                $importedresponses = 1;
+                        @libfilekeys = split(/,/,&metadata($location,'keys',undef,undef,undef,
+                                                           $depthcount+1));
+                        if (grep(/^responseorder$/,@libfilekeys)) {
+                            my $libresponseorder = &metadata($location,'responseorder',undef,undef,
+                                                             undef,$depthcount+1);
+                            if ($libresponseorder ne '') {
+                                if ($#origfiletagids<0) {
+                                    undef(%importedrespids);
+                                    undef(%importedpartids);
+                                }
+                                my @respids = split(/\s*,\s*/,$libresponseorder);
+                                if (@respids) {
+                                    $importedrespids{$importid} = join(',',map { $importid.'_'.$_ } @respids);
+                                }
+                                if ($importedrespids{$importid} ne '') {
+                                    $importedresponses = 1;
 # We need to get the original file and the imported file to get the response order correct
 # Load and inspect original file
-                                if ($#origfiletagids<0) {
-                                    my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
-                                    $origfile=&getfile($origfilelocation);
-                                    @origfiletagids=($origfile=~/<((?:\w+)response|import|part)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                                    if ($#origfiletagids<0) {
+                                        my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                        $origfile=&getfile($origfilelocation);
+                                        @origfiletagids=($origfile=~/<((?:\w+)response|import|part)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                                    }
                                 }
                             }
                         }
@@ -11952,10 +12008,7 @@ sub metadata {
 # hash populated for imported library file
                         %metaentry = %currmetaentry;
                         undef(%currmetaentry);
-                        if ($importmode eq 'problem') {
-# Import as problem/response
-                           $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
-                        } elsif ($importmode eq 'part') {
+                        if ($importmode eq 'part') {
 # Import as part(s)
                            $importedparts=1;
 # We need to get the original file and the imported file to get the part order correct
@@ -11970,10 +12023,23 @@ sub metadata {
                                    @origfiletagids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
                                }
                            }
-
-# Load and inspect imported file
-                           my $impfile=&getfile($location);
-                           my @impfilepartids=($impfile=~/<part[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                           my @impfilepartids;
+# If <partorder> tag is included in metadata for the imported file
+# get the parts in the imported file from that.
+                           if (grep(/^partorder$/,@libfilekeys)) {
+                               %currmetaentry = %metaentry;
+                               my $libpartorder = &metadata($location,'partorder',undef,undef,undef,
+                                                            $depthcount+1);
+                               %metaentry = %currmetaentry;
+                               undef(%currmetaentry);
+                               if ($libpartorder ne '') {
+                                   @impfilepartids=split(/\s*,\s*/,$libpartorder);
+                               }
+                           } else {
+# If no <partorder> tag available, load and inspect imported file
+                               my $impfile=&getfile($location);
+                               @impfilepartids=($impfile=~/<part[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                           }
                            if ($#impfilepartids>=0) {
 # This problem had parts
                                $importedpartids{$token->[2]->{'id'}}=join(',',@impfilepartids);
@@ -11984,13 +12050,28 @@ sub metadata {
                                $importedpartids{$token->[2]->{'id'}}=$token->[2]->{'id'};
                            }
                         } else {
+# Import as problem or as normal import
+                            $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
+                            unless ($importmode eq 'problem') {
 # Normal import
-                           $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
-                           if (defined($token->[2]->{'id'})) {
-                              $unikey.='_'.$token->[2]->{'id'};
-                           }
+                                if (defined($token->[2]->{'id'})) {
+                                    $unikey.='_'.$token->[2]->{'id'};
+                                }
+                            }
+# Check metadata for imported file to
+# see if it contained parts
+                            if (grep(/^partorder$/,@libfilekeys)) {
+                                %currmetaentry = %metaentry;
+                                my $libpartorder = &metadata($location,'partorder',undef,undef,undef,
+                                                             $depthcount+1);
+                                %metaentry = %currmetaentry;
+                                undef(%currmetaentry);
+                                if ($libpartorder ne '') {
+                                    $importedparts = 1;
+                                    $importedpartids{$token->[2]->{'id'}}=$libpartorder;
+                                }
+                            }
                         }
-
 			if ($depthcount<20) {
 			    my $metadata = 
 				&metadata($uri,'keys',$toolsymb,$location,$unikey,
@@ -12102,12 +12183,14 @@ sub metadata {
                 } elsif ($origfiletagids[$index] eq 'import') {
                     if ($importedparts) {
 # We have imported parts at this position
-                        $metaentry{':partorder'}.=','.$importedpartids{$origid};
+                        if ($importedpartids{$origid} ne '') {
+                            $metaentry{':partorder'}.=','.$importedpartids{$origid};
+                        }
                     }
                     if ($importedresponses) {
 # We have imported responses at this position
-                        if (ref($importedrespids{$origid}) eq 'ARRAY') {
-                            $metaentry{':responseorder'}.=','.join(',',map { $origid.'_'.$_ } @{$importedrespids{$origid}});
+                        if ($importedrespids{$origid} ne '') {
+                            $metaentry{':responseorder'}.=','.$importedrespids{$origid};
                         }
                     }
                 } else {
@@ -12124,7 +12207,6 @@ sub metadata {
                 $metaentry{':responseorder'}=~s/^\,//;
             }
         }
-
 	$metaentry{':keys'} = join(',',keys(%metathesekeys));
 	&metadata_generate_part0(\%metathesekeys,\%metaentry,$uri);
 	$metaentry{':allpossiblekeys'}=join(',',keys(%metathesekeys));
@@ -13470,15 +13552,17 @@ sub get_dns {
     }
 
     my %alldns;
-    open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab");
-    foreach my $dns (<$config>) {
-	next if ($dns !~ /^\^(\S*)/x);
-        my $line = $1;
-        my ($host,$protocol) = split(/:/,$line);
-        if ($protocol ne 'https') {
-            $protocol = 'http';
+    if (open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab")) {
+        foreach my $dns (<$config>) {
+	    next if ($dns !~ /^\^(\S*)/x);
+            my $line = $1;
+            my ($host,$protocol) = split(/:/,$line);
+            if ($protocol ne 'https') {
+                $protocol = 'http';
+            }
+	    $alldns{$host} = $protocol;
         }
-	$alldns{$host} = $protocol;
+        close($config);
     }
     while (%alldns) {
 	my ($dns) = sort { $b cmp $a } keys(%alldns);
@@ -13486,19 +13570,33 @@ sub get_dns {
         my $response = &LONCAPA::LWPReq::makerequest('',$request,'',\%perlvar,30,0);
         delete($alldns{$dns});
 	next if ($response->is_error());
-	my @content = split("\n",$response->content);
-	unless ($nocache) {
-	    &do_cache_new('dns',$url,\@content,30*24*60*60);
-	}
-	&$func(\@content,$hashref);
-	return;
+        if ($url eq '/adm/dns/loncapaCRL') {
+            return &$func($response);
+        } else {
+	    my @content = split("\n",$response->content);
+	    unless ($nocache) {
+	        &do_cache_new('dns',$url,\@content,30*24*60*60);
+	    }
+	    &$func(\@content,$hashref);
+            return;
+        }
+    }
+    my $which = (split('/',$url,4))[3];
+    if ($which eq 'loncapaCRL') {
+        my $diskfile = "$perlvar{'lonCertificateDirectory'}/$perlvar{'lonnetCertRevocationList'}";
+        if (-e $diskfile) {
+            &logthis("unable to contact DNS, on disk file $diskfile not updated");
+        } else {
+            &logthis("unable to contact DNS, no on disk file $diskfile available");
+        }
+    } else {
+        &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
+        if (open(my $config,"<","$perlvar{'lonTabDir'}/dns_$which.tab")) {
+            my @content = <$config>;
+            close($config);
+            &$func(\@content,$hashref);
+        }
     }
-    close($config);
-    my $which = (split('/',$url))[3];
-    &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
-    open($config,"<","$perlvar{'lonTabDir'}/dns_$which.tab");
-    my @content = <$config>;
-    &$func(\@content,$hashref);
     return;
 }
 
@@ -13558,6 +13656,79 @@ sub fetch_dns_checksums {
     return \%checksums;
 }
 
+sub fetch_crl_pemfile {
+    return &get_dns("/adm/dns/loncapaCRL",\&save_crl_pem,1,1);
+}
+
+sub save_crl_pem {
+    my ($response) = @_;
+    my ($msg,$hadchanges);
+    if (ref($response)) {
+        my $now = time;
+        my $lonca = $perlvar{'lonCertificateDirectory'}.'/'.$perlvar{'lonnetCertificateAuthority'};
+        my $tmpcrl = $tmpdir.'/'.$perlvar{'lonnetCertRevocationList'}.'_'.$now.'.'.$$.'.tmp';
+        if (open(my $fh,'>',"$tmpcrl")) {
+            print $fh $response->content;
+            close($fh);
+            if (-e $lonca) {
+                if (open(PIPE,"openssl crl -in $tmpcrl -inform pem -CAfile $lonca -noout 2>&1 |")) {
+                    my $check = <PIPE>;
+                    close(PIPE);
+                    chomp($check);
+                    if ($check eq 'verify OK') {
+                        my $dest = "$perlvar{'lonCertificateDirectory'}/$perlvar{'lonnetCertRevocationList'}";
+                        my $backup;
+                        if (-e $dest) {
+                            if (&File::Copy::move($dest,"$dest.bak")) {
+                                $backup = 'ok';
+                            }
+                        }
+                        if (&File::Copy::move($tmpcrl,$dest)) {
+                            $msg = 'ok';
+                            if ($backup) {
+                                my (%oldnums,%newnums);
+                                if (open(PIPE, "openssl crl -inform PEM -text -noout -in $dest.bak |grep 'Serial Number' |")) {
+                                    while (<PIPE>) {
+                                        $oldnums{(split(/:/))[1]} = 1;
+                                    }
+                                    close(PIPE);
+                                }
+                                if (open(PIPE, "openssl crl -inform PEM -text -noout -in $dest |grep 'Serial Number' |")) {
+                                    while(<PIPE>) {
+                                        $newnums{(split(/:/))[1]} = 1;
+                                    }
+                                    close(PIPE);
+                                }
+                                foreach my $key (sort {$b <=> $a } (keys(%newnums))) {
+                                    unless (exists($oldnums{$key})) {
+                                        $hadchanges = 1;
+                                        last;
+                                    }
+                                }
+                                unless ($hadchanges) {
+                                    foreach my $key (sort {$b <=> $a } (keys(%oldnums))) {
+                                        unless (exists($newnums{$key})) {
+                                            $hadchanges = 1;
+                                            last;
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    } else {
+                        unlink($tmpcrl);
+                    }
+                } else {
+                    unlink($tmpcrl);
+                }
+            } else {
+                unlink($tmpcrl);
+            }
+        }
+    }
+    return ($msg,$hadchanges);
+}
+
 # ------------------------------------------------------------ Read domain file
 {
     my $loaded;