--- loncom/lonnet/perl/lonnet.pm	2020/01/17 16:45:28	1.1172.2.118
+++ loncom/lonnet/perl/lonnet.pm	2020/07/20 12:45:31	1.1172.2.118.2.6
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1172.2.118 2020/01/17 16:45:28 raeburn Exp $
+# $Id: lonnet.pm,v 1.1172.2.118.2.6 2020/07/20 12:45:31 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -263,9 +263,10 @@ sub get_server_loncaparev {
                 if ($caller eq 'loncron') {
                     my $ua=new LWP::UserAgent;
                     $ua->timeout(4);
+                    my $hostname = &hostname($lonhost);
                     my $protocol = $protocol{$lonhost};
                     $protocol = 'http' if ($protocol ne 'https');
-                    my $url = $protocol.'://'.&hostname($lonhost).'/adm/about.html';
+                    my $url = $protocol.'://'.$hostname.'/adm/about.html';
                     my $request=new HTTP::Request('GET',$url);
                     my $response=$ua->request($request);
                     unless ($response->is_error()) {
@@ -953,13 +954,13 @@ sub spareserver {
     }
 
     if (!$want_server_name) {
-        my $protocol = 'http';
-        if ($protocol{$spare_server} eq 'https') {
-            $protocol = $protocol{$spare_server};
-        }
         if (defined($spare_server)) {
             my $hostname = &hostname($spare_server);
             if (defined($hostname)) {
+                my $protocol = 'http';
+                if ($protocol{$spare_server} eq 'https') {
+                    $protocol = $protocol{$spare_server};
+                }
 	        $spare_server = $protocol.'://'.$hostname;
             }
         }
@@ -1161,6 +1162,28 @@ sub choose_server {
     return ($login_host,$hostname,$portal_path,$isredirect,$lowest_load);
 }
 
+sub get_course_sessions {
+    my ($cnum,$cdom,$lastactivity) = @_;
+    my %servers = &internet_dom_servers($cdom);
+    my %returnhash;
+    foreach my $server (sort(keys(%servers))) {
+        my $rep = &reply("coursesessions:$cdom:$cnum:$lastactivity",$server);
+        my @pairs=split(/\&/,$rep);
+        unless (($rep eq 'unknown_cmd') || ($rep =~ /^error/)) {
+            foreach my $item (@pairs) {
+                my ($key,$value)=split(/=/,$item,2);
+                $key = &unescape($key);
+                next if ($key =~ /^error: 2 /);
+                if (exists($returnhash{$key})) {
+                    next if ($value < $returnhash{$key});
+                }
+                $returnhash{$key}=$value;
+            }
+        }
+    }
+    return %returnhash;
+}
+
 # --------------------------------------------- Try to change a user's password
 
 sub changepass {
@@ -1850,7 +1873,12 @@ sub get_dom {
         }
     }
     if ($udom && $uhome && ($uhome ne 'no_host')) {
-        my $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        my $rep;
+        if ($namespace =~ /^enc/) {
+            $rep=&reply("encrypt:egetdom:$udom:$namespace:$items",$uhome);
+        } else {
+            $rep=&reply("getdom:$udom:$namespace:$items",$uhome);
+        }
         my %returnhash;
         if ($rep eq '' || $rep =~ /^error: 2 /) {
             return %returnhash;
@@ -1894,7 +1922,11 @@ sub put_dom {
             $items.=&escape($item).'='.&freeze_escape($$storehash{$item}).'&';
         }
         $items=~s/\&$//;
-        return &reply("putdom:$udom:$namespace:$items",$uhome);
+        if ($namespace =~ /^enc/) {
+            return &reply("encrypt:putdom:$udom:$namespace:$items",$uhome);
+        } else {
+            return &reply("putdom:$udom:$namespace:$items",$uhome);
+        }
     } else {
         &logthis("put_dom failed - no homeserver and/or domain");
     }
@@ -2040,7 +2072,7 @@ sub usersearch {
             unless ($tryserver eq $perlvar{'lonHostID'}) {
                 if ($srch->{'srchby'} eq 'email') {
                     my $lcrev = &get_server_loncaparev($dom,$tryserver);
-                    my ($major,$minor,$subver) = ($lcrev =~ /^\'?(\d+)\.(\d+)\.(\d+)[\w.\-]+)\'?$/);
+                    my ($major,$minor,$subver) = ($lcrev =~ /^\'?(\d+)\.(\d+)\.(\d+)[\w.\-]+\'?$/);
                     next if (($major eq '' && $minor eq '') || ($major < 2) ||
                              (($major == 2) && ($minor < 11)) ||
                              (($major == 2) && ($minor == 11) && ($subver < 3)));
@@ -2511,6 +2543,22 @@ sub get_passwdconf {
     return %passwdconf;
 }
 
+sub course_portal_url {
+    my ($cnum,$cdom) = @_;
+    my $chome = &homeserver($cnum,$cdom);
+    my $hostname = &hostname($chome);
+    my $protocol = $protocol{$chome};
+    $protocol = 'http' if ($protocol ne 'https');
+    my %domdefaults = &get_domain_defaults($cdom);
+    my $firsturl;
+    if ($domdefaults{'portal_def'}) {
+        $firsturl = $domdefaults{'portal_def'};
+    } else {
+        $firsturl = $protocol.'://'.$hostname;
+    }
+    return $firsturl;
+}
+
 # --------------------------------------------------- Assign a key to a student
 
 sub assign_access_key {
@@ -3046,6 +3094,32 @@ sub repcopy {
     }
 }
 
+# ------------------------------------------------- Unsubscribe from a resource
+
+sub unsubscribe {
+    my ($fname) = @_;
+    my $answer;
+    if ($fname=~/\/(aboutme|syllabus|bulletinboard|smppg)$/) { return $answer; }
+    $fname=~s/[\n\r]//g;
+    my $author=$fname;
+    $author=~s/\/home\/httpd\/html\/res\/([^\/]*)\/([^\/]*).*/$1\/$2/;
+    my ($udom,$uname)=split(/\//,$author);
+    my $home=homeserver($uname,$udom);
+    if ($home eq 'no_host') {
+        $answer = 'no_host';
+    } elsif (grep { $_ eq $home } &current_machine_ids()) {
+        $answer = 'home';
+    } else {
+        my $defdom = $perlvar{'lonDefDomain'};
+        if (&will_trust('content',$defdom,$udom)) {
+            $answer = reply("unsub:$fname",$home);
+        } else {
+            $answer = 'untrusted';
+        }
+    }
+    return $answer;
+}
+
 # ------------------------------------------------ Get server side include body
 sub ssi_body {
     my ($filelink,%form)=@_;
@@ -3172,13 +3246,13 @@ sub remove_stale_resfile {
                     (grep { $_ eq $homeserver } &current_machine_ids())) {
                 my $fname = &filelocation('',$url);
                 if (-e $fname) {
-                    my $ua=new LWP::UserAgent;
-                    $ua->timeout(5);
-                    my $protocol = $protocol{$homeserver};
-                    $protocol = 'http' if ($protocol ne 'https');
                     my $hostname = &hostname($homeserver);
                     if ($hostname) {
+                        my $protocol = $protocol{$homeserver};
+                        $protocol = 'http' if ($protocol ne 'https');
                         my $uri = $protocol.'://'.$hostname.'/raw/'.&declutter($url);
+                        my $ua=new LWP::UserAgent;
+                        $ua->timeout(5);
                         my $request=new HTTP::Request('HEAD',$uri);
                         my $response=$ua->request($request);
                         if ($response->is_success()) {
@@ -3204,12 +3278,18 @@ sub remove_stale_resfile {
                                     $stale = 1;
                                 }
                                 if ($stale) {
-                                    unlink($fname);
-                                    if ($uri!~/\.meta$/) {
-                                        unlink($fname.'.meta');
+                                    if (unlink($fname)) {
+                                        if ($uri!~/\.meta$/) {
+                                            if (-e $fname.'.meta') {
+                                                unlink($fname.'.meta');
+                                            }
+                                        }
+                                        my $unsubresult = &unsubscribe($fname);
+                                        unless ($unsubresult eq 'ok') {
+                                            &logthis("no unsub of $fname from $homeserver, reason: $unsubresult");
+                                        }
+                                        $removed = 1;
                                     }
-                                    &reply("unsub:$fname",$homeserver);
-                                    $removed = 1;
                                 }
                             }
                         }
@@ -3359,6 +3439,26 @@ sub can_edit_resource {
                         $forceedit = 1;
                     }
                     $cfile = $resurl;
+                } elsif (($resurl =~ m{^/ext/}) && ($symb ne '')) {
+                    my ($map,$id,$res) = &decode_symb($symb);
+                    if ($map =~ /\.page$/) {
+                        $incourse = 1;
+                        if ($env{'form.forceedit'}) {
+                            $forceview = 1;
+                            $cfile = $map;
+                        } else {
+                            $forceedit = 1;
+                            $cfile =  '/adm/wrapper'.$resurl;
+                        }
+                    }
+                } elsif ($resurl =~ m{^/adm/wrapper/adm/$cdom/$cnum/\d+/ext\.tool$}) {
+                    $incourse = 1;
+                    if ($env{'form.forceedit'}) {
+                        $forceview = 1;
+                    } else {
+                        $forceedit = 1;
+                    }
+                    $cfile = $resurl;
                 } elsif ($resurl =~ m{^/?adm/viewclasslist$}) {
                     $incourse = 1;
                     if ($env{'form.forceedit'}) {
@@ -3383,6 +3483,14 @@ sub can_edit_resource {
                     $forceedit = 1;
                 }
                 $cfile = $resurl;
+            } elsif (($resurl =~ m{^/adm/wrapper/adm/$cdom/$cnum/\d+/ext\.tool$}) && ($env{'form.folderpath'} =~ /^supplemental/)) {
+                $incourse = 1;
+                if ($env{'form.forceedit'}) {
+                    $forceview = 1;
+                } else {
+                    $forceedit = 1;
+                }
+                $cfile = $resurl;
             } elsif (($resurl eq '/adm/extresedit') && ($symb || $env{'form.folderpath'})) {
                 $incourse = 1;
                 $forceview = 1;
@@ -3392,8 +3500,13 @@ sub can_edit_resource {
                     $cfile = &clutter($res);
                 } else {
                     $cfile = $env{'form.suppurl'};
-                    $cfile =~ s{^http://}{};
-                    $cfile = '/adm/wrapper/ext/'.$cfile;
+                    my $escfile = &unescape($cfile);
+                    if ($escfile =~ m{^/adm/$cdom/$cnum/\d+/ext\.tool$}) {
+                        $cfile = '/adm/wrapper'.$escfile;
+                    } else {
+                        $escfile =~ s{^http://}{};
+                        $cfile = &escape("/adm/wrapper/ext/$escfile");
+                    }
                 }
             } elsif ($resurl =~ m{^/?adm/viewclasslist$}) {
                 if ($env{'form.forceedit'}) {
@@ -5366,9 +5479,10 @@ my %cachedtimes=();
 my $cachedtime='';
 
 sub load_all_first_access {
-    my ($uname,$udom)=@_;
+    my ($uname,$udom,$ignorecache)=@_;
     if (($cachedkey eq $uname.':'.$udom) &&
-        (abs($cachedtime-time)<5) && (!$env{'form.markaccess'})) {
+        (abs($cachedtime-time)<5) && (!$env{'form.markaccess'}) &&
+        (!$ignorecache)) {
         return;
     }
     $cachedtime=time;
@@ -5377,7 +5491,7 @@ sub load_all_first_access {
 }
 
 sub get_first_access {
-    my ($type,$argsymb,$argmap)=@_;
+    my ($type,$argsymb,$argmap,$ignorecache)=@_;
     my ($symb,$courseid,$udom,$uname)=&whichuser();
     if ($argsymb) { $symb=$argsymb; }
     my ($map,$id,$res)=&decode_symb($symb);
@@ -5389,7 +5503,7 @@ sub get_first_access {
     } else {
 	$res=$symb;
     }
-    &load_all_first_access($uname,$udom);
+    &load_all_first_access($uname,$udom,$ignorecache);
     return $cachedtimes{"$courseid\0$res"};
 }
 
@@ -6806,7 +6920,7 @@ sub currentdump {
    #
    my %returnhash=();
    #
-   if ($rep eq "unknown_cmd") { 
+   if ($rep eq 'unknown_cmd') {
        # an old lond will not know currentdump
        # Do a dump and make it look like a currentdump
        my @tmp = &dumpstore($courseid,$sdom,$sname,'.');
@@ -7739,7 +7853,7 @@ sub allowed {
 
     if (defined($env{'allowed.'.$priv})) { return $env{'allowed.'.$priv}; }
 # Free bre access to adm and meta resources
-    if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard)$})) 
+    if (((($uri=~/^adm\//) && ($uri !~ m{/(?:smppg|bulletinboard|ext\.tool)$})) 
 	 || (($uri=~/\.meta$/) && ($uri!~m|^uploaded/|) )) 
 	&& ($priv eq 'bre')) {
 	return 'F';
@@ -8436,7 +8550,8 @@ sub get_commblock_resources {
                             }
                         }
                     }
-                    if ($interval[0] =~ /^\d+$/) {
+                    if ($interval[0] =~ /^(\d+)/) {
+                        my $timelimit = $1; 
                         my $first_access;
                         if ($type eq 'resource') {
                             $first_access=&get_first_access($interval[1],$item);
@@ -8446,7 +8561,7 @@ sub get_commblock_resources {
                             $first_access=&get_first_access($interval[1]);
                         }
                         if ($first_access) {
-                            my $timesup = $first_access+$interval[0];
+                            my $timesup = $first_access+$timelimit;
                             if ($timesup > $now) {
                                 my $activeblock;
                                 foreach my $res (@to_test) {
@@ -10172,14 +10287,19 @@ sub writecoursepref {
 
 sub createcourse {
     my ($udom,$description,$url,$course_server,$nonstandard,$inst_code,
-        $course_owner,$crstype,$cnum,$context,$category)=@_;
+        $course_owner,$crstype,$cnum,$context,$category,$callercontext)=@_;
     $url=&declutter($url);
     my $cid='';
     if ($context eq 'requestcourses') {
         my $can_create = 0;
         my ($ownername,$ownerdom) = split(':',$course_owner);
         if ($udom eq $ownerdom) {
-            if (&usertools_access($ownername,$ownerdom,$category,undef,
+            my $reload;
+            if (($callercontext eq 'auto') &&
+               ($ownerdom eq $env{'user.domain'}) && ($ownername eq $env{'user.name'})) {
+                $reload = 'reload';
+            }
+            if (&usertools_access($ownername,$ownerdom,$category,$reload,
                                   $context)) {
                 $can_create = 1;
             }
@@ -11188,7 +11308,7 @@ sub get_userresdata {
 #  Parameters:
 #     $name      - Course/user name.
 #     $domain    - Name of the domain the user/course is registered on.
-#     $type      - Type of thing $name is (must be 'course' or 'user'
+#     $type      - Type of thing $name is (must be 'course' or 'user')
 #     @which     - Array of names of resources desired.
 #  Returns:
 #     The value of the first reasource in @which that is found in the
@@ -11207,13 +11327,44 @@ sub resdata {
     }
     if (!ref($result)) { return $result; }    
     foreach my $item (@which) {
-	if (defined($result->{$item->[0]})) {
-	    return [$result->{$item->[0]},$item->[1]];
-	}
+        if (ref($item) eq 'ARRAY') {
+	    if (defined($result->{$item->[0]})) {
+	        return [$result->{$item->[0]},$item->[1]];
+	    }
+        }
     }
     return undef;
 }
 
+sub get_domain_ltitools {
+    my ($cdom) = @_;
+    my %ltitools;
+    my ($result,$cached)=&is_cached_new('ltitools',$cdom);
+    if (defined($cached)) {
+        if (ref($result) eq 'HASH') {
+            %ltitools = %{$result};
+        }
+    } else {
+        my %domconfig = &get_dom('configuration',['ltitools'],$cdom);
+        if (ref($domconfig{'ltitools'}) eq 'HASH') {
+            %ltitools = %{$domconfig{'ltitools'}};
+            my %encdomconfig = &get_dom('encconfig',['ltitools'],$cdom);
+            if (ref($encdomconfig{'ltitools'}) eq 'HASH') {
+                foreach my $id (keys(%ltitools)) {
+                    if (ref($encdomconfig{'ltitools'}{$id}) eq 'HASH') {
+                        foreach my $item ('key','secret') {
+                            $ltitools{$id}{$item} = $encdomconfig{'ltitools'}{$id}{$item};
+                        }
+                    }
+                }
+            }
+        }
+        my $cachetime = 24*60*60;
+        &do_cache_new('ltitools',$cdom,\%ltitools,$cachetime);
+    }
+    return %ltitools;
+}
+
 sub get_numsuppfiles {
     my ($cnum,$cdom,$ignorecache)=@_;
     my $hashid=$cnum.':'.$cdom;
@@ -11669,7 +11820,7 @@ sub metadata {
     # if it is a non metadata possible uri return quickly
     if (($uri eq '') || 
 	(($uri =~ m|^/*adm/|) && 
-	     ($uri !~ m|^adm/includes|) && ($uri !~ m{/(smppg|bulletinboard)$})) ||
+	     ($uri !~ m|^adm/includes|) && ($uri !~ m{/(smppg|bulletinboard|ext\.tool)$})) ||
         ($uri =~ m|/$|) || ($uri =~ m|/.meta$|) || ($uri =~ m{^/*uploaded/.+\.sequence$})) {
 	return undef;
     }
@@ -12256,18 +12407,16 @@ sub symbverify {
 
     if (tie(%bighash,'GDBM_File',$env{'request.course.fn'}.'.db',
                             &GDBM_READER(),0640)) {
-        my $noclutter;
         if (($thisurl =~ m{^/adm/wrapper/ext/}) || ($thisurl =~ m{^ext/})) {
             $thisurl =~ s/\?.+$//;
             if ($map =~ m{^uploaded/.+\.page$}) {
                 $thisurl =~ s{^(/adm/wrapper|)/ext/}{http://};
                 $thisurl =~ s{^\Qhttp://https://\E}{https://};
-                $noclutter = 1;
             }
         }
         my $ids;
-        if ($noclutter) {
-            $ids=$bighash{'ids_'.$thisurl};
+        if ($map =~ m{^uploaded/.+\.page$}) {
+            $ids=$bighash{'ids_'.&clutter_with_no_wrapper($thisurl)};
         } else {
             $ids=$bighash{'ids_'.&clutter($thisurl)};
         }
@@ -13013,9 +13162,10 @@ sub repcopy_userfile {
     my $request;
     $uri=~s/^\///;
     my $homeserver = &homeserver($cnum,$cdom);
+    my $hostname = &hostname($homeserver);
     my $protocol = $protocol{$homeserver};
     $protocol = 'http' if ($protocol ne 'https');
-    $request=new HTTP::Request('GET',$protocol.'://'.&hostname($homeserver).'/raw/'.$uri);
+    $request=new HTTP::Request('GET',$protocol.'://'.$hostname.'/raw/'.$uri);
     my $response=$ua->request($request,$transferfile);
 # did it work?
     if ($response->is_error()) {
@@ -13039,9 +13189,10 @@ sub tokenwrapper {
 	$file=~s|(\?\.*)*$||;
         &appenv({"userfile.$udom/$uname/$file" => $env{'request.course.id'}});
         my $homeserver = &homeserver($uname,$udom);
+        my $hostname = &hostname($homeserver);
         my $protocol = $protocol{$homeserver};
         $protocol = 'http' if ($protocol ne 'https');
-        return $protocol.'://'.&hostname($homeserver).'/'.$uri.
+        return $protocol.'://'.$hostname.'/'.$uri.
                (($uri=~/\?/)?'&':'?').'token='.$token.
                                '&tokenissued='.$perlvar{'lonHostID'};
     } else {
@@ -13057,9 +13208,10 @@ sub getuploaded {
     my ($reqtype,$uri,$cdom,$cnum,$info,$rtncode) = @_;
     $uri=~s/^\///;
     my $homeserver = &homeserver($cnum,$cdom);
+    my $hostname = &hostname($homeserver);
     my $protocol = $protocol{$homeserver};
     $protocol = 'http' if ($protocol ne 'https');
-    $uri = $protocol.'://'.&hostname($homeserver).'/raw/'.$uri;
+    $uri = $protocol.'://'.$hostname.'/raw/'.$uri;
     my $ua=new LWP::UserAgent;
     my $request=new HTTP::Request($reqtype,$uri);
     my $response=$ua->request($request);
@@ -13234,6 +13386,45 @@ sub shared_institution {
     return $same_intdom;
 }
 
+sub uses_sts {
+    my ($ignore_cache) = @_;
+    my $lonhost = $perlvar{'lonHostID'};
+    my $hostname = &hostname($lonhost);
+    my $sts_on;
+    if ($protocol{$lonhost} eq 'https') {
+        my $cachetime = 12*3600;
+        if (!$ignore_cache) {
+            ($sts_on,my $cached)=&is_cached_new('stspolicy',$lonhost);
+            if (defined($cached)) {
+                return $sts_on;
+            }
+        }
+        my $ua=new LWP::UserAgent;
+        my $url = $protocol{$lonhost}.'://'.$hostname.'/index.html';
+        my $request=new HTTP::Request('HEAD',$url);
+        my $response=$ua->request($request);
+        if ($response->is_success) {
+            my $has_sts = $response->header('Strict-Transport-Security');
+            if ($has_sts eq '') {
+                $sts_on = 0;
+            } else {
+                if ($has_sts =~ /\Qmax-age=\E(\d+)/) {
+                    my $maxage = $1;
+                    if ($maxage) {
+                        $sts_on = 1;
+                    } else {
+                        $sts_on = 0;
+                    }
+                } else {
+                    $sts_on = 0;
+                }
+            }
+            return &do_cache_new('stspolicy',$lonhost,$sts_on,$cachetime);
+        }
+    }
+    return;
+}
+
 # ------------------------------------------------------------- Declutters URLs
 
 sub declutter {
@@ -13284,6 +13475,8 @@ sub clutter {
 #		&logthis("Got a blank emb style");
 	    }
 	}
+    } elsif ($thisfn =~ m{^/adm/$match_domain/$match_courseid/\d+/ext\.tool$}) {
+        $thisfn='/adm/wrapper'.$thisfn;
     }
     return $thisfn;
 }