--- loncom/lonnet/perl/lonnet.pm	2020/04/10 13:33:32	1.1172.2.93.2.3
+++ loncom/lonnet/perl/lonnet.pm	2020/07/17 23:36:00	1.1172.2.93.4.13
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1172.2.93.2.3 2020/04/10 13:33:32 raeburn Exp $
+# $Id: lonnet.pm,v 1.1172.2.93.4.13 2020/07/17 23:36:00 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -142,7 +142,7 @@ our @EXPORT = qw(%env);
 sub logtouch {
     my $execdir=$perlvar{'lonDaemons'};
     unless (-e "$execdir/logs/lonnet.log") {	
-	open(my $fh,">>$execdir/logs/lonnet.log");
+	open(my $fh,">>","$execdir/logs/lonnet.log");
 	close $fh;
     }
     my ($wwwuid,$wwwgid)=(getpwnam('www'))[2,3];
@@ -154,7 +154,7 @@ sub logthis {
     my $execdir=$perlvar{'lonDaemons'};
     my $now=time;
     my $local=localtime($now);
-    if (open(my $fh,">>$execdir/logs/lonnet.log")) {
+    if (open(my $fh,">>","$execdir/logs/lonnet.log")) {
 	my $logstring = $local. " ($$): ".$message."\n"; # Keep any \'s in string.
 	print $fh $logstring;
 	close($fh);
@@ -167,7 +167,7 @@ sub logperm {
     my $execdir=$perlvar{'lonDaemons'};
     my $now=time;
     my $local=localtime($now);
-    if (open(my $fh,">>$execdir/logs/lonnet.perm.log")) {
+    if (open(my $fh,">>","$execdir/logs/lonnet.perm.log")) {
 	print $fh "$now:$message:$local\n";
 	close($fh);
     }
@@ -436,7 +436,7 @@ sub reconlonc {
 
     &logthis("Trying to reconnect lonc");
     my $loncfile="$perlvar{'lonDaemons'}/logs/lonc.pid";
-    if (open(my $fh,"<$loncfile")) {
+    if (open(my $fh,"<",$loncfile)) {
 	my $loncpid=<$fh>;
         chomp($loncpid);
         if (kill 0 => $loncpid) {
@@ -476,7 +476,7 @@ sub critical {
             $dumpcount++;
             {
 		my $dfh;
-		if (open($dfh,">$dfilename")) {
+		if (open($dfh,">",$dfilename)) {
 		    print $dfh "$cmd\n"; 
 		    close($dfh);
 		}
@@ -485,7 +485,7 @@ sub critical {
             my $wcmd='';
             {
 		my $dfh;
-		if (open($dfh,"<$dfilename")) {
+		if (open($dfh,"<",$dfilename)) {
 		    $wcmd=<$dfh>; 
 		    close($dfh);
 		}
@@ -601,7 +601,7 @@ sub transfer_profile_to_env {
 
 # ---------------------------------------------------- Check for valid session 
 sub check_for_valid_session {
-    my ($r,$name,$userhashref) = @_;
+    my ($r,$name,$userhashref,$domref) = @_;
     my %cookies=CGI::Cookie->parse($r->header_in('Cookie'));
     if ($name eq '') {
         $name = 'lonID';
@@ -616,7 +616,16 @@ sub check_for_valid_session {
     } else {
         $lonidsdir=$r->dir_config('lonIDsDir');
     }
-    return undef if (!-e "$lonidsdir/$handle.id");
+    if (!-e "$lonidsdir/$handle.id") {
+        if ((ref($domref)) && ($name eq 'lonID') &&
+            ($handle =~ /^($match_username)\_\d+\_($match_domain)\_(.+)$/)) {
+            my ($possuname,$possudom,$possuhome) = ($1,$2,$3);
+            if ((&domain($possudom) ne '') && (&homeserver($possuname,$possudom) eq $possuhome)) {
+                $$domref = $possudom;
+            }
+        }
+        return undef;
+    }
 
     my $opened = open(my $idf,'+<',"$lonidsdir/$handle.id");
     return undef if (!$opened);
@@ -686,16 +695,19 @@ sub appenv {
                 $env{$key}=$newenv->{$key};
             }
         }
-        my $opened = open(my $env_file,'+<',$env{'user.environment'});
-        if ($opened
-	    && &timed_flock($env_file,LOCK_EX)
-	    &&
-	    tie(my %disk_env,'GDBM_File',$env{'user.environment'},
-	        (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
-	    while (my ($key,$value) = each(%{$newenv})) {
-	        $disk_env{$key} = $value;
-	    }
-	    untie(%disk_env);
+        my $lonids = $perlvar{'lonIDsDir'};
+        if ($env{'user.environment'} =~ m{^\Q$lonids/\E$match_username\_\d+\_$match_domain\_[\w\-.]+\.id$}) {
+            my $opened = open(my $env_file,'+<',$env{'user.environment'});
+            if ($opened
+	        && &timed_flock($env_file,LOCK_EX)
+	        &&
+	        tie(my %disk_env,'GDBM_File',$env{'user.environment'},
+	            (&GDBM_WRITER()|&GDBM_NOLOCK()),0640)) {
+	        while (my ($key,$value) = each(%{$newenv})) {
+	            $disk_env{$key} = $value;
+	        }
+	        untie(%disk_env);
+            }
         }
     }
     return 'ok';
@@ -1832,6 +1844,17 @@ sub inst_directory_query {
     my $homeserver = &domain($udom,'primary');
     my $outcome;
     if ($homeserver ne '') {
+        unless ($homeserver eq $perlvar{'lonHostID'}) {
+            if ($srch->{'srchby'} eq 'email') {
+                my $lcrev = &get_server_loncaparev(undef,$homeserver);
+                my ($major,$minor,$subver) = ($lcrev =~ /^\'?(\d+)\.(\d+)\.([\w.\-]+)\'?$/);
+                if (($major eq '' && $minor eq '') || ($major < 2) ||
+                    (($major == 2) && ($minor < 11)) || 
+                    (($major == 2) && ($minor == 11) && ($subver !~ /^2\.B/))) {
+                    return;
+                }
+            }
+        }
 	my $queryid=&reply("querysend:instdirsearch:".
 			   &escape($srch->{'srchby'}).':'.
 			   &escape($srch->{'srchterm'}).':'.
@@ -1873,6 +1896,15 @@ sub usersearch {
     my $query = 'usersearch';
     foreach my $tryserver (keys(%libserv)) {
         if (&host_domain($tryserver) eq $dom) {
+            unless ($tryserver eq $perlvar{'lonHostID'}) {
+                if ($srch->{'srchby'} eq 'email') {
+                    my $lcrev = &get_server_loncaparev(undef,$tryserver);
+                    my ($major,$minor,$subver) = ($lcrev =~ /^\'?(\d+)\.(\d+)\.([\w.\-]+)\'?$/);
+                    next if (($major eq '' && $minor eq '') || ($major < 2) ||
+                             (($major == 2) && ($minor < 11)) || 
+                             (($major == 2) && ($minor == 11) && ($subver !~ /^2\.B/)));
+                }
+            }
             my $host=&hostname($tryserver);
             my $queryid=
                 &reply("querysend:".&escape($query).':'.
@@ -3240,7 +3272,7 @@ sub process_coursefile {
                                  $home);
             }
         } elsif ($action eq 'uploaddoc') {
-            open(my $fh,'>'.$filepath.'/'.$fname);
+            open(my $fh,'>',$filepath.'/'.$fname);
             print $fh $env{'form.'.$source};
             close($fh);
             if ($parser eq 'parse') {
@@ -3298,7 +3330,7 @@ sub store_edited_file {
     ($fpath,$fname) = ($file =~ m|^(.*)/([^/]+)$|);
     $fpath=$docudom.'/'.$docuname.'/'.$fpath;
     my $filepath = &build_filepath($fpath);
-    open(my $fh,'>'.$filepath.'/'.$fname);
+    open(my $fh,'>',$filepath.'/'.$fname);
     print $fh $content;
     close($fh);
     my $home=&homeserver($docuname,$docudom);
@@ -3372,13 +3404,16 @@ sub resizeImage {
 # input: $formname - the contents of the file are in $env{"form.$formname"}
 #                    the desired filename is in $env{"form.$formname.filename"}
 #        $context - possible values: coursedoc, existingfile, overwrite, 
-#                                    canceloverwrite, or ''. 
+#                                    canceloverwrite, scantron or ''.
 #                   if 'coursedoc': upload to the current course
 #                   if 'existingfile': write file to tmp/overwrites directory 
 #                   if 'canceloverwrite': delete file written to tmp/overwrites directory
 #                   $context is passed as argument to &finishuserfileupload
 #        $subdir - directory in userfile to store the file into
-#        $parser - instruction to parse file for objects ($parser = parse)    
+#        $parser - instruction to parse file for objects ($parser = parse) or
+#                  if context is 'scantron', $parser is hashref of csv column mapping
+#                  (e.g.,{ PaperID => 0, LastName => 1, FirstName => 2, ID => 3, 
+#                          Section => 4, CODE => 5, FirstQuestion => 9 }).    
 #        $allfiles - reference to hash for embedded objects
 #        $codebase - reference to hash for codebase of java objects
 #        $desuname - username for permanent storage of uploaded file
@@ -3414,12 +3449,12 @@ sub userfileupload {
                          '_'.$env{'user.domain'}.'/pending';
         } elsif (($context eq 'existingfile') || ($context eq 'canceloverwrite')) {
             my ($docuname,$docudom);
-            if ($destudom) {
+            if ($destudom =~ /^$match_domain$/) {
                 $docudom = $destudom;
             } else {
                 $docudom = $env{'user.domain'};
             }
-            if ($destuname) {
+            if ($destuname =~ /^$match_username$/) {
                 $docuname = $destuname;
             } else {
                 $docuname = $env{'user.name'};
@@ -3449,7 +3484,7 @@ sub userfileupload {
                 mkdir($fullpath,0777);
             }
         }
-        open(my $fh,'>'.$fullpath.'/'.$fname);
+        open(my $fh,'>',$fullpath.'/'.$fname);
         print $fh $env{'form.'.$formname};
         close($fh);
         if ($context eq 'existingfile') {
@@ -3524,7 +3559,7 @@ sub finishuserfileupload {
 
 # Save the file
     {
-	if (!open(FH,'>'.$filepath.'/'.$file)) {
+	if (!open(FH,'>',$filepath.'/'.$file)) {
 	    &logthis('Failed to create '.$filepath.'/'.$file);
 	    print STDERR ('Failed to create '.$filepath.'/'.$file."\n");
 	    return '/adm/notfound.html';
@@ -3568,7 +3603,7 @@ sub finishuserfileupload {
             }
         }
     }
-    if ($parser eq 'parse') {
+    if (($context ne 'scantron') && ($parser eq 'parse')) {
         if ((ref($mimetype)) && ($$mimetype eq 'text/html')) {
             my $parse_result = &extract_embedded_items($filepath.'/'.$file,
                                                        $allfiles,$codebase);
@@ -3577,12 +3612,16 @@ sub finishuserfileupload {
 	   	         ' for embedded media: '.$parse_result); 
             }
         }
+    } elsif (($context eq 'scantron') && (ref($parser) eq 'HASH')) {
+        my $format = $env{'form.scantron_format'};
+        &bubblesheet_converter($docudom,$filepath.'/'.$file,$parser,$format);
     }
     if (($thumbwidth =~ /^\d+$/) && ($thumbheight =~ /^\d+$/)) {
         my $input = $filepath.'/'.$file;
         my $output = $filepath.'/'.'tn-'.$file;
         my $thumbsize = $thumbwidth.'x'.$thumbheight;
-        system("convert -sample $thumbsize $input $output");
+        my @args = ('convert','-sample',$thumbsize,$input,$output);
+        system({$args[0]} @args);
         if (-e $filepath.'/'.'tn-'.$file) {
             $fetchthumb  = 1; 
         }
@@ -3816,6 +3855,246 @@ sub embedded_dependency {
     return;
 }
 
+sub bubblesheet_converter {
+    my ($cdom,$fullpath,$config,$format) = @_;
+    if ((&domain($cdom) ne '') &&
+        ($fullpath =~ m{^\Q$perlvar{'lonDocRoot'}/userfiles/$cdom/\E$match_courseid/scantron_orig}) &&
+        (-e $fullpath) && (ref($config) eq 'HASH') && ($format ne '')) {
+        my (%csvcols,%csvoptions);
+        if (ref($config->{'fields'}) eq 'HASH') {
+            %csvcols = %{$config->{'fields'}};
+        }
+        if (ref($config->{'options'}) eq 'HASH') {
+            %csvoptions = %{$config->{'options'}};
+        }
+        my %csvbynum = reverse(%csvcols);
+        my %scantronconf = &get_scantron_config($format,$cdom);
+        if (keys(%scantronconf)) {
+            my %bynum = (
+                          $scantronconf{CODEstart} => 'CODEstart',
+                          $scantronconf{IDstart}   => 'IDstart',
+                          $scantronconf{PaperID}   => 'PaperID',
+                          $scantronconf{FirstName} => 'FirstName',
+                          $scantronconf{LastName}  => 'LastName',
+                          $scantronconf{Qstart}    => 'Qstart',
+                        );
+            my @ordered;
+            foreach my $item (sort { $a <=> $b } keys(%bynum)) {
+                push(@ordered,$bynum{$item});
+            }
+            my %mapstart = (
+                              CODEstart => 'CODE',
+                              IDstart   => 'ID',
+                              PaperID   => 'PaperID',
+                              FirstName => 'FirstName',
+                              LastName  => 'LastName',
+                              Qstart    => 'FirstQuestion',
+                           );
+            my %maplength = (
+                              CODEstart => 'CODElength',
+                              IDstart   => 'IDlength',
+                              PaperID   => 'PaperIDlength',
+                              FirstName => 'FirstNamelength',
+                              LastName  => 'LastNamelength',
+            );
+            if (open(my $fh,'<',$fullpath)) {
+                my $output;
+                my %lettdig = &letter_to_digits();
+                my %diglett = reverse(%lettdig);
+                my $numletts = scalar(keys(%lettdig));
+                my $num = 0;
+                while (my $line=<$fh>) {
+                    $num ++;
+                    next if (($num == 1) && ($csvoptions{'hdr'} == 1));
+                    $line =~ s{[\r\n]+$}{};
+                    my %found;
+                    my @values = split(/,/,$line);
+                    my ($qstart,$record);
+                    for (my $i=0; $i<@values; $i++) {
+                        if ((($qstart ne '') && ($i > $qstart)) ||
+                            ($csvbynum{$i} eq 'FirstQuestion')) {
+                            if ($values[$i] eq '') {
+                                $values[$i] = $scantronconf{'Qoff'};
+                            } elsif ($scantronconf{'Qon'} eq 'number') {
+                                if ($values[$i] =~ /^[A-Ja-j]$/) {
+                                    $values[$i] = $lettdig{uc($values[$i])};
+                                }
+                            } elsif ($scantronconf{'Qon'} eq 'letter') {
+                                if ($values[$i] =~ /^[0-9]$/) {
+                                    $values[$i] = $diglett{$values[$i]};
+                                }
+                            } else {
+                                if ($values[$i] =~ /^[0-9A-Ja-j]$/) {
+                                    my $digit;
+                                    if ($values[$i] =~ /^[A-Ja-j]$/) {
+                                        $digit = $lettdig{uc($values[$i])}-1;
+                                        if ($values[$i] eq 'J') {
+                                            $digit += $numletts;
+                                        }
+                                    } elsif ($values[$i] =~ /^[0-9]$/) {
+                                        $digit = $values[$i]-1;
+                                        if ($values[$i] eq '0') {
+                                            $digit += $numletts;
+                                        }
+                                    }
+                                    my $qval='';
+                                    for (my $j=0; $j<$scantronconf{'Qlength'}; $j++) {
+                                        if ($j == $digit) {
+                                            $qval .= $scantronconf{'Qon'};
+                                        } else {
+                                            $qval .= $scantronconf{'Qoff'};
+                                        }
+                                    }
+                                    $values[$i] = $qval;
+                                }
+                            }
+                            if (length($values[$i]) > $scantronconf{'Qlength'}) {
+                                $values[$i] = substr($values[$i],0,$scantronconf{'Qlength'});
+                            }
+                            my $numblank = $scantronconf{'Qlength'} - length($values[$i]);
+                            if ($numblank > 0) {
+                                 $values[$i] .= ($scantronconf{'Qoff'} x $numblank);
+                            }
+                            if ($csvbynum{$i} eq 'FirstQuestion') {
+                                $qstart = $i;
+                                $found{$csvbynum{$i}} = $values[$i];
+                            } else {
+                                $found{'FirstQuestion'} .= $values[$i];
+                            }
+                        } elsif (exists($csvbynum{$i})) {
+                            if ($csvoptions{'rem'}) {
+                                $values[$i] =~ s/^\s+//;
+                            }
+                            if (($csvbynum{$i} eq 'PaperID') && ($csvoptions{'pad'})) {
+                                while (length($values[$i]) < $scantronconf{$maplength{$csvbynum{$i}}}) {
+                                    $values[$i] = '0'.$values[$i];
+                                }
+                            }
+                            $found{$csvbynum{$i}} = $values[$i];
+                        }
+                    }
+                    foreach my $item (@ordered) {
+                        my $currlength = 1+length($record);
+                        my $numspaces = $scantronconf{$item} - $currlength;
+                        if ($numspaces > 0) {
+                            $record .= (' ' x $numspaces);
+                        }
+                        if (($mapstart{$item} ne '') && (exists($found{$mapstart{$item}}))) {
+                            unless ($item eq 'Qstart') {
+                                if (length($found{$mapstart{$item}}) > $scantronconf{$maplength{$item}}) {
+                                    $found{$mapstart{$item}} = substr($found{$mapstart{$item}},0,$scantronconf{$maplength{$item}});
+                                }
+                            }
+                            $record .= $found{$mapstart{$item}};
+                        }
+                    }
+                    $output .= "$record\n";
+                }
+                close($fh);
+                if ($output) {
+                    if (open(my $fh,'>',$fullpath)) {
+                        print $fh $output;
+                        close($fh);
+                    }
+                }
+            }
+        }
+        return;
+    }
+}
+
+sub letter_to_digits {
+    my %lettdig = (
+                    A => 1,
+                    B => 2,
+                    C => 3,
+                    D => 4,
+                    E => 5,
+                    F => 6,
+                    G => 7,
+                    H => 8,
+                    I => 9,
+                    J => 0,
+                  );
+    return %lettdig;
+}
+
+sub get_scantron_config {
+    my ($which,$cdom) = @_;
+    my @lines = &get_scantronformat_file($cdom);
+    my %config;
+    #FIXME probably should move to XML it has already gotten a bit much now
+    foreach my $line (@lines) {
+        my ($name,$descrip)=split(/:/,$line);
+        if ($name ne $which ) { next; }
+        chomp($line);
+        my @config=split(/:/,$line);
+        $config{'name'}=$config[0];
+        $config{'description'}=$config[1];
+        $config{'CODElocation'}=$config[2];
+        $config{'CODEstart'}=$config[3];
+        $config{'CODElength'}=$config[4];
+        $config{'IDstart'}=$config[5];
+        $config{'IDlength'}=$config[6];
+        $config{'Qstart'}=$config[7];
+        $config{'Qlength'}=$config[8];
+        $config{'Qoff'}=$config[9];
+        $config{'Qon'}=$config[10];
+        $config{'PaperID'}=$config[11];
+        $config{'PaperIDlength'}=$config[12];
+        $config{'FirstName'}=$config[13];
+        $config{'FirstNamelength'}=$config[14];
+        $config{'LastName'}=$config[15];
+        $config{'LastNamelength'}=$config[16];
+        $config{'BubblesPerRow'}=$config[17];
+        last;
+    }
+    return %config;
+}
+
+sub get_scantronformat_file {
+    my ($cdom) = @_;
+    if ($cdom eq '') {
+        $cdom= $env{'course.'.$env{'request.course.id'}.'.domain'};
+    }
+    my %domconfig = &get_dom('configuration',['scantron'],$cdom);
+    my $gottab = 0;
+    my @lines;
+    if (ref($domconfig{'scantron'}) eq 'HASH') {
+        if ($domconfig{'scantron'}{'scantronformat'} ne '') {
+            my $formatfile = &getfile($perlvar{'lonDocRoot'}.$domconfig{'scantron'}{'scantronformat'});
+            if ($formatfile ne '-1') {
+                @lines = split("\n",$formatfile,-1);
+                $gottab = 1;
+            }
+        }
+    }
+    if (!$gottab) {
+        my $confname = $cdom.'-domainconfig';
+        my $default = $perlvar{'lonDocRoot'}.'/res/'.$cdom.'/'.$confname.'/default.tab';
+        my $formatfile = &getfile($default);
+        if ($formatfile ne '-1') {
+            @lines = split("\n",$formatfile,-1);
+            $gottab = 1;
+        }
+    }
+    if (!$gottab) {
+        my @domains = &current_machine_domains();
+        if (grep(/^\Q$cdom\E$/,@domains)) {
+            if (open(my $fh,'<',$perlvar{'lonTabDir'}.'/scantronformat.tab')) {
+                @lines = <$fh>;
+                close($fh);
+            }
+        } else {
+            if (open(my $fh,'<',$perlvar{'lonTabDir'}.'/default_scantronformat.tab')) {
+                @lines = <$fh>;
+                close($fh);
+            }
+        }
+    }
+    return @lines;
+}
+
 sub removeuploadedurl {
     my ($url)=@_;	
     my (undef,undef,$udom,$uname,$fname)=split('/',$url,5);    
@@ -4542,7 +4821,7 @@ sub postannounce {
 
 sub getannounce {
 
-    if (open(my $fh,$perlvar{'lonDocRoot'}.'/announcement.txt')) {
+    if (open(my $fh,"<",$perlvar{'lonDocRoot'}.'/announcement.txt')) {
 	my $announcement='';
 	while (my $line = <$fh>) { $announcement .= $line; }
 	close($fh);
@@ -6228,7 +6507,7 @@ sub currentdump {
    #
    my %returnhash=();
    #
-   if ($rep eq 'unknown_cmd') {
+   if ($rep eq 'unknown_cmd') { 
        # an old lond will not know currentdump
        # Do a dump and make it look like a currentdump
        my @tmp = &dumpstore($courseid,$sdom,$sname,'.');
@@ -8161,7 +8440,7 @@ sub fetch_enrollment_query {
                         if ($xml_classlist =~ /^error/) {
                             &logthis('fetch_enrollment_query - autoretrieve error: '.$xml_classlist.' for '.$filename.' from server: '.$homeserver.' '.$context.' '.$cnum);
                         } else {
-                            if ( open(FILE,">$destname") ) {
+                            if ( open(FILE,">",$destname) ) {
                                 print FILE &unescape($xml_classlist);
                                 close(FILE);
                             } else {
@@ -8190,7 +8469,7 @@ sub get_query_reply {
     for (1..$loopmax) {
 	sleep($sleep);
         if (-e $replyfile.'.end') {
-	    if (open(my $fh,$replyfile)) {
+	    if (open(my $fh,"<",$replyfile)) {
 		$reply = join('',<$fh>);
 		close($fh);
 	   } else { return 'error: reply_file_error'; }
@@ -8582,6 +8861,33 @@ sub auto_validate_class_sec {
     return $response;
 }
 
+sub auto_validate_instclasses {
+    my ($cdom,$cnum,$owners,$classesref) = @_;
+    my ($homeserver,%validations);
+    $homeserver = &homeserver($cnum,$cdom);
+    unless ($homeserver eq 'no_host') {
+        my $ownerlist;
+        if (ref($owners) eq 'ARRAY') {
+            $ownerlist = join(',',@{$owners});
+        } else {
+            $ownerlist = $owners;
+        }
+        if (ref($classesref) eq 'HASH') {
+            my $classes = &freeze_escape($classesref);
+            my $response=&reply('autovalidateinstclasses:'.&escape($ownerlist).
+                                ':'.$cdom.':'.$classes,$homeserver);
+            unless ($response =~ /(con_lost|error|no_such_host|refused)/) {
+                my @items = split(/&/,$response);
+                foreach my $item (@items) {
+                    my ($key,$value) = split('=',$item);
+                    $validations{&unescape($key)} = &thaw_unescape($value);
+                }
+            }
+        }
+    }
+    return %validations;
+}
+
 sub auto_crsreq_update {
     my ($cdom,$cnum,$crstype,$action,$ownername,$ownerdomain,$fullname,$title,
         $code,$accessstart,$accessend,$inbound) = @_;
@@ -9517,14 +9823,19 @@ sub writecoursepref {
 
 sub createcourse {
     my ($udom,$description,$url,$course_server,$nonstandard,$inst_code,
-        $course_owner,$crstype,$cnum,$context,$category)=@_;
+        $course_owner,$crstype,$cnum,$context,$category,$callercontext)=@_;
     $url=&declutter($url);
     my $cid='';
     if ($context eq 'requestcourses') {
         my $can_create = 0;
         my ($ownername,$ownerdom) = split(':',$course_owner);
         if ($udom eq $ownerdom) {
-            if (&usertools_access($ownername,$ownerdom,$category,undef,
+            my $reload;
+            if (($callercontext eq 'auto') &&
+               ($ownerdom eq $env{'user.domain'}) && ($ownername eq $env{'user.name'})) {
+                $reload = 'reload';
+            }
+            if (&usertools_access($ownername,$ownerdom,$category,$reload,
                                   $context)) {
                 $can_create = 1;
             }
@@ -9815,7 +10126,7 @@ sub save_selected_files {
     my ($user, $path, @files) = @_;
     my $filename = $user."savedfiles";
     my @other_files = &files_not_in_path($user, $path);
-    open (OUT, '>'.$tmpdir.$filename);
+    open (OUT,'>',LONCAPA::tempdir().$filename);
     foreach my $file (@files) {
         print (OUT $env{'form.currentpath'}.$file."\n");
     }
@@ -9829,7 +10140,7 @@ sub save_selected_files {
 sub clear_selected_files {
     my ($user) = @_;
     my $filename = $user."savedfiles";
-    open (OUT, '>'.LONCAPA::tempdir().$filename);
+    open (OUT,'>',LONCAPA::tempdir().$filename);
     print (OUT undef);
     close (OUT);
     return ("ok");    
@@ -9839,7 +10150,7 @@ sub files_in_path {
     my ($user, $path) = @_;
     my $filename = $user."savedfiles";
     my %return_files;
-    open (IN, '<'.LONCAPA::tempdir().$filename);
+    open (IN,'<',LONCAPA::tempdir().$filename);
     while (my $line_in = <IN>) {
         chomp ($line_in);
         my @paths_and_file = split (m!/!, $line_in);
@@ -9861,7 +10172,7 @@ sub files_not_in_path {
     my $filename = $user."savedfiles";
     my @return_files;
     my $path_part;
-    open(IN, '<'.LONCAPA::.$filename);
+    open(IN,'<',LONCAPA::tempdir().$filename);
     while (my $line = <IN>) {
         #ok, I know it's clunky, but I want it to work
         my @paths_and_file = split(m|/|, $line);
@@ -10549,40 +10860,33 @@ sub resdata {
     return undef;
 }
 
-sub get_domain_lti {
-    my ($cdom,$context) = @_;
-    my ($name,%lti);
-    if ($context eq 'consumer') {
-        $name = 'ltitools';
-    } elsif ($context eq 'provider') {
-        $name = 'lti';
-    } else {
-        return %lti;
-    }
-    my ($result,$cached)=&is_cached_new($name,$cdom);
+sub get_domain_ltitools {
+    my ($cdom) = @_;
+    my %ltitools;
+    my ($result,$cached)=&is_cached_new('ltitools',$cdom);
     if (defined($cached)) {
         if (ref($result) eq 'HASH') {
-            %lti = %{$result};
+            %ltitools = %{$result};
         }
     } else {
-        my %domconfig = &get_dom('configuration',[$name],$cdom);
-        if (ref($domconfig{$name}) eq 'HASH') {
-            %lti = %{$domconfig{$name}};
-            my %encdomconfig = &get_dom('encconfig',[$name],$cdom);
-            if (ref($encdomconfig{$name}) eq 'HASH') {
-                foreach my $id (keys(%lti)) {
-                    if (ref($encdomconfig{$name}{$id}) eq 'HASH') {
+        my %domconfig = &get_dom('configuration',['ltitools'],$cdom);
+        if (ref($domconfig{'ltitools'}) eq 'HASH') {
+            %ltitools = %{$domconfig{'ltitools'}};
+            my %encdomconfig = &get_dom('encconfig',['ltitools'],$cdom);
+            if (ref($encdomconfig{'ltitools'}) eq 'HASH') {
+                foreach my $id (keys(%ltitools)) {
+                    if (ref($encdomconfig{'ltitools'}{$id}) eq 'HASH') {
                         foreach my $item ('key','secret') {
-                            $lti{$id}{$item} = $encdomconfig{$name}{$id}{$item};
+                            $ltitools{$id}{$item} = $encdomconfig{'ltitools'}{$id}{$item};
                         }
                     }
                 }
             }
         }
         my $cachetime = 24*60*60;
-        &do_cache_new($name,$cdom,\%lti,$cachetime);
+        &do_cache_new('ltitools',$cdom,\%ltitools,$cachetime);
     }
-    return %lti;
+    return %ltitools;
 }
 
 sub get_numsuppfiles {
@@ -11033,6 +11337,7 @@ sub add_prefix_and_part {
 
 my %metaentry;
 my %importedpartids;
+my %importedrespids;
 sub metadata {
     my ($uri,$what,$liburi,$prefix,$depthcount)=@_;
     $uri=&declutter($uri);
@@ -11060,9 +11365,11 @@ sub metadata {
     }
     {
 # Imported parts would go here
-        my %importedids=();
-        my @origfileimportpartids=();
+        my @origfiletagids=();
         my $importedparts=0;
+
+# Imported responseids would go here
+        my $importedresponses=0;
 #
 # Is this a recursive call for a library?
 #
@@ -11157,8 +11464,37 @@ sub metadata {
                         my $dir=$filename;
                         $dir=~s|[^/]*$||;
                         $location=&filelocation($dir,$location);
-                       
+
+                        my $importid=$token->[2]->{'id'};
                         my $importmode=$token->[2]->{'importmode'};
+#
+# Check metadata for imported file to
+# see if it contained response items
+#
+                        my %currmetaentry = %metaentry;
+                        my $libresponseorder = &metadata($location,'responseorder');
+                        my $origfile;
+                        if ($libresponseorder ne '') {
+                            if ($#origfiletagids<0) {
+                                undef(%importedrespids);
+                                undef(%importedpartids);
+                            }
+                            @{$importedrespids{$importid}} = split(/\s*,\s*/,$libresponseorder);
+                            if (@{$importedrespids{$importid}} > 0) {
+                                $importedresponses = 1;
+# We need to get the original file and the imported file to get the response order correct
+# Load and inspect original file
+                                if ($#origfiletagids<0) {
+                                    my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                    $origfile=&getfile($origfilelocation);
+                                    @origfiletagids=($origfile=~/<((?:\w+)response|import|part)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                                }
+                            }
+                        }
+# Do not overwrite contents of %metaentry hash for resource itself with 
+# hash populated for imported library file
+                        %metaentry = %currmetaentry;
+                        undef(%currmetaentry);
                         if ($importmode eq 'problem') {
 # Import as problem/response
                            $unikey=&add_prefix_and_part($prefix,$token->[2]->{'part'});
@@ -11167,12 +11503,15 @@ sub metadata {
                            $importedparts=1;
 # We need to get the original file and the imported file to get the part order correct
 # Good news: we do not need to worry about nested libraries, since parts cannot be nested
-# Load and inspect original file
-                           if ($#origfileimportpartids<0) {
-                              undef(%importedpartids);
-                              my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
-                              my $origfile=&getfile($origfilelocation);
-                              @origfileimportpartids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+# Load and inspect original file if we didn't do that already
+                           if ($#origfiletagids<0) {
+                               undef(%importedrespids);
+                               undef(%importedpartids);
+                               if ($origfile eq '') {
+                                   my $origfilelocation=$perlvar{'lonDocRoot'}.&clutter($uri);
+                                   $origfile=&getfile($origfilelocation);
+                                   @origfiletagids=($origfile=~/<(part|import)[^>]*id\s*=\s*[\"\']([^\"\']+)[\"\'][^>]*>/gs);
+                               }
                            }
 
 # Load and inspect imported file
@@ -11286,20 +11625,48 @@ sub metadata {
 	    grep { ! $seen{$_} ++ } (split(',',$metaentry{':packages'}));
 	$metaentry{':packages'} = join(',',@uniq_packages);
 
-        if ($importedparts) {
+        if (($importedresponses) || ($importedparts)) {
+            if ($importedparts) {
 # We had imported parts and need to rebuild partorder
-           $metaentry{':partorder'}='';
-           $metathesekeys{'partorder'}=1;
-           for (my $index=0;$index<$#origfileimportpartids;$index+=2) {
-               if ($origfileimportpartids[$index] eq 'part') {
-# original part, part of the problem
-                  $metaentry{':partorder'}.=','.$origfileimportpartids[$index+1];
-               } else {
-# we have imported parts at this position
-                  $metaentry{':partorder'}.=','.$importedpartids{$origfileimportpartids[$index+1]};
-               }
-           }
-           $metaentry{':partorder'}=~s/^\,//;
+                $metaentry{':partorder'}='';
+                $metathesekeys{'partorder'}=1;
+            }
+            if ($importedresponses) {
+# We had imported responses and need to rebuild responseorder
+                $metaentry{':responseorder'}='';
+                $metathesekeys{'responseorder'}=1;
+            }
+            for (my $index=0;$index<$#origfiletagids;$index+=2) {
+                my $origid = $origfiletagids[$index+1];
+                if ($origfiletagids[$index] eq 'part') {
+# Original part, part of the problem
+                    if ($importedparts) {
+                        $metaentry{':partorder'}.=','.$origid;
+                    }
+                } elsif ($origfiletagids[$index] eq 'import') {
+                    if ($importedparts) {
+# We have imported parts at this position
+                        $metaentry{':partorder'}.=','.$importedpartids{$origid};
+                    }
+                    if ($importedresponses) {
+# We have imported responses at this position
+                        if (ref($importedrespids{$origid}) eq 'ARRAY') {
+                            $metaentry{':responseorder'}.=','.join(',',map { $origid.'_'.$_ } @{$importedrespids{$origid}});
+                        }
+                    }
+                } else {
+# Original response item, part of the problem
+                    if ($importedresponses) {
+                        $metaentry{':responseorder'}.=','.$origid;
+                    }
+                }
+            }
+            if ($importedparts) {
+                $metaentry{':partorder'}=~s/^\,//;
+            }
+            if ($importedresponses) {
+                $metaentry{':responseorder'}=~s/^\,//;
+            }
         }
 
 	$metaentry{':keys'} = join(',',keys(%metathesekeys));
@@ -12387,7 +12754,7 @@ sub readfile {
     my $file = shift;
     if ( (! -e $file ) || ($file eq '') ) { return -1; };
     my $fh;
-    open($fh,"<$file");
+    open($fh,"<",$file);
     my $a='';
     while (my $line = <$fh>) { $a .= $line; }
     return $a;
@@ -12500,7 +12867,7 @@ sub machine_ids {
 
 sub additional_machine_domains {
     my @domains;
-    open(my $fh,"<$perlvar{'lonTabDir'}/expected_domains.tab");
+    open(my $fh,"<","$perlvar{'lonTabDir'}/expected_domains.tab");
     while( my $line = <$fh>) {
         $line =~ s/\s//g;
         push(@domains,$line);
@@ -12646,7 +13013,7 @@ sub get_dns {
     }
 
     my %alldns;
-    open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab");
     foreach my $dns (<$config>) {
 	next if ($dns !~ /^\^(\S*)/x);
         my $line = $1;
@@ -12674,7 +13041,7 @@ sub get_dns {
     close($config);
     my $which = (split('/',$url))[3];
     &logthis("unable to contact DNS defaulting to on disk file dns_$which.tab\n");
-    open($config,"<$perlvar{'lonTabDir'}/dns_$which.tab");
+    open($config,"<","$perlvar{'lonTabDir'}/dns_$which.tab");
     my @content = <$config>;
     &$func(\@content,$hashref);
     return;
@@ -12767,7 +13134,7 @@ sub fetch_dns_checksums {
 	my ($ignore_cache,$nocache) = @_;
 	&get_dns('/adm/dns/domain',\&parse_domain_tab,$ignore_cache,$nocache);
 	my $fh;
-	if (open($fh,"<".$perlvar{'lonTabDir'}.'/domain.tab')) {
+	if (open($fh,"<",$perlvar{'lonTabDir'}.'/domain.tab')) {
 	    my @lines = <$fh>;
 	    &parse_domain_tab(\@lines);
 	}
@@ -12819,8 +13186,23 @@ sub fetch_dns_checksums {
 	    my ($id,$domain,$role,$name,$protocol,$intdom)=split(/:/,$configline);
 	    $name=~s/\s//g;
 	    if ($id && $domain && $role && $name) {
+                if ((exists($hostname{$id})) && ($hostname{$id} ne '')) {
+                    my $curr = $hostname{$id};
+                    my $skip;
+                    if (ref($name_to_host{$curr}) eq 'ARRAY') {
+                        if (($curr eq $name) && (@{$name_to_host{$curr}} == 1)) {
+                            $skip = 1;
+                        } else {
+                            @{$name_to_host{$curr}} = grep { $_ ne $id } @{$name_to_host{$curr}};
+                        }
+                    }
+                    unless ($skip) {
+                        push(@{$name_to_host{$name}},$id);
+                    }
+                } else {
+                    push(@{$name_to_host{$name}},$id);
+                }
 		$hostname{$id}=$name;
-		push(@{$name_to_host{$name}}, $id);
 		$hostdom{$id}=$domain;
 		if ($role eq 'library') { $libserv{$id}=$name; }
                 if (defined($protocol)) {
@@ -12853,7 +13235,7 @@ sub fetch_dns_checksums {
     sub load_hosts_tab {
 	my ($ignore_cache,$nocache) = @_;
 	&get_dns('/adm/dns/hosts',\&parse_hosts_tab,$ignore_cache,$nocache);
-	open(my $config,"<$perlvar{'lonTabDir'}/hosts.tab");
+	open(my $config,"<","$perlvar{'lonTabDir'}/hosts.tab");
 	my @config = <$config>;
 	&parse_hosts_tab(\@config);
 	close($config);
@@ -13119,7 +13501,7 @@ sub all_loncaparevs {
 {
     sub load_loncaparevs {
         if (-e "$perlvar{'lonTabDir'}/loncaparevs.tab") {
-            if (open(my $config,"<$perlvar{'lonTabDir'}/loncaparevs.tab")) {
+            if (open(my $config,"<","$perlvar{'lonTabDir'}/loncaparevs.tab")) {
                 while (my $configline=<$config>) {
                     chomp($configline);
                     my ($hostid,$loncaparev)=split(/:/,$configline);
@@ -13135,7 +13517,7 @@ sub all_loncaparevs {
 {
     sub load_serverhomeIDs {
         if (-e "$perlvar{'lonTabDir'}/serverhomeIDs.tab") {
-            if (open(my $config,"<$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
+            if (open(my $config,"<","$perlvar{'lonTabDir'}/serverhomeIDs.tab")) {
                 while (my $configline=<$config>) {
                     chomp($configline);
                     my ($name,$id)=split(/:/,$configline);
@@ -13160,7 +13542,7 @@ BEGIN {
 
 # ------------------------------------------------------ Read spare server file
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/spare.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/spare.tab");
 
     while (my $configline=<$config>) {
        chomp($configline);
@@ -13174,7 +13556,7 @@ BEGIN {
 }
 # ------------------------------------------------------------ Read permissions
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/roles.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/roles.tab");
 
     while (my $configline=<$config>) {
 	chomp($configline);
@@ -13188,7 +13570,7 @@ BEGIN {
 
 # -------------------------------------------- Read plain texts for permissions
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/rolesplain.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/rolesplain.tab");
 
     while (my $configline=<$config>) {
 	chomp($configline);
@@ -13208,7 +13590,7 @@ BEGIN {
 
 # ---------------------------------------------------------- Read package table
 {
-    open(my $config,"<$perlvar{'lonTabDir'}/packages.tab");
+    open(my $config,"<","$perlvar{'lonTabDir'}/packages.tab");
 
     while (my $configline=<$config>) {
 	if ($configline !~ /\S/ || $configline=~/^#/) { next; }
@@ -13254,7 +13636,7 @@ BEGIN {
 # ---------------------------------------------------------- Read managers table
 {
     if (-e "$perlvar{'lonTabDir'}/managers.tab") {
-        if (open(my $config,"<$perlvar{'lonTabDir'}/managers.tab")) {
+        if (open(my $config,"<","$perlvar{'lonTabDir'}/managers.tab")) {
             while (my $configline=<$config>) {
                 chomp($configline);
                 next if ($configline =~ /^\#/);
@@ -13920,6 +14302,88 @@ Returns:
 
 =back
 
+=head2 Bubblesheet Configuration
+
+=over 4
+
+=item *
+
+get_scantron_config($which)
+
+$which - the name of the configuration to parse from the file.
+
+Parses and returns the bubblesheet configuration line selected as a
+hash of configuration file fields.
+
+
+Returns:
+    If the named configuration is not in the file, an empty
+    hash is returned.
+
+    a hash with the fields
+      name         - internal name for the this configuration setup
+      description  - text to display to operator that describes this config
+      CODElocation - if 0 or the string 'none'
+                          - no CODE exists for this config
+                     if -1 || the string 'letter'
+                          - a CODE exists for this config and is
+                            a string of letters
+                     Unsupported value (but planned for future support)
+                          if a positive integer
+                               - The CODE exists as the first n items from
+                                 the question section of the form
+                          if the string 'number'
+                               - The CODE exists for this config and is
+                                 a string of numbers
+      CODEstart   - (only matter if a CODE exists) column in the line where
+                     the CODE starts
+      CODElength  - length of the CODE
+      IDstart     - column where the student/employee ID starts
+      IDlength    - length of the student/employee ID info
+      Qstart      - column where the information from the bubbled
+                    'questions' start
+      Qlength     - number of columns comprising a single bubble line from
+                    the sheet. (usually either 1 or 10)
+      Qon         - either a single character representing the character used
+                    to signal a bubble was chosen in the positional setup, or
+                    the string 'letter' if the letter of the chosen bubble is
+                    in the final, or 'number' if a number representing the
+                    chosen bubble is in the file (1->A 0->J)
+      Qoff        - the character used to represent that a bubble was
+                    left blank
+      PaperID     - if the scanning process generates a unique number for each
+                    sheet scanned the column that this ID number starts in
+      PaperIDlength - number of columns that comprise the unique ID number
+                      for the sheet of paper
+      FirstName   - column that the first name starts in
+      FirstNameLength - number of columns that the first name spans
+      LastName    - column that the last name starts in
+      LastNameLength - number of columns that the last name spans
+      BubblesPerRow - number of bubbles available in each row used to
+                      bubble an answer. (If not specified, 10 assumed).
+
+
+=item *
+
+get_scantronformat_file($cdom)
+
+$cdom - the course's domain (optional); if not supplied, uses
+domain for current $env{'request.course.id'}.
+
+Returns an array containing lines from the scantron format file for
+the domain of the course.
+
+If a url for a custom.tab file is listed in domain's configuration.db,
+lines are from this file.
+
+Otherwise, if a default.tab has been published in RES space by the
+domainconfig user, lines are from this file.
+
+Otherwise, fall back to getting lines from the legacy file on the
+local server:  /home/httpd/lonTabs/default_scantronformat.tab
+
+=back
+
 =head2 Resource Subroutines
 
 =over 4
@@ -14611,6 +15075,7 @@ userspace, probably shouldn't be called
   formname: same as for userfileupload()
   fname: filename (including subdirectories) for the file
   parser: if 'parse', will parse (html) file to extract references to objects, links etc.
+          if hashref, and context is scantron, will convert csv format to standard format
   allfiles: reference to hash used to store objects found by parser
   codebase: reference to hash used for codebases of java objects found by parser
   thumbwidth: width (pixels) of thumbnail to be created for uploaded image