--- loncom/lonnet/perl/lonnet.pm	2019/01/27 23:16:36	1.1403
+++ loncom/lonnet/perl/lonnet.pm	2019/04/24 01:44:46	1.1407
@@ -1,7 +1,7 @@
 # The LearningOnline Network
 # TCP networking package
 #
-# $Id: lonnet.pm,v 1.1403 2019/01/27 23:16:36 raeburn Exp $
+# $Id: lonnet.pm,v 1.1407 2019/04/24 01:44:46 raeburn Exp $
 #
 # Copyright Michigan State University Board of Trustees
 #
@@ -77,7 +77,7 @@ use CGI::Cookie;
 
 use Encode;
 
-use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir
+use vars qw(%perlvar %spareid %pr %prp $memcache %packagetab $tmpdir $deftex
             $_64bit %env %protocol %loncaparevs %serverhomeIDs %needsrelease
             %managerstab);
 
@@ -101,6 +101,7 @@ use LONCAPA::Configuration;
 use LONCAPA::lonmetadata;
 use LONCAPA::Lond;
 use LONCAPA::LWPReq;
+use LONCAPA::transliterate;
 
 use File::Copy;
 
@@ -2720,6 +2721,29 @@ sub course_portal_url {
     return $firsturl;
 }
 
+# --------------------------------------------- Get domain config for passwords
+
+sub get_passwdconf {
+    my ($dom) = @_;
+    my (%passwdconf,$gotconf,$lookup);
+    my ($result,$cached)=&is_cached_new('passwdconf',$dom);
+    if (defined($cached)) {
+        if (ref($result) eq 'HASH') {
+            %passwdconf = %{$result};
+            $gotconf = 1;
+        }
+    }
+    unless ($gotconf) {
+        my %domconfig = &get_dom('configuration',['passwords'],$dom);
+        if (ref($domconfig{'passwords'}) eq 'HASH') {
+            %passwdconf = %{$domconfig{'passwords'}};
+        }
+        my $cachetime = 24*60*60;
+        &do_cache_new('passwdconf',$dom,\%passwdconf,$cachetime);
+    }
+    return %passwdconf;
+}
+
 # --------------------------------------------------- Assign a key to a student
 
 sub assign_access_key {
@@ -3856,6 +3880,9 @@ sub clean_filename {
     }
 # Replace spaces by underscores
     $fname=~s/\s+/\_/g;
+# Transliterate non-ascii text to ascii
+    my $lang = &Apache::lonlocal::current_language();
+    $fname = &LONCAPA::transliterate::fname_to_ascii($fname,$lang);
 # Replace all other weird characters by nothing
     $fname=~s{[^/\w\.\-]}{}g;
 # Replace all .\d. sequences with _\d. so they no longer look like version
@@ -3863,6 +3890,7 @@ sub clean_filename {
     $fname=~s/\.(\d+)(?=\.)/_$1/g;
     return $fname;
 }
+
 # This Function checks if an Image's dimensions exceed either $resizewidth (width) 
 # or $resizeheight (height) - both pixels. If so, the image is scaled to produce an 
 # image with the same aspect ratio as the original, but with dimensions which do 
@@ -3937,6 +3965,14 @@ sub userfileupload {
     $fname=&clean_filename($fname);
     # See if there is anything left
     unless ($fname) { return 'error: no uploaded file'; }
+    # If filename now begins with a . prepend unix timestamp _ milliseconds
+    if ($fname =~ /^\./) {
+        my ($s,$usec) = &gettimeofday();
+        while (length($usec) < 6) {
+            $usec = '0'.$usec;
+        }
+        $fname = $s.'_'.substr($usec,0,3).$fname;
+    }
     # Files uploaded to help request form, or uploaded to "create course" page are handled differently
     if ((($formname eq 'screenshot') && ($subdir eq 'helprequests')) ||
         (($formname eq 'coursecreatorxml') && ($subdir eq 'batchupload')) ||
@@ -4361,7 +4397,13 @@ sub bubblesheet_converter {
     if ((&domain($cdom) ne '') &&
         ($fullpath =~ m{^\Q$perlvar{'lonDocRoot'}/userfiles/$cdom/\E$match_courseid/scantron_orig}) &&
         (-e $fullpath) && (ref($config) eq 'HASH') && ($format ne '')) {
-        my %csvcols = %{$config};
+        my (%csvcols,%csvoptions);
+        if (ref($config->{'fields'}) eq 'HASH') {  
+            %csvcols = %{$config->{'fields'}};
+        }
+        if (ref($config->{'options'}) eq 'HASH') {
+            %csvoptions = %{$config->{'options'}};
+        }
         my %csvbynum = reverse(%csvcols);
         my %scantronconf = &get_scantron_config($format,$cdom);
         if (keys(%scantronconf)) {
@@ -4397,7 +4439,10 @@ sub bubblesheet_converter {
                 my %lettdig = &letter_to_digits();
                 my %diglett = reverse(%lettdig);
                 my $numletts = scalar(keys(%lettdig));
+                my $num = 0;
                 while (my $line=<$fh>) {
+                    $num ++;
+                    next if (($num == 1) && ($csvoptions{'hdr'} == 1));
                     $line =~ s{[\r\n]+$}{};
                     my %found;
                     my @values = split(/,/,$line);
@@ -4454,8 +4499,10 @@ sub bubblesheet_converter {
                                 $found{'FirstQuestion'} .= $values[$i];
                             }
                         } elsif (exists($csvbynum{$i})) {
-                            $values[$i] =~ s/^\s+//;
-                            if ($csvbynum{$i} eq 'PaperID') {
+                            if ($csvoptions{'rem'}) {
+                                $values[$i] =~ s/^\s+//;
+                            }
+                            if (($csvbynum{$i} eq 'PaperID') && ($csvoptions{'pad'})) {
                                 while (length($values[$i]) < $scantronconf{$maplength{$csvbynum{$i}}}) {
                                     $values[$i] = '0'.$values[$i];
                                 }
@@ -14800,6 +14847,11 @@ BEGIN {
 
 }
 
+# ------------- set default texengine (domain default overrides this)
+{
+    $deftex = LONCAPA::texengine();
+}
+
 $memcache=new Cache::Memcached({'servers'           => ['127.0.0.1:11211'],
 				'compress_threshold'=> 20_000,
  			        });
@@ -15525,7 +15577,6 @@ Returns:
                       for the sheet of paper
       FirstName   - column that the first name starts in
       FirstNameLength - number of columns that the first name spans
-
       LastName    - column that the last name starts in
       LastNameLength - number of columns that the last name spans
       BubblesPerRow - number of bubbles available in each row used to