--- loncom/homework/cleanxml/post_xml.pm 2015/12/18 22:07:42 1.4
+++ loncom/homework/cleanxml/post_xml.pm 2016/11/10 21:53:56 1.11
@@ -1,7 +1,7 @@
# The LearningOnline Network
# Third step to clean a file.
#
-# $Id: post_xml.pm,v 1.4 2015/12/18 22:07:42 damieng Exp $
+# $Id: post_xml.pm,v 1.11 2016/11/10 21:53:56 damieng Exp $
#
# Copyright Michigan State University Board of Trustees
#
@@ -41,6 +41,8 @@ use Cwd 'abs_path';
use XML::LibXML;
use HTML::TokeParser; # used to parse sty files
use Tie::IxHash; # for ordered hashes
+use tth;
+use Apache::html_to_xml;
no warnings 'recursion'; # yes, fix_paragraph is using heavy recursion, I know
@@ -50,7 +52,7 @@ my @inline_like_block = ('stringresponse
my @responses = ('stringresponse','optionresponse','numericalresponse','formularesponse','mathresponse','organicresponse','reactionresponse','customresponse','externalresponse','essayresponse','radiobuttonresponse','matchresponse','rankresponse','imageresponse','functionplotresponse');
my @block_html = ('html','head','body','section','h1','h2','h3','h4','h5','h6','div','p','ul','ol','li','table','tbody','tr','td','th','dl','dt','dd','pre','noscript','hr','address','blockquote','object','applet','embed','map','form','fieldset','iframe','center','frameset');
my @no_newline_inside = ('import','parserlib','scriptlib','data','function','label','xlabel','ylabel','tic','text','rectangle','image','title','h1','h2','h3','h4','h5','h6','li','td','p');
-my @preserve_elements = ('script','answer','pre');
+my @preserve_elements = ('script','answer','pre','style');
my @accepting_style = ('section','h1','h2','h3','h4','h5','h6','div','p','li','td','th','dt','dd','pre','blockquote');
my @latex_math = ('\alpha', '\theta', '\omicron', '\tau', '\beta', '\vartheta', '\pi', '\upsilon', '\gamma', '\gamma', '\varpi', '\phi', '\delta', '\kappa', '\rho', '\varphi', '\epsilon', '\lambda', '\varrho', '\chi', '\varepsilon', '\mu', '\sigma', '\psi', '\zeta', '\nu', '\varsigma', '\omega', '\eta', '\xi',
'\Gamma', '\Lambda', '\Sigma', '\Psi', '\Delta', '\Xi', '\Upsilon', '\Omega', '\Theta', '\Pi', '\Phi',
@@ -126,12 +128,16 @@ sub post_xml {
remove_useless_notsolved($root);
+ fix_comments($root);
+
fix_paragraphs_inside($root, \@all_block);
remove_empty_style($root);
fix_empty_lc_elements($root);
+ reduce_empty_p($root);
+
lowercase_attribute_values($root);
replace_numericalresponse_unit_attribute($root);
@@ -150,7 +156,7 @@ sub fix_structure {
# the root element has already been added in pre_xml
my $root = $doc->documentElement;
# inside the root, replace html, problem and library elements by their content
- my @toreplace = ('html','problem','library');
+ my @toreplace = ('html','problem','library','Task');
foreach my $name (@toreplace) {
my @elements = $root->getElementsByTagName($name);
foreach my $element (@elements) {
@@ -552,11 +558,11 @@ sub replace_m {
# Returns the HTML equivalent of LaTeX input, using tth
sub tth {
my ($text) = @_;
- my ($fh, $tmp_path) = tempfile();
- binmode($fh, ':utf8');
- print $fh $text;
- close $fh;
- my $output = `tth -r -w2 -u -y0 < $tmp_path 2>/dev/null`;
+ my $output = &tth::tth($text);
+ my $errorstring = &tth::ttherror();
+ if ($errorstring) {
+ die $errorstring;
+ }
# hopefully the temp file will not be removed before this point (otherwise we should use unlink_on_destroy 0)
$output =~ s/^\s*|\s*$//;
$output =~ s/<div class="p"><!----><\/div>/<br\/>/; # why is tth using such ugly markup for \newline ?
@@ -567,7 +573,7 @@ sub tth {
sub html_to_dom {
my ($text) = @_;
$text = '<root>'.$text.'</root>';
- my $textref = html_to_xml::html_to_xml(\$text);
+ my $textref = Apache::html_to_xml::html_to_xml(\$text);
utf8::upgrade($$textref); # otherwise the XML parser fails when the HTML parser turns into a character
my $dom_doc = XML::LibXML->load_xml(string => $textref);
my $root = $dom_doc->documentElement;
@@ -1808,12 +1814,31 @@ sub remove_useless_notsolved {
}
}
+# Use <pre> for multi-line comments without elements.
+sub fix_comments {
+ my ($root) = @_;
+ my $doc = $root->ownerDocument;
+ my @comments = $root->getElementsByTagName('comment');
+ foreach my $comment (@comments) {
+ my $first = $comment->firstChild;
+ if (defined $first) {
+ if ($first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /\n/ &&
+ !defined $first->nextSibling) {
+ my $pre = $doc->createElement('pre');
+ $comment->removeChild($first);
+ $comment->appendChild($pre);
+ $pre->appendChild($first);
+ }
+ }
+ }
+}
+
# adds a paragraph inside if needed and calls fix_paragraph for all paragraphs (including new ones)
sub fix_paragraphs_inside {
my ($node, $all_block) = @_;
# blocks in which paragrahs will be added:
- my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','solved','notsolved','languageblock','instructorcomment','togglebox','standalone','form');
- my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote');
+ my @blocks_with_p = ('loncapa','library','problem','part','problemtype','window','block','while','postanswerdate','preduedate','languageblock','instructorcomment','togglebox','standalone','body','form');
+ my @fix_p_if_br_or_p = (@responses,'foil','item','text','label','hintgroup','hintpart','hint','web','windowlink','div','li','dd','td','th','blockquote','solved','notsolved');
if ((string_in_array(\@blocks_with_p, $node->nodeName) && paragraph_needed($node)) ||
(string_in_array(\@fix_p_if_br_or_p, $node->nodeName) && paragraph_inside($node))) {
# if non-empty, add paragraphs where needed between all br and remove br
@@ -1833,6 +1858,18 @@ sub fix_paragraphs_inside {
push(@new_children, $doc->createElement('p'));
}
$p = undef;
+ # ignore the next node if it is a br (the paragraph default margin will take as much space)
+ # (ignoring whitespace)
+ while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) {
+ my $next2 = $next->nextSibling;
+ $node->removeChild($next);
+ $next = $next2;
+ }
+ if (defined $next && $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'br') {
+ my $next2 = $next->nextSibling;
+ $node->removeChild($next);
+ $next = $next2;
+ }
} elsif ($child->nodeType == XML_ELEMENT_NODE && string_in_array(\@inline_like_block, $child->nodeName)) {
# inline_like_block: use the paragraph if there is one, otherwise do not create one
if (defined $p) {
@@ -2009,6 +2046,20 @@ sub fix_paragraph {
if (!defined $left || !$left_needs_p) {
$replacement->appendChild($middle);
}
+ # ignore the next node if it is a br (the paragraph default margin will take as much space)
+ my $first_right;
+ if (defined $right) {
+ $first_right = $right->firstChild;
+ # ignore non-nbsp whitespace
+ while (defined $first_right && $first_right->nodeType == XML_TEXT_NODE &&
+ $first_right->nodeValue =~ /^[ \t\f\n\r]*$/) {
+ $first_right = $first_right->nextSibling;
+ }
+ }
+ if (defined $first_right && $first_right->nodeType == XML_ELEMENT_NODE &&
+ $first_right->nodeName eq 'br') {
+ $right->removeChild($first_right);
+ }
} else {
fix_paragraphs_inside($n, $all_block);
$replacement->appendChild($n);
@@ -2246,6 +2297,33 @@ sub fix_empty_lc_elements {
}
}
+# remove consecutive empty paragraphs (they will not show anyway)
+sub reduce_empty_p {
+ my ($node) = @_;
+ my $next;
+ for (my $child=$node->firstChild; defined $child; $child=$next) {
+ $next = $child->nextSibling;
+ while (defined $next && $next->nodeType == XML_TEXT_NODE && $next->nodeValue =~ /^[ \t\f\n\r]*$/) {
+ $next = $next->nextSibling;
+ }
+ if ($child->nodeType == XML_ELEMENT_NODE && $child->nodeName eq 'p' && defined $next &&
+ $next->nodeType == XML_ELEMENT_NODE && $next->nodeName eq 'p') {
+ my $first = $child->firstChild;
+ if (!defined $first || (!defined $first->nextSibling &&
+ $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+ $first = $next->firstChild;
+ if (!defined $first || (!defined $first->nextSibling &&
+ $first->nodeType == XML_TEXT_NODE && $first->nodeValue =~ /^[ \t\f\n\r]*$/)) {
+ $node->removeChild($child);
+ }
+ }
+ }
+ if ($child->nodeType == XML_ELEMENT_NODE) {
+ reduce_empty_p($child);
+ }
+ }
+}
+
# turn some attribute values into lowercase when they should be
sub lowercase_attribute_values {
my ($root) = @_;
@@ -2394,8 +2472,29 @@ sub pretty {
my $type = $node->nodeType;
if ($type == XML_ELEMENT_NODE) {
my $name = $node->nodeName;
- if ((string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) &&
- !string_in_array(\@preserve_elements, $name)) {
+ if (string_in_array(\@preserve_elements, $name)) {
+ # remove newlines at the beginning and the end of preserve elements
+ if (defined $node->firstChild && ($node->firstChild->nodeType == XML_TEXT_NODE ||
+ $node->firstChild->nodeType == XML_CDATA_SECTION_NODE)) {
+ my $text = $node->firstChild->nodeValue;
+ $text =~ s/^\n+//;
+ if ($text eq '') {
+ $node->removeChild($node->firstChild);
+ } else {
+ $node->firstChild->setData($text);
+ }
+ }
+ if (defined $node->lastChild && ($node->lastChild->nodeType == XML_TEXT_NODE ||
+ $node->lastChild->nodeType == XML_CDATA_SECTION_NODE)) {
+ my $text = $node->lastChild->nodeValue;
+ $text =~ s/\n+$//;
+ if ($text eq '') {
+ $node->removeChild($node->lastChild);
+ } else {
+ $node->lastChild->setData($text);
+ }
+ }
+ } elsif (string_in_array($all_block, $name) || string_in_array(\@inline_like_block, $name)) {
# make sure there is a newline at the beginning and at the end if there is anything inside
if (defined $node->firstChild && !string_in_array(\@no_newline_inside, $name)) {
my $first = $node->firstChild;
@@ -2478,26 +2577,6 @@ sub pretty {
if ($text eq '') {
$node->removeChild($node->lastChild);
} else {
- $node->lastChild->setData($text);
- }
- }
- } elsif (string_in_array(\@preserve_elements, $name)) {
- # collapse newlines at the beginning and the end of scripts
- if (defined $node->firstChild && $node->firstChild->nodeType == XML_TEXT_NODE) {
- my $text = $node->firstChild->nodeValue;
- $text =~ s/^\n( *\n)+/\n/;
- if ($text eq '') {
- $node->removeChild($node->firstChild);
- } else {
- $node->firstChild->setData($text);
- }
- }
- if (defined $node->lastChild && $node->lastChild->nodeType == XML_TEXT_NODE) {
- my $text = $node->lastChild->nodeValue;
- $text =~ s/\n( *\n)+$/\n/;
- if ($text eq '') {
- $node->removeChild($node->lastChild);
- } else {
$node->lastChild->setData($text);
}
}