[PATCH] Bug 8378 - <fine> syntax broken NFC and charset utf8

[wajasu]

NFC normalize enqueued letters and add content-type charset=utf-8

This prevents utf8 codes from causing mysql to truncate the 'content'
from the point of certain codes, when stored in the message_queue table.
This was happenning with the currency symbol generated by
Locale::Currency:Format currency_format routine. NFC normalization
was only done on the attachment content with its content-type
containing "text", as in text/plain.

For emails AND attachments, the charset="utf-8" was added to the
content-type so mail clients would correctly iterate the utf8 codes,
thus preventing mobijake.
---
 C4/Letters.pm                    | 17 +++++++++++++++--
 misc/cronjobs/overdue_notices.pl |  3 ++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/C4/Letters.pm b/C4/Letters.pm
index 4347d8b..b428a91 100644
--- a/C4/Letters.pm
+++ b/C4/Letters.pm
@@ -31,6 +31,7 @@ use C4::SMS;
 use C4::Debug;
 use Date::Calc qw( Add_Delta_Days );
 use Encode;
+use Unicode::Normalize;
 use Carp;
 
 use vars qw($VERSION @ISA @EXPORT @EXPORT_OK %EXPORT_TAGS);
@@ -762,6 +763,12 @@ sub EnqueueLetter {
         return;
     }
 
+    # It was found that the some utf8 codes, cause the text to be truncated from that point onward when stored,
+    # so we normalize utf8 with NFC so that mysql will store 'all' of the content in its TEXT column type
+    # Note: It is also done in _add_attachments accordingly.
+    $params->{'letter'}->{'title'} = NFC($params->{'letter'}->{'title'});     # subject
+    $params->{'letter'}->{'content'} = NFC($params->{'letter'}->{'content'});
+
     # If we have any attachments we should encode then into the body.
     if ( $params->{'attachments'} ) {
         $params->{'letter'} = _add_attachments(
@@ -932,11 +939,17 @@ sub _add_attachments {
     $message->attach(
         Type => $letter->{'content-type'} || 'TEXT',
         Data => $letter->{'is_html'}
-            ? _wrap_html($letter->{'content'}, $letter->{'title'})
-            : $letter->{'content'},
+            ? _wrap_html($letter->{'content'}, NFC($letter->{'title'}))
+            : NFC($letter->{'content'}),
     );
 
     foreach my $attachment ( @$attachments ) {
+
+        if ($attachment->{'content'} =~ m/text/o) { # NFC normailze any "text" related  content-type attachments
+            $attachment->{'content'} = NFC($attachment->{'content'});
+        }
+        $attachment->{'filename'} = NFC($attachment->{'filename'});
+
         $message->attach(
             Type     => $attachment->{'type'},
             Data     => $attachment->{'content'},
diff --git a/misc/cronjobs/overdue_notices.pl b/misc/cronjobs/overdue_notices.pl
index 0dc83b1..e83bd39 100755
--- a/misc/cronjobs/overdue_notices.pl
+++ b/misc/cronjobs/overdue_notices.pl
@@ -609,11 +609,12 @@ END_SQL
             
         my $attachment = {
             filename => defined $csvfilename ? 'attachment.csv' : 'attachment.txt',
-            type => 'text/plain',
+            type => 'text/plain; charset="utf-8"',
             content => $content, 
         };
 
         my $letter = {
+            'content-type' => 'text/plain; charset="utf-8"',
             title   => 'Overdue Notices',
             content => 'These messages were not sent directly to the patrons.',
         };
-- 
1.7.11.4


------=_Part_3913_14838644.1345745357015--