From a796122f7b0e1050df2fe5ac635107e4aa3750be Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Thu, 14 May 2020 12:28:17 +0200 Subject: [PATCH] Fix to solve feedbacks of https://www.dolibarr.fr/forum/t/email-collector-avec-office-365-bug-et-contournement/32726 --- .../class/emailcollector.class.php | 20 +++++++++++++++---- .../install/mysql/migration/11.0.0-12.0.0.sql | 2 ++ .../llx_emailcollector_emailcollector.sql | 3 ++- htdocs/langs/en_US/admin.lang | 1 + 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/htdocs/emailcollector/class/emailcollector.class.php b/htdocs/emailcollector/class/emailcollector.class.php index 11cf2477720..969b83cac5a 100644 --- a/htdocs/emailcollector/class/emailcollector.class.php +++ b/htdocs/emailcollector/class/emailcollector.class.php @@ -100,8 +100,9 @@ class EmailCollector extends CommonObject 'ref' => array('type'=>'varchar(128)', 'label'=>'Ref', 'enabled'=>1, 'visible'=>1, 'notnull'=>1, 'showoncombobox'=>1, 'index'=>1, 'position'=>10, 'searchall'=>1, 'help'=>'Example: MyCollector1'), 'label' => array('type'=>'varchar(255)', 'label'=>'Label', 'visible'=>1, 'enabled'=>1, 'position'=>30, 'notnull'=>-1, 'searchall'=>1, 'help'=>'Example: My Email collector'), 'description' => array('type'=>'text', 'label'=>'Description', 'visible'=>-1, 'enabled'=>1, 'position'=>60, 'notnull'=>-1), - 'host' => array('type'=>'varchar(255)', 'label'=>'EMailHost', 'visible'=>1, 'enabled'=>1, 'position'=>100, 'notnull'=>1, 'searchall'=>1, 'comment'=>"IMAP server", 'help'=>'Example: imap.gmail.com'), - 'login' => array('type'=>'varchar(128)', 'label'=>'Login', 'visible'=>1, 'enabled'=>1, 'position'=>101, 'notnull'=>-1, 'index'=>1, 'comment'=>"IMAP login", 'help'=>'Example: myaccount@gmail.com'), + 'host' => array('type'=>'varchar(255)', 'label'=>'EMailHost', 'visible'=>1, 'enabled'=>1, 'position'=>90, 'notnull'=>1, 'searchall'=>1, 'comment'=>"IMAP server", 'help'=>'Example: imap.gmail.com'), + 'hostcharset' => array('type'=>'varchar(16)', 'label'=>'HostCharset', 'visible'=>-1, 'enabled'=>1, 'position'=>91, 'notnull'=>0, 'searchall'=>0, 'comment'=>"IMAP server charset", 'help'=>'Example: "UTF-8" (May be "US-ASCII" with some Office365)'), + 'login' => array('type'=>'varchar(128)', 'label'=>'Login', 'visible'=>1, 'enabled'=>1, 'position'=>101, 'notnull'=>-1, 'index'=>1, 'comment'=>"IMAP login", 'help'=>'Example: myaccount@gmail.com'), 'password' => array('type'=>'password', 'label'=>'Password', 'visible'=>-1, 'enabled'=>1, 'position'=>102, 'notnull'=>-1, 'comment'=>"IMAP password", 'help'=>'WithGMailYouCanCreateADedicatedPassword'), 'source_directory' => array('type'=>'varchar(255)', 'label'=>'MailboxSourceDirectory', 'visible'=>-1, 'enabled'=>1, 'position'=>103, 'notnull'=>1, 'default' => 'Inbox', 'help'=>'Example: INBOX'), //'filter' => array('type'=>'text', 'label'=>'Filter', 'visible'=>1, 'enabled'=>1, 'position'=>105), @@ -173,6 +174,7 @@ class EmailCollector extends CommonObject public $host; + public $hostcharset; public $login; public $password; public $source_directory; @@ -985,9 +987,10 @@ class EmailCollector extends CommonObject $nbemailprocessed = 0; $nbemailok = 0; $nbactiondone = 0; + $charset = ($this->hostcharset ? $this->hostcharset : "UTF-8"); // Scan IMAP inbox - $arrayofemail = imap_search($connection, $search, null, "UTF-8"); + $arrayofemail = imap_search($connection, $search, null, $charset); if ($arrayofemail === false) { // Nothing found or search string not understood @@ -1086,6 +1089,7 @@ class EmailCollector extends CommonObject } $header = imap_fetchheader($connection, $imapemail, 0); + $header = preg_replace('/\r\n\s+/m', ' ', $header); // When a header line is on several lines, merge lines $matches = array(); preg_match_all('/([^: ]+): (.+?(?:\r\n\s(?:.+?))*)\r\n/m', $header, $matches); $headers = array_combine($matches[1], $matches[2]); @@ -1132,7 +1136,10 @@ class EmailCollector extends CommonObject // Can use also imap_mime_header_decode($str) // Can use also mb_decode_mimeheader($str) // Can use also iconv_mime_decode($str, ICONV_MIME_DECODE_CONTINUE_ON_ERROR, 'UTF-8') - if (function_exists('imap_mime_header_decode')) { + if (function_exists('iconv_mime_decode')) { + $overview[0]->subject = iconv_mime_decode($overview[0]->subject, ICONV_MIME_DECODE_CONTINUE_ON_ERROR, 'UTF-8'); + } + elseif (function_exists('imap_mime_header_decode')) { $elements = imap_mime_header_decode($overview[0]->subject); $newstring = ''; if (!empty($elements)) { @@ -1146,6 +1153,8 @@ class EmailCollector extends CommonObject elseif (function_exists('mb_decode_mimeheader')) { $overview[0]->subject = mb_decode_mimeheader($overview[0]->subject); } + // Removed emojis + $overview[0]->subject = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $overview[0]->subject); // Parse IMAP email structure global $htmlmsg, $plainmsg, $charset, $attachments; @@ -1153,6 +1162,9 @@ class EmailCollector extends CommonObject //$htmlmsg,$plainmsg,$charset,$attachments $messagetext = $plainmsg ? $plainmsg : dol_string_nohtmltag($htmlmsg, 0); + // Removed emojis + $messagetext = preg_replace('/[\x{10000}-\x{10FFFF}]/u', "\xEF\xBF\xBD", $messagetext); + /*var_dump($plainmsg); var_dump($htmlmsg); var_dump($messagetext);*/ diff --git a/htdocs/install/mysql/migration/11.0.0-12.0.0.sql b/htdocs/install/mysql/migration/11.0.0-12.0.0.sql index 8a68e763160..7daf41a05d3 100644 --- a/htdocs/install/mysql/migration/11.0.0-12.0.0.sql +++ b/htdocs/install/mysql/migration/11.0.0-12.0.0.sql @@ -285,3 +285,5 @@ ALTER TABLE llx_prelevement_facture ADD COLUMN fk_facture_fourn INTEGER NULL; ALTER TABLE llx_menu MODIFY COLUMN module varchar(255); UPDATE llx_actioncomm SET fk_action = 50 where fk_action = 40 AND code = 'TICKET_MSG'; + +ALTER TABLE llx_emailcollector_emailcollector ADD COLUMN hostcharset varchar(16) DEFAULT 'UTF-8'; diff --git a/htdocs/install/mysql/tables/llx_emailcollector_emailcollector.sql b/htdocs/install/mysql/tables/llx_emailcollector_emailcollector.sql index 10f3e4f8b9b..87581f418e8 100644 --- a/htdocs/install/mysql/tables/llx_emailcollector_emailcollector.sql +++ b/htdocs/install/mysql/tables/llx_emailcollector_emailcollector.sql @@ -21,7 +21,8 @@ CREATE TABLE llx_emailcollector_emailcollector( ref varchar(128) NOT NULL, label varchar(255), description text, - host varchar(255), + host varchar(255), + hostcharset varchar(16) DEFAULT 'UTF-8', login varchar(128), password varchar(128), source_directory varchar(255) NOT NULL, diff --git a/htdocs/langs/en_US/admin.lang b/htdocs/langs/en_US/admin.lang index 1bd7f3d0297..5ce79a12231 100644 --- a/htdocs/langs/en_US/admin.lang +++ b/htdocs/langs/en_US/admin.lang @@ -40,6 +40,7 @@ WebUserGroup=Web server user/group NoSessionFound=Your PHP configuration seems to not allow listing of active sessions. The directory used to save sessions (%s) may be protected (for example by OS permissions or by PHP directive open_basedir). DBStoringCharset=Database charset to store data DBSortingCharset=Database charset to sort data +HostCharset=Host charset ClientCharset=Client charset ClientSortingCharset=Client collation WarningModuleNotActive=Module %s must be enabled