From 60d9db248fb7809085db61cb0f5f25fb9c60a539 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Fri, 18 Mar 2022 13:38:07 +0100 Subject: [PATCH 1/2] NEW The backup tools has an "lowmemory" for mysqldump on large database --- htdocs/admin/tools/dolibarr_export.php | 29 ++++++++--- htdocs/core/class/utils.class.php | 67 +++++++++++++++++++------- htdocs/langs/en_US/admin.lang | 2 + 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/htdocs/admin/tools/dolibarr_export.php b/htdocs/admin/tools/dolibarr_export.php index 6ed3bd00863..36136aeef24 100644 --- a/htdocs/admin/tools/dolibarr_export.php +++ b/htdocs/admin/tools/dolibarr_export.php @@ -214,10 +214,6 @@ if (in_array($type, array('mysql', 'mysqli'))) { print '
'; print '
'.$langs->trans("ExportOptions").''; - print '
'; - print ''; - print ''; - print '
'; if (!empty($conf->global->MYSQL_OLD_OPTION_DISABLE_FK)) { print '
'; @@ -239,14 +235,35 @@ if (in_array($type, array('mysql', 'mysqli'))) { print ''; print ''; print ''; - print '
'; + print '

'; - print ''; + print '
'; + print ''; + print ''; + print '
'; + + print ''; print ''; print '
'; + $execmethod = 0; + if (!empty($conf->global->MAIN_EXEC_USE_POPEN)) { + $execmethod = $conf->global->MAIN_EXEC_USE_POPEN; + } + if (empty($execmethod)) { + $execmethod = 1; + } + if ($execmethod == 1) { + // If we use the "exec" method for shell, we ask if we need to use the alternative low memory exec mode. + print ''; + print ''; + print '
'; + } + print ''; diff --git a/htdocs/core/class/utils.class.php b/htdocs/core/class/utils.class.php index f59b248a384..4a4db439822 100644 --- a/htdocs/core/class/utils.class.php +++ b/htdocs/core/class/utils.class.php @@ -189,7 +189,7 @@ class Utils * @param int $usedefault 1=Use default backup profile (Set this to 1 when used as cron) * @param string $file 'auto' or filename to build * @param int $keeplastnfiles Keep only last n files (not used yet) - * @param int $execmethod 0=Use default method (that is 1 by default), 1=Use the PHP 'exec', 2=Use the 'popen' method + * @param int $execmethod 0=Use default method (that is 1 by default), 1=Use the PHP 'exec' - need size of dump in memory, but low memory method is used if GETPOST('lowmemorydump') is set, 2=Use the 'popen' method (low memory method) * @return int 0 if OK, < 0 if KO (this function is used also by cron so only 0 is OK) */ public function dumpDatabase($compression = 'none', $type = 'auto', $usedefault = 1, $file = 'auto', $keeplastnfiles = 0, $execmethod = 0) @@ -278,8 +278,8 @@ class Utils if (!empty($dolibarr_main_db_port)) { $param .= " -P ".$dolibarr_main_db_port; } - if (!GETPOST("use_transaction", "alpha")) { - $param .= " -l --single-transaction"; + if (GETPOST("use_transaction", "alpha")) { + $param .= " --single-transaction"; } if (GETPOST("disable_fk", "alpha") || $usedefault) { $param .= " -K"; @@ -342,17 +342,42 @@ class Utils $handle = ''; + $lowmemorydump = GETPOSTISSET("lowmemorydump", "alpha") ? GETPOST("lowmemorydump") : getDolGlobalString('MAIN_LOW_MEMORY_DUMP'); + // Start call method to execute dump $fullcommandcrypted = $command." ".$paramcrypted." 2>&1"; $fullcommandclear = $command." ".$paramclear." 2>&1"; - if ($compression == 'none') { - $handle = fopen($outputfile, 'w'); - } elseif ($compression == 'gz') { - $handle = gzopen($outputfile, 'w'); - } elseif ($compression == 'bz') { - $handle = bzopen($outputfile, 'w'); - } elseif ($compression == 'zstd') { - $handle = fopen($outputfile, 'w'); + if (!$lowmemorydump) { + if ($compression == 'none') { + $handle = fopen($outputfile, 'w'); + } elseif ($compression == 'gz') { + $handle = gzopen($outputfile, 'w'); + } elseif ($compression == 'bz') { + $handle = bzopen($outputfile, 'w'); + } elseif ($compression == 'zstd') { + $handle = fopen($outputfile, 'w'); + } + } else { + if ($compression == 'none') { + $fullcommandclear .= " > ".$outputfile; + $fullcommandcrypted .= " > ".$outputfile; + $handle = 1; + } elseif ($compression == 'gz') { + $fullcommandclear .= " | gzip > ".$outputfile; + $fullcommandcrypted .= " | gzip > ".$outputfile; + $paramcrypted.=" | gzip"; + $handle = 1; + } elseif ($compression == 'bz') { + $fullcommandclear .= " | bzip2 > ".$outputfile; + $fullcommandcrypted .= " | bzip2 > ".$outputfile; + $paramcrypted.=" | bzip2"; + $handle = 1; + } elseif ($compression == 'zstd') { + $fullcommandclear .= " | zstd > ".$outputfile; + $fullcommandcrypted .= " | zstd > ".$outputfile; + $paramcrypted.=" | zstd"; + $handle = 1; + } } $ok = 0; @@ -374,7 +399,8 @@ class Utils } - // TODO Replace with executeCLI function + // TODO Replace with executeCLI function but + // we must first introduce a low memory mode if ($execmethod == 1) { $output_arr = array(); $retval = null; @@ -394,16 +420,23 @@ class Utils if ($i == 1 && preg_match('/Warning.*Using a password/i', $read)) { continue; } - fwrite($handle, $read.($execmethod == 2 ? '' : "\n")); - if (preg_match('/'.preg_quote('-- Dump completed', '/').'/i', $read)) { - $ok = 1; - } elseif (preg_match('/'.preg_quote('SET SQL_NOTES=@OLD_SQL_NOTES', '/').'/i', $read)) { - $ok = 1; + if (!$lowmemorydump) { + fwrite($handle, $read.($execmethod == 2 ? '' : "\n")); + if (preg_match('/'.preg_quote('-- Dump completed', '/').'/i', $read)) { + $ok = 1; + } elseif (preg_match('/'.preg_quote('SET SQL_NOTES=@OLD_SQL_NOTES', '/').'/i', $read)) { + $ok = 1; + } + } else { + // If we have a result here in lowmemorydump mode, something is strange } } + } elseif ($lowmemorydump) { + $ok = 1; } } } + if ($execmethod == 2) { // With this method, there is no way to get the return code, only output $handlein = popen($fullcommandclear, 'r'); $i = 0; diff --git a/htdocs/langs/en_US/admin.lang b/htdocs/langs/en_US/admin.lang index cffd3532c05..9cae7524022 100644 --- a/htdocs/langs/en_US/admin.lang +++ b/htdocs/langs/en_US/admin.lang @@ -2225,3 +2225,5 @@ PreviousHash=Previous hash LateWarningAfter="Late" warning after TemplateforBusinessCards=Template for a business card in different size InventorySetup= Inventory Setup +ExportUseLowMemoryMode=Use a low memory mode +ExportUseLowMemoryModeHelp=Use the low memory mode to execute the exec of the dump (compression is done through a pipe instead of into the PHP memory). This method does not allow to check that file is completed and error message can't be reported if it fails. From 4ef9d29b9d0514ffc6695e5ca01263d3c881449a Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Fri, 18 Mar 2022 14:09:53 +0100 Subject: [PATCH 2/2] Fix missing constraints --- htdocs/install/mysql/migration/15.0.0-16.0.0.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/htdocs/install/mysql/migration/15.0.0-16.0.0.sql b/htdocs/install/mysql/migration/15.0.0-16.0.0.sql index 5e9a0250a38..2933acd00d7 100644 --- a/htdocs/install/mysql/migration/15.0.0-16.0.0.sql +++ b/htdocs/install/mysql/migration/15.0.0-16.0.0.sql @@ -91,6 +91,10 @@ INSERT INTO llx_c_forme_juridique (fk_pays, code, libelle, active) VALUES (154, INSERT INTO llx_c_forme_juridique (fk_pays, code, libelle, active) VALUES (154, '15419', '626 - Régimen Simplificado de Confianza', 1); +ALTER TABLE llx_partnership ADD UNIQUE INDEX uk_fk_type_fk_soc (fk_type, fk_soc, date_partnership_start); +ALTER TABLE llx_partnership ADD UNIQUE INDEX uk_fk_type_fk_member (fk_type, fk_member, date_partnership_start); + + -- v16 ALTER TABLE llx_projet_task_time ADD COLUMN fk_product integer NULL;