From 06a0d922ccd1b16f658fe1f5946ef52383d57baa Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Mon, 16 Aug 2021 22:01:58 +0200 Subject: [PATCH] NEW Can use the result_mode of mysqli driver. Save memory for list count --- htdocs/comm/action/list.php | 33 +++++++++++++++---- htdocs/core/db/Database.interface.php | 15 +++++---- htdocs/core/db/mysqli.class.php | 9 ++--- htdocs/core/db/pgsql.class.php | 5 +-- htdocs/core/db/sqlite3.class.php | 5 +-- htdocs/debugbar/class/TraceableDB.php | 15 +++++---- .../modulebuilder/template/myobject_list.php | 8 +++++ 7 files changed, 62 insertions(+), 28 deletions(-) diff --git a/htdocs/comm/action/list.php b/htdocs/comm/action/list.php index 9c587a934e2..7d1eff2586c 100644 --- a/htdocs/comm/action/list.php +++ b/htdocs/comm/action/list.php @@ -519,19 +519,40 @@ $sql .= $hookmanager->resPrint; $sql .= $db->order($sortfield, $sortorder); -$nbtotalofrecords = ''; +$nbtotalofrecords = ''; // TODO We can set and use an optimized request in $sqlforcount with no fields and no useless join to calculate nb of records if (empty($conf->global->MAIN_DISABLE_FULL_SCANLIST)) { - // TODO Set and use an optimized request in $sqlforcount with no fields and no useless join to caluclate nb of records - $result = $db->query($sql); - $nbtotalofrecords = $db->num_rows($result); + /* This old method to get and count full list returns all record so use a high amount of memory. + $resql = $db->query($sql); + $nbtotalofrecords = $db->num_rows($resql); + */ + /* The new method does not consume memory on mysql (not tested on pgsql) */ + $resql = $db->query($sql, 0, 'auto', 1); + while ($db->fetch_object($resql)) { + $nbtotalofrecords++; + } if (($page * $limit) > $nbtotalofrecords) { // if total resultset is smaller then paging size (filtering), goto and load page 0 $page = 0; $offset = 0; } + $db->free($resql); } -$sql .= $db->plimit($limit + 1, $offset); -//print $sql; +// if total of record found is smaller than limit, no need to do paging and to restart another select with limits set. +if (is_numeric($nbtotalofrecords) && ($limit > $nbtotalofrecords || empty($limit))) { + $num = $nbtotalofrecords; +} else { + if ($limit) { + $sql .= $db->plimit($limit + 1, $offset); + } + + $resql = $db->query($sql); + if (!$resql) { + dol_print_error($db); + exit; + } + + $num = $db->num_rows($resql); +} dol_syslog("comm/action/list.php", LOG_DEBUG); $resql = $db->query($sql); diff --git a/htdocs/core/db/Database.interface.php b/htdocs/core/db/Database.interface.php index 9996d09f036..50e013ce8b7 100644 --- a/htdocs/core/db/Database.interface.php +++ b/htdocs/core/db/Database.interface.php @@ -214,13 +214,14 @@ interface Database /** * Execute a SQL request and return the resultset * - * @param string $query SQL query string - * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). - * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. - * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param string $query SQL query string + * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). + * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. + * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param int $result_mode Result mode * @return resource Resultset of answer */ - public function query($query, $usesavepoint = 0, $type = 'auto'); + public function query($query, $usesavepoint = 0, $type = 'auto', $result_mode = 0); /** * Connexion to server @@ -493,8 +494,8 @@ interface Database /** * Returns the current line (as an object) for the resultset cursor * - * @param resource $resultset Cursor of the desired request - * @return Object Object result line or false if KO or end of cursor + * @param resource $resultset Cursor of the desired request + * @return Object Object result line or false if KO or end of cursor */ public function fetch_object($resultset); // phpcs:enable diff --git a/htdocs/core/db/mysqli.class.php b/htdocs/core/db/mysqli.class.php index 36974d29218..bef1209dd84 100644 --- a/htdocs/core/db/mysqli.class.php +++ b/htdocs/core/db/mysqli.class.php @@ -262,9 +262,10 @@ class DoliDBMysqli extends DoliDB * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param int $result_mode Result mode * @return bool|mysqli_result Resultset of answer */ - public function query($query, $usesavepoint = 0, $type = 'auto') + public function query($query, $usesavepoint = 0, $type = 'auto', $result_mode = 0) { global $conf, $dolibarr_main_db_readonly; @@ -289,9 +290,9 @@ class DoliDBMysqli extends DoliDB if (!$this->database_name) { // Ordre SQL ne necessitant pas de connexion a une base (exemple: CREATE DATABASE) - $ret = $this->db->query($query); + $ret = $this->db->query($query, $result_mode); } else { - $ret = $this->db->query($query); + $ret = $this->db->query($query, $result_mode); } if (!preg_match("/^COMMIT/i", $query) && !preg_match("/^ROLLBACK/i", $query)) { @@ -316,7 +317,7 @@ class DoliDBMysqli extends DoliDB // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps /** - * Renvoie la ligne courante (comme un objet) pour le curseur resultset + * Returns the current line (as an object) for the resultset cursor * * @param mysqli_result $resultset Curseur de la requete voulue * @return object|null Object result line or null if KO or end of cursor diff --git a/htdocs/core/db/pgsql.class.php b/htdocs/core/db/pgsql.class.php index 0513226ac31..5245a9dac3c 100644 --- a/htdocs/core/db/pgsql.class.php +++ b/htdocs/core/db/pgsql.class.php @@ -494,9 +494,10 @@ class DoliDBPgsql extends DoliDB * @param string $query SQL query string * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param int $result_mode Result mode (not used with pgsql) * @return false|resource Resultset of answer */ - public function query($query, $usesavepoint = 0, $type = 'auto') + public function query($query, $usesavepoint = 0, $type = 'auto', $result_mode = 0) { global $conf, $dolibarr_main_db_readonly; @@ -570,7 +571,7 @@ class DoliDBPgsql extends DoliDB // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps /** - * Renvoie la ligne courante (comme un objet) pour le curseur resultset + * Returns the current line (as an object) for the resultset cursor * * @param resource $resultset Curseur de la requete voulue * @return false|object Object result line or false if KO or end of cursor diff --git a/htdocs/core/db/sqlite3.class.php b/htdocs/core/db/sqlite3.class.php index 395155973be..53bcb6806dc 100644 --- a/htdocs/core/db/sqlite3.class.php +++ b/htdocs/core/db/sqlite3.class.php @@ -393,9 +393,10 @@ class DoliDBSqlite3 extends DoliDB * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollbock to savepoint if error (this allow to have some request with errors inside global transactions). * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param int $result_mode Result mode (not used with sqlite) * @return SQLite3Result Resultset of answer */ - public function query($query, $usesavepoint = 0, $type = 'auto') + public function query($query, $usesavepoint = 0, $type = 'auto', $result_mode = 0) { global $conf, $dolibarr_main_db_readonly; @@ -504,7 +505,7 @@ class DoliDBSqlite3 extends DoliDB // phpcs:disable PEAR.NamingConventions.ValidFunctionName.ScopeNotCamelCaps /** - * Renvoie la ligne courante (comme un objet) pour le curseur resultset + * Returns the current line (as an object) for the resultset cursor * * @param SQLite3Result $resultset Curseur de la requete voulue * @return false|object Object result line or false if KO or end of cursor diff --git a/htdocs/debugbar/class/TraceableDB.php b/htdocs/debugbar/class/TraceableDB.php index af11f4c1d67..85dd3080512 100644 --- a/htdocs/debugbar/class/TraceableDB.php +++ b/htdocs/debugbar/class/TraceableDB.php @@ -288,17 +288,18 @@ class TraceableDB extends DoliDB /** * Execute a SQL request and return the resultset * - * @param string $query SQL query string - * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). - * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. - * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) - * @return resource Resultset of answer + * @param string $query SQL query string + * @param int $usesavepoint 0=Default mode, 1=Run a savepoint before and a rollback to savepoint if error (this allow to have some request with errors inside global transactions). + * Note that with Mysql, this parameter is not used as Myssql can already commit a transaction even if one request is in error, without using savepoints. + * @param string $type Type of SQL order ('ddl' for insert, update, select, delete or 'dml' for create, alter...) + * @param int $result_mode Result mode + * @return resource Resultset of answer */ - public function query($query, $usesavepoint = 0, $type = 'auto') + public function query($query, $usesavepoint = 0, $type = 'auto', $result_mode = 0) { $this->startTracing(); - $resql = $this->db->query($query, $usesavepoint, $type); + $resql = $this->db->query($query, $usesavepoint, $type, $result_mode); $this->endTracing($query, $resql); diff --git a/htdocs/modulebuilder/template/myobject_list.php b/htdocs/modulebuilder/template/myobject_list.php index 1e319e18ec2..259b4570615 100644 --- a/htdocs/modulebuilder/template/myobject_list.php +++ b/htdocs/modulebuilder/template/myobject_list.php @@ -349,12 +349,20 @@ $sql .= $db->order($sortfield, $sortorder); // Count total nb of records $nbtotalofrecords = ''; if (empty($conf->global->MAIN_DISABLE_FULL_SCANLIST)) { + /* This old method to get and count full list returns all record so use a high amount of memory. $resql = $db->query($sql); $nbtotalofrecords = $db->num_rows($resql); + */ + /* The new method does not consume memory on mysql (not tested on pgsql) */ + $resql = $db->query($sql, 0, 'auto', 1); + while ($db->fetch_object($resql)) { + $nbtotalofrecords++; + } if (($page * $limit) > $nbtotalofrecords) { // if total of record found is smaller than page * limit, goto and load page 0 $page = 0; $offset = 0; } + $db->free($resql); } // if total of record found is smaller than limit, no need to do paging and to restart another select with limits set. if (is_numeric($nbtotalofrecords) && ($limit > $nbtotalofrecords || empty($limit))) {