From 2e2cc553505f51ae86a4965a5428691005553476 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 29 May 2018 12:15:28 +0200 Subject: [PATCH 1/4] Backport fix of file checker generation --- build/generate_filelist_xml.php | 137 +++++++++++++++++++++++++------- 1 file changed, 109 insertions(+), 28 deletions(-) diff --git a/build/generate_filelist_xml.php b/build/generate_filelist_xml.php index b227b544bab..5f54c95d7e0 100755 --- a/build/generate_filelist_xml.php +++ b/build/generate_filelist_xml.php @@ -1,6 +1,6 @@ #!/usr/bin/env php +/* Copyright (C) 2015-2017 Laurent Destailleur * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -17,7 +17,7 @@ */ /** - * \file build/generate_filecheck_xml.php + * \file build/generate_filelist_xml.php * \ingroup dev * \brief This script create a xml checksum file */ @@ -40,55 +40,128 @@ require_once(DOL_DOCUMENT_ROOT."/core/lib/files.lib.php"); * Main */ +$includecustom=0; +$includeconstants=array(); + if (empty($argv[1])) { - print "Usage: ".$script_file." release=x.y.z\n"; + print "Usage: ".$script_file." release=auto|x.y.z[-mybuild] [includecustom=1] [includeconstant=CC:MY_CONF_NAME:value]\n"; + print "Example: ".$script_file." release=6.0.0 includecustom=1 includeconstant=FR:INVOICE_CAN_ALWAYS_BE_REMOVED:0 includeconstant=all:MAILING_NO_USING_PHPMAIL:1\n"; exit -1; } parse_str($argv[1]); -if ($release != DOL_VERSION) +$i=0; +while ($i < $argc) { - print 'Error: release is not version declared into filefunc.in.php.'."\n"; - exit -1; + if (! empty($argv[$i])) parse_str($argv[$i]); + if (preg_match('/includeconstant=/',$argv[$i])) + { + $tmp=explode(':', $includeconstant, 3); + if (count($tmp) != 3) + { + print "Error: Bad parameter includeconstant ".$includeconstant."\n"; + exit -1; + } + $includeconstants[$tmp[0]][$tmp[1]] = $tmp[2]; + } + $i++; } +// If release is auto, we take current version +$tmpver=explode('-', $release, 2); +if ($tmpver[0] == 'auto') +{ + $release=DOL_VERSION; + if ($tmpver[1]) $release.='-'.$tmpver[1]; +} + +if (empty($includecustom)) +{ + $tmpver=explode('-', $release, 2); + if (DOL_VERSION != $tmpver[0]) + { + print 'Error: When parameter "includecustom" is not set and there is no suffix in release parameter, version declared into filefunc.in.php ('.DOL_VERSION.') must be exact same value than "release" parameter ('.$tmpver[0].')'."\n"; + print "Usage: ".$script_file." release=x.y.z[-...] [includecustom=1]\n"; + exit -1; + } +} +else +{ + if (! preg_match('/'.preg_quote(DOL_VERSION,'/').'-/',$release)) + { + print 'Error: When parameter "includecustom" is set, version declared into filefunc.inc.php ('.DOL_VERSION.') must be used with a suffix into "release" parmater (ex: '.DOL_VERSION.'-mydistrib).'."\n"; + print "Usage: ".$script_file." release=x.y.z[-...] [includecustom=1]\n"; + exit -1; + } +} + +print "Release : ".$release."\n"; +print "Include custom in signature : ".$includecustom."\n"; +print "Include constants in signature : "; +foreach ($includeconstants as $countrycode => $tmp) +{ + foreach($tmp as $constname => $constvalue) + { + print $constname.'='.$constvalue." "; + } +} +print "\n"; + //$outputfile=dirname(__FILE__).'/../htdocs/install/filelist-'.$release.'.xml'; $outputdir=dirname(dirname(__FILE__)).'/htdocs/install'; print 'Delete current files '.$outputdir.'/filelist*.xml'."\n"; dol_delete_file($outputdir.'/filelist*.xml',0,1,1); +$checksumconcat=array(); + $outputfile=$outputdir.'/filelist-'.$release.'.xml'; $fp = fopen($outputfile,'w'); fputs($fp, ''."\n"); -fputs($fp, ''."\n"); +fputs($fp, ''."\n"); -fputs($fp, ''."\n"); +foreach ($includeconstants as $countrycode => $tmp) +{ + fputs($fp, ''."\n"); + foreach($tmp as $constname => $constvalue) + { + $valueforchecksum=(empty($constvalue)?'0':$constvalue); + $checksumconcat[]=$valueforchecksum; + fputs($fp, ' '.$valueforchecksum.''."\n"); + } + fputs($fp, ''."\n"); +} -$checksumconcat=array(); +fputs($fp, ''."\n"); -$dir_iterator1 = new RecursiveDirectoryIterator(dirname(__FILE__).'/../htdocs/'); +/*$dir_iterator1 = new RecursiveDirectoryIterator(dirname(__FILE__).'/../htdocs/'); $iterator1 = new RecursiveIteratorIterator($dir_iterator1); -// need to ignore document custom etc -$files = new RegexIterator($iterator1, '#^(?:[A-Z]:)?(?:/(?!(?:custom|documents|conf|install|nltechno))[^/]+)+/[^/]+\.(?:php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$#i'); +// Need to ignore document custom etc. Note: this also ignore natively symbolic links. +$files = new RegexIterator($iterator1, '#^(?:[A-Z]:)?(?:/(?!(?:'.($includecustom?'':'custom\/|').'documents\/|conf\/|install\/))[^/]+)+/[^/]+\.(?:php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$#i'); +*/ +$regextoinclude='\.(php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$'; +$regextoexclude='('.($includecustom?'':'custom|').'documents|conf|install|public\/test|Shared\/PCLZip|nusoap\/lib\/Mail|php\/example|php\/test|geoip\/sample.*\.php|ckeditor\/samples|ckeditor\/adapters)$'; // Exclude dirs +$files = dol_dir_list(DOL_DOCUMENT_ROOT, 'files', 1, $regextoinclude, $regextoexclude, 'fullname'); $dir=''; $needtoclose=0; -foreach ($files as $file) { - $newdir = str_replace(dirname(__FILE__).'/../htdocs', '', dirname($file)); - if ($newdir!=$dir) { +foreach ($files as $filetmp) { + $file = $filetmp['fullname']; + //$newdir = str_replace(dirname(__FILE__).'/../htdocs', '', dirname($file)); + $newdir = str_replace(DOL_DOCUMENT_ROOT, '', dirname($file)); + if ($newdir!=$dir) { if ($needtoclose) - fputs($fp, ''."\n"); - fputs($fp, ''."\n"); + fputs($fp, ' '."\n"); + fputs($fp, ' '."\n"); $dir = $newdir; $needtoclose=1; } if (filetype($file)=="file") { $md5=md5_file($file); $checksumconcat[]=$md5; - fputs($fp, ''.$md5.''."\n"); + fputs($fp, ' '.$md5.''."\n"); } } -fputs($fp, ''."\n"); +fputs($fp, ' '."\n"); fputs($fp, ''."\n"); asort($checksumconcat); // Sort list of checksum @@ -102,28 +175,36 @@ $checksumconcat=array(); fputs($fp, ''."\n"); -$dir_iterator2 = new RecursiveDirectoryIterator(dirname(__FILE__).'/../scripts/'); +// TODO Replace RecursiveDirectoryIterator with dol_dir_list +/*$dir_iterator2 = new RecursiveDirectoryIterator(dirname(__FILE__).'/../scripts/'); $iterator2 = new RecursiveIteratorIterator($dir_iterator2); -// need to ignore document custom etc -$files = new RegexIterator($iterator2, '#^(?:[A-Z]:)?(?:/(?!(?:custom|documents|conf|install|nltechno))[^/]+)+/[^/]+\.(?:php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$#i'); +// Need to ignore document custom etc. Note: this also ignore natively symbolic links. +$files = new RegexIterator($iterator2, '#^(?:[A-Z]:)?(?:/(?!(?:custom|documents|conf|install))[^/]+)+/[^/]+\.(?:php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$#i'); +*/ +$regextoinclude='\.(php|css|html|js|json|tpl|jpg|png|gif|sql|lang)$'; +$regextoexclude='(custom|documents|conf|install)$'; // Exclude dirs +$files = dol_dir_list(dirname(__FILE__).'/../scripts/', 'files', 1, $regextoinclude, $regextoexclude, 'fullname'); $dir=''; $needtoclose=0; -foreach ($files as $file) { - $newdir = str_replace(dirname(__FILE__).'/../scripts', '', dirname($file)); +foreach ($files as $filetmp) { + $file = $filetmp['fullname']; + //$newdir = str_replace(dirname(__FILE__).'/../scripts', '', dirname($file)); + $newdir = str_replace(DOL_DOCUMENT_ROOT, '', dirname($file)); + $newdir = str_replace(dirname(__FILE__).'/../scripts', '', dirname($file)); if ($newdir!=$dir) { if ($needtoclose) - fputs($fp, ''."\n"); - fputs($fp, ''."\n"); + fputs($fp, ' '."\n"); + fputs($fp, ' '."\n"); $dir = $newdir; $needtoclose=1; } if (filetype($file)=="file") { $md5=md5_file($file); $checksumconcat[]=$md5; - fputs($fp, ''.$md5.''."\n"); + fputs($fp, ' '.$md5.''."\n"); } } -fputs($fp, ''."\n"); +fputs($fp, ' '."\n"); fputs($fp, ''."\n"); asort($checksumconcat); // Sort list of checksum From 9c32e49bd3ffb334332d494f4b81618f49388b44 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 5 Jun 2018 18:05:33 +0200 Subject: [PATCH 2/4] FIX: dol_delete_file must work in a context without db handler loaded --- htdocs/core/lib/files.lib.php | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/htdocs/core/lib/files.lib.php b/htdocs/core/lib/files.lib.php index c93ed83fe11..4f47919e3c6 100644 --- a/htdocs/core/lib/files.lib.php +++ b/htdocs/core/lib/files.lib.php @@ -1136,7 +1136,7 @@ function dol_delete_file($file,$disableglob=0,$nophperrors=0,$nohook=0,$object=n if (preg_match('/\.\./',$file) || preg_match('/[<>|]/',$file)) { dol_syslog("Refused to delete file ".$file, LOG_WARNING); - return False; + return false; } if (empty($nohook)) @@ -1185,17 +1185,20 @@ function dol_delete_file($file,$disableglob=0,$nophperrors=0,$nohook=0,$object=n { $rel_filetodelete = preg_replace('/^[\\/]/', '', $rel_filetodelete); - dol_syslog("Try to remove also entries in database for full relative path = ".$rel_filetodelete, LOG_DEBUG); - include_once DOL_DOCUMENT_ROOT.'/ecm/class/ecmfiles.class.php'; - $ecmfile=new EcmFiles($db); - $result = $ecmfile->fetch(0, '', $rel_filetodelete); - if ($result >= 0 && $ecmfile->id > 0) + if (is_object($db)) // $db may not be defined when lib is in a context with define('NOREQUIREDB',1) { - $result = $ecmfile->delete($user); - } - if ($result < 0) - { - setEventMessages($ecmfile->error, $ecmfile->errors, 'warnings'); + dol_syslog("Try to remove also entries in database for full relative path = ".$rel_filetodelete, LOG_DEBUG); + include_once DOL_DOCUMENT_ROOT.'/ecm/class/ecmfiles.class.php'; + $ecmfile=new EcmFiles($db); + $result = $ecmfile->fetch(0, '', $rel_filetodelete); + if ($result >= 0 && $ecmfile->id > 0) + { + $result = $ecmfile->delete($user); + } + if ($result < 0) + { + setEventMessages($ecmfile->error, $ecmfile->errors, 'warnings'); + } } } } From 4c21ad4e4e331c64bebcce76da018f5cd49dd883 Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 5 Jun 2018 18:05:33 +0200 Subject: [PATCH 3/4] FIX: dol_delete_file must work in a context without db handler loaded Conflicts: htdocs/core/lib/files.lib.php --- htdocs/core/lib/files.lib.php | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/htdocs/core/lib/files.lib.php b/htdocs/core/lib/files.lib.php index 5bf386e1fac..b7e69b243cf 100644 --- a/htdocs/core/lib/files.lib.php +++ b/htdocs/core/lib/files.lib.php @@ -1031,7 +1031,7 @@ function dol_delete_file($file,$disableglob=0,$nophperrors=0,$nohook=0,$object=n if (preg_match('/\.\./',$file) || preg_match('/[<>|]/',$file)) { dol_syslog("Refused to delete file ".$file, LOG_WARNING); - return False; + return false; } if (empty($nohook)) @@ -1080,18 +1080,21 @@ function dol_delete_file($file,$disableglob=0,$nophperrors=0,$nohook=0,$object=n { $rel_filetodelete = preg_replace('/^[\\/]/', '', $rel_filetodelete); - dol_syslog("Try to remove also entries in database for full relative path = ".$rel_filetodelete, LOG_DEBUG); - include_once DOL_DOCUMENT_ROOT.'/ecm/class/ecmfiles.class.php'; - $ecmfile=new EcmFiles($db); - $result = $ecmfile->fetch(0, '', $rel_filetodelete); - if ($result >= 0 && $ecmfile->id > 0) - { - $result = $ecmfile->delete($user); - } - if ($result < 0) - { - setEventMessages($ecmfile->error, $ecmfile->errors, 'warnings'); - } + if (is_object($db)) // $db may not be defined when lib is in a context with define('NOREQUIREDB',1) + { + dol_syslog("Try to remove also entries in database for full relative path = ".$rel_filetodelete, LOG_DEBUG); + include_once DOL_DOCUMENT_ROOT.'/ecm/class/ecmfiles.class.php'; + $ecmfile=new EcmFiles($db); + $result = $ecmfile->fetch(0, '', $rel_filetodelete); + if ($result >= 0 && $ecmfile->id > 0) + { + $result = $ecmfile->delete($user); + } + if ($result < 0) + { + setEventMessages($ecmfile->error, $ecmfile->errors, 'warnings'); + } + } } } else dol_syslog("Failed to remove file ".$filename, LOG_WARNING); @@ -1130,7 +1133,7 @@ function dol_delete_dir($dir,$nophperrors=0) if (preg_match('/\.\./',$dir) || preg_match('/[<>|]/',$dir)) { dol_syslog("Refused to delete dir ".$dir, LOG_WARNING); - return False; + return false; } $dir_osencoded=dol_osencode($dir); @@ -1711,7 +1714,7 @@ function dol_uncompress($inputfile,$outputdir) dol_syslog("Class ZipArchive is set so we unzip using ZipArchive to unzip into ".$outputdir); $zip = new ZipArchive; $res = $zip->open($inputfile); - if ($res === TRUE) + if ($res === true) { $zip->extractTo($outputdir.'/'); $zip->close(); From 8785d6954858a4679c30e100586bc18581a703de Mon Sep 17 00:00:00 2001 From: Laurent Destailleur Date: Tue, 5 Jun 2018 18:21:56 +0200 Subject: [PATCH 4/4] Do not load database handler --- build/generate_filelist_xml.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/generate_filelist_xml.php b/build/generate_filelist_xml.php index 5f54c95d7e0..5f71ff54072 100755 --- a/build/generate_filelist_xml.php +++ b/build/generate_filelist_xml.php @@ -22,6 +22,8 @@ * \brief This script create a xml checksum file */ +if (! defined('NOREQUIREDB')) define('NOREQUIREDB','1'); // Do not create database handler $db + $sapi_type = php_sapi_name(); $script_file = basename(__FILE__); $path=dirname(__FILE__).'/';