Merge pull request #5826 from atm-maxime/new_modimport_update_option

NEW modimport update option
This commit is contained in:
Laurent Destailleur 2016-12-12 16:30:51 +01:00 committed by GitHub
commit 4ac7460b26
6 changed files with 172 additions and 60 deletions

View File

@ -54,6 +54,9 @@ class ImportCsv extends ModeleImports
var $cacheconvert=array(); // Array to cache list of value found after a convertion
var $cachefieldtable=array(); // Array to cache list of value found into fields@tables
var $nbinsert = 0; // # of insert done during the import
var $nbupdate = 0; // # of update done during the import
/**
@ -266,9 +269,10 @@ class ImportCsv extends ModeleImports
* @param Object $objimport Object import (contains objimport->array_import_tables, objimport->array_import_fields, objimport->array_import_convertvalue, ...)
* @param int $maxfields Max number of fields to use
* @param string $importid Import key
* @param array $updatekeys Array of keys to use to try to do update
* @return int <0 if KO, >0 if OK
*/
function import_insert($arrayrecord,$array_match_file_to_database,$objimport,$maxfields,$importid)
function import_insert($arrayrecord,$array_match_file_to_database,$objimport,$maxfields,$importid,$updatekeys)
{
global $langs,$conf,$user;
global $thirdparty_static; // Specific to thirdparty import
@ -299,13 +303,15 @@ class ImportCsv extends ModeleImports
else
{
$last_insert_id_array = array(); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id (eg: extrafields fk_object will be set with the last inserted object's id)
$updatedone = false;
$insertdone = false;
// For each table to insert, me make a separate insert
foreach($objimport->array_import_tables[0] as $alias => $tablename)
{
// Build sql request
$sql='';
$listfields='';
$listvalues='';
$listfields=array();
$listvalues=array();
$i=0;
$errorforthistable=0;
@ -539,36 +545,35 @@ class ImportCsv extends ModeleImports
}
// Define $listfields and $listvalues to build SQL request
if ($listfields) { $listfields.=', '; $listvalues.=', '; }
$listfields.=$fieldname;
$listfields[] = $fieldname;
// Note: arrayrecord (and 'type') is filled with ->import_read_record called by import.php page before calling import_insert
if (empty($newval) && $arrayrecord[($key-1)]['type'] < 0) $listvalues.=($newval=='0'?$newval:"null");
elseif (empty($newval) && $arrayrecord[($key-1)]['type'] == 0) $listvalues.="''";
else $listvalues.="'".$this->db->escape($newval)."'";
if (empty($newval) && $arrayrecord[($key-1)]['type'] < 0) $listvalues[] = ($newval=='0'?$newval:"null");
elseif (empty($newval) && $arrayrecord[($key-1)]['type'] == 0) $listvalues[] = "''";
else $listvalues[] = "'".$this->db->escape($newval)."'";
}
$i++;
}
// We add hidden fields (but only if there is at least one field to add into table)
if ($listfields && is_array($objimport->array_import_fieldshidden[0]))
if (!empty($listfields) && is_array($objimport->array_import_fieldshidden[0]))
{
// Loop on each hidden fields to add them into listfields/listvalues
foreach($objimport->array_import_fieldshidden[0] as $key => $val)
{
if (! preg_match('/^'.preg_quote($alias).'\./', $key)) continue; // Not a field of current table
if ($listfields) { $listfields.=', '; $listvalues.=', '; }
if ($val == 'user->id')
{
$listfields.=preg_replace('/^'.preg_quote($alias).'\./','',$key);
$listvalues.=$user->id;
$listfields[] = preg_replace('/^'.preg_quote($alias).'\./','',$key);
$listvalues[] = $user->id;
}
elseif (preg_match('/^lastrowid-/',$val))
{
$tmp=explode('-',$val);
$lastinsertid=(isset($last_insert_id_array[$tmp[1]]))?$last_insert_id_array[$tmp[1]]:0;
$listfields.=preg_replace('/^'.preg_quote($alias).'\./','',$key);
$listvalues.=$lastinsertid;
$keyfield = preg_replace('/^'.preg_quote($alias).'\./','',$key);
$listfields[] = $keyfield;
$listvalues[] = $lastinsertid;
//print $key."-".$val."-".$listfields."-".$listvalues."<br>";exit;
}
}
@ -578,47 +583,114 @@ class ImportCsv extends ModeleImports
// If no error for this $alias/$tablename, we have a complete $listfields and $listvalues that are defined
if (! $errorforthistable)
{
//print "$alias/$tablename/$listfields/$listvalues<br>";
if ($listfields)
//print "$alias/$tablename/$listfields/$listvalues<br>";
if (!empty($listfields))
{
//var_dump($objimport->array_import_convertvalue); exit;
// Build SQL request
if (empty($tablewithentity_cache[$tablename]))
{
$sql ='INSERT INTO '.$tablename.'('.$listfields.', import_key';
if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$objimport->array_import_tables_creator[0][$alias];
$sql.=') VALUES('.$listvalues.", '".$importid."'";
}
else
{
$sql ='INSERT INTO '.$tablename.'('.$listfields.', import_key, entity';
if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$objimport->array_import_tables_creator[0][$alias];
$sql.=') VALUES('.$listvalues.", '".$importid."', ".$conf->entity ;
}
if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$user->id;
$sql.=')';
dol_syslog("import_csv.modules", LOG_DEBUG);
//print '> '.join(',',$arrayrecord);
//print 'sql='.$sql;
//print '<br>'."\n";
// Run insert request
if ($sql)
{
$resql=$this->db->query($sql);
$last_insert_id_array[$tablename] = $this->db->last_insert_id($tablename); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id. This must be done just after the INSERT request, else we risk losing the id (because another sql query will be issued somewhere in Dolibarr).
if ($resql)
{
//print '.';
$updatedone = false;
$insertdone = false;
if(!empty($updatekeys)) {
// We do SELECT to get the rowid, if we already have the rowid, it's to be used below for related tables (extrafields)
if(empty($lastinsertid)) {
$sqlSelect = 'SELECT rowid FROM '.$tablename;
$data = array_combine($listfields, $listvalues);
$where = array();
$filters = array();
foreach ($updatekeys as $key) {
$col = $objimport->array_import_updatekeys[0][$key];
$key=preg_replace('/^.*\./i','',$key);
$where[] = $key.' = '.$data[$key];
$filters[] = $col.' = '.$data[$key];
}
$sqlSelect.= ' WHERE '.implode(' AND ', $where);
$resql=$this->db->query($sqlSelect);
if($resql) {
$res = $this->db->fetch_object($resql);
if($resql->num_rows == 1) {
$lastinsertid = $res->rowid;
$last_insert_id_array[$tablename] = $lastinsertid;
} else if($resql->num_rows > 1) {
$this->errors[$error]['lib']=$langs->trans('MultipleRecordFoundWithTheseFilters', implode($filters, ', '));
$this->errors[$error]['type']='SQL';
$error++;
} else {
// No record found with filters, insert will be tried below
}
}
else
{
//print 'E';
$this->errors[$error]['lib']=$this->db->lasterror();
$this->errors[$error]['type']='SQL';
$error++;
}
}
else
if(!empty($lastinsertid)) {
// Build SQL UPDATE request
$sqlstart = 'UPDATE '.$tablename;
$data = array_combine($listfields, $listvalues);
$set = array();
foreach ($data as $key => $val) {
$set[] = $key.' = '.$val;
}
$sqlstart.= ' SET '.implode(', ', $set);
if(empty($keyfield)) $keyfield = 'rowid';
$sqlend = ' WHERE '.$keyfield.' = '.$lastinsertid;
$sql = $sqlstart.$sqlend;
// Run update request
$resql=$this->db->query($sql);
if($resql) {
// No error, update has been done. $this->db->db->affected_rows can be 0 if data hasn't changed
$updatedone = true;
}
else
{
//print 'E';
$this->errors[$error]['lib']=$this->db->lasterror();
$this->errors[$error]['type']='SQL';
$error++;
}
}
}
// Update not done, we do insert
if(!$error && !$updatedone) {
// Build SQL INSERT request
$sqlstart = 'INSERT INTO '.$tablename.'('.implode(', ', $listfields).', import_key';
$sqlend = ') VALUES('.implode(', ', $listvalues).", '".$importid."'";
if (! empty($tablewithentity_cache[$tablename])) {
$sqlstart.= ', entity';
$sqlend.= ', '.$conf->entity;
}
if (! empty($objimport->array_import_tables_creator[0][$alias])) {
$sqlstart.= ', '.$objimport->array_import_tables_creator[0][$alias];
$sqlend.=', '.$user->id;
}
$sql = $sqlstart.$sqlend.')';
dol_syslog("import_csv.modules", LOG_DEBUG);
// Run insert request
if ($sql)
{
//print 'E';
$this->errors[$error]['lib']=$this->db->lasterror();
$this->errors[$error]['type']='SQL';
$error++;
$resql=$this->db->query($sql);
$last_insert_id_array[$tablename] = $this->db->last_insert_id($tablename); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id. This must be done just after the INSERT request, else we risk losing the id (because another sql query will be issued somewhere in Dolibarr).
if ($resql)
{
$insertdone = true;
}
else
{
//print 'E';
$this->errors[$error]['lib']=$this->db->lasterror();
$this->errors[$error]['type']='SQL';
$error++;
}
}
}
}
@ -630,6 +702,9 @@ class ImportCsv extends ModeleImports
if ($error) break;
}
if($updatedone) $this->nbupdate++;
if($insertdone) $this->nbinsert++;
}
return 1;

View File

@ -248,6 +248,7 @@ class modProduct extends DolibarrModules
$this->import_regex_array[$r]=array('p.ref'=>'[^ ]','p.tosell'=>'^[0|1]$','p.tobuy'=>'^[0|1]$','p.fk_product_type'=>'^[0|1]$','p.datec'=>'^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]$','p.recuperableonly'=>'^[0|1]$');
$import_sample=array('p.ref'=>"PREF123456",'p.label'=>"My product",'p.description'=>"This is a description example for record",'p.note'=>"Some note",'p.price'=>"100",'p.price_ttc'=>"110",'p.tva_tx'=>'10','p.tosell'=>"0 or 1",'p.tobuy'=>"0 or 1",'p.fk_product_type'=>"0 for product/1 for service",'p.finished'=>'','p.duration'=>"1y",'p.datec'=>'2008-12-31','p.recuperableonly'=>'0 or 1');
$this->import_examplevalues_array[$r]=array_merge($import_sample,$import_extrafield_sample);
$this->import_updatekeys_array[$r]=array('p.ref'=>'Ref','p.barcode'=>'BarCode');
if (! empty($conf->fournisseur->enabled))
{
@ -276,6 +277,7 @@ class modProduct extends DolibarrModules
'sp.unitprice'=>'50',
'sp.remise_percent'=>'0'
);
$this->import_updatekeys_array[$r]=array('sp.fk_product'=>'ProductOrService','sp.ref_fourn'=>'SupplierRef');
}
if (! empty($conf->global->PRODUIT_MULTIPRICES))

View File

@ -354,6 +354,7 @@ class modSociete extends DolibarrModules
//$this->import_convertvalue_array[$r]=array('s.fk_soc'=>array('rule'=>'lastrowid',table='t');
$this->import_regex_array[$r]=array('s.status'=>'^[0|1]','s.client'=>'^[0|1|2|3]','s.fournisseur'=>'^[0|1]','s.fk_typent'=>'id@'.MAIN_DB_PREFIX.'c_typent','s.datec'=>'^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]( [0-9][0-9]:[0-9][0-9]:[0-9][0-9])?$');
$this->import_examplevalues_array[$r]=array('s.nom'=>"MyBigCompany",'s.status'=>"0 (closed) or 1 (active)",'s.client'=>'0 (no customer no prospect)/1 (customer)/2 (prospect)/3 (customer and prospect)','s.fournisseur'=>'0 or 1','s.datec'=>dol_print_date(dol_now(),'%Y-%m-%d'),'s.code_client'=>'CU01-0001 or empty or "auto"','s.code_fournisseur'=>'SU01-0001 or empty or "auto"','s.address'=>"61 jump street",'s.zip'=>"123456",'s.town'=>"Big town",'s.fk_pays'=>'US, FR, DE...','s.phone'=>"0101010101",'s.fax'=>"0101010102",'s.url'=>"http://mycompany.com",'s.email'=>"test@mycompany.com",'s.siret'=>"",'s.siren'=>"",'s.ape'=>"",'s.idprof4'=>"",'s.idprof5'=>"",'s.idprof6'=>"",'s.tva_intra'=>"FR0123456789",'s.capital'=>"10000",'s.note_private'=>"This is an example of private note for record",'s.note_public'=>"This is an example of public note for record",'s.fk_typent'=>"2",'s.fk_effectif'=>"3","s.fk_forme_juridique"=>"1",'s.fk_prospectlevel'=>'PL_MEDIUM','s.fk_stcomm'=>'0','s.default_lang'=>'en_US','s.barcode'=>'123456789','s.datec'=>"2015-01-01 or 2015-01-01 12:30:00");
$this->import_updatekeys_array[$r]=array('s.nom'=>'Name','s.code_client'=>'CustomerCode','s.code_fournisseur'=>'SupplierCode','s.code_compta'=>'CustomerAccountancyCode','s.code_compta_fournisseur'=>'SupplierAccountancyCode');
// Import list of contact and attributes
$r++;

View File

@ -38,6 +38,7 @@ class Import
var $array_import_fieldshidden;
var $array_import_entities;
var $array_import_regex;
var $array_import_updatekeys;
var $array_import_examplevalues;
var $array_import_convertvalue;
var $array_import_run_sql_after;
@ -153,6 +154,8 @@ class Import
$this->array_import_entities[$i]=$module->import_entities_array[$r];
// Tableau des alias a exporter (cle=champ, valeur=alias)
$this->array_import_regex[$i]=$module->import_regex_array[$r];
// Array of columns allowed as UPDATE options
$this->array_import_updatekeys[$i]=$module->import_updatekeys_array[$r];
// Array of examples
$this->array_import_examplevalues[$i]=$module->import_examplevalues_array[$r];
// Tableau des regles de conversion d'une valeur depuis une autre source (cle=champ, valeur=tableau des regles)

View File

@ -76,7 +76,8 @@ $import_name = GETPOST('import_name');
$hexa = GETPOST('hexa');
$importmodelid = GETPOST('importmodelid');
$excludefirstline = (GETPOST('excludefirstline') ? GETPOST('excludefirstline') : 1);
$endatlinenb = (GETPOST('endatlinenb') ? GETPOST('endatlinenb') : '');
$endatlinenb = (GETPOST('endatlinenb') ? GETPOST('endatlinenb') : '');
$updatekeys = (GETPOST('updatekeys') ? GETPOST('updatekeys') : array());
$separator = (GETPOST('separator') ? GETPOST('separator') : (! empty($conf->global->IMPORT_CSV_SEPARATOR_TO_USE)?$conf->global->IMPORT_CSV_SEPARATOR_TO_USE:','));
$enclosure = (GETPOST('enclosure') ? GETPOST('enclosure') : '"');
@ -1184,8 +1185,9 @@ if ($step == 5 && $datatoimport)
$param='&leftmenu=import&format='.$format.'&datatoimport='.$datatoimport.'&filetoimport='.urlencode($filetoimport).'&nboflines='.$nboflines.'&separator='.urlencode($separator).'&enclosure='.urlencode($enclosure);
$param2 = $param; // $param2 = $param without excludefirstline and endatlinenb
if ($excludefirstline) $param.='&excludefirstline='.$excludefirstline;
if ($endatlinenb) $param.='&endatlinenb='.$endatlinenb;
if ($excludefirstline) $param.='&excludefirstline='.$excludefirstline;
if ($endatlinenb) $param.='&endatlinenb='.$endatlinenb;
if (!empty($updatekeys)) $param.='&updatekeys[]='.implode('&updatekeys[]=', $updatekeys);
llxHeader('',$langs->trans("NewImport"),'EN:Module_Imports_En|FR:Module_Imports|ES:M&oacute;dulo_Importaciones');
@ -1291,6 +1293,24 @@ if ($step == 5 && $datatoimport)
}
print '</td></tr>';
print '<tr><td>';
print $langs->trans("KeysToUseForUpdates");
print '</td><td>';
if($action=='launchsimu') {
print $form->multiselectarray('updatekeysbis', $objimport->array_import_updatekeys[0], $updatekeys, 0, 0, '', 1, '', 'disabled');
foreach($updatekeys as $val) {
print '<input type="hidden" name="updatekeys[]" value="'.$val.'">';
}
print ' &nbsp; <a href="'.$_SERVER["PHP_SELF"].'?step=5'.$param.'">'.$langs->trans("Modify").'</a>';
} else {
print $form->multiselectarray('updatekeys', $objimport->array_import_updatekeys[0], $updatekeys, 0, 0, '', 1, '80%');
print $form->textwithpicto("", $langs->trans("SelectColumnsOfYourFileForUpdateAttempt"));
}
/*echo '<pre>';
print_r($objimport->array_import_updatekeys);
echo '</pre>';*/
print '</td></tr>';
print '</table>';
print '<br>';
@ -1424,7 +1444,7 @@ if ($step == 5 && $datatoimport)
if ($endatlinenb && ($sourcelinenb > $endatlinenb)) continue;
// Run import
$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid);
$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid,$updatekeys);
if (count($obj->errors)) $arrayoferrors[$sourcelinenb]=$obj->errors;
if (count($obj->warnings)) $arrayofwarnings[$sourcelinenb]=$obj->warnings;
@ -1460,9 +1480,13 @@ if ($step == 5 && $datatoimport)
$db->rollback(); // We force rollback because this was just a simulation.
// Show OK
if (! count($arrayoferrors) && ! count($arrayofwarnings)) print '<center>'.img_picto($langs->trans("OK"),'tick').' <b>'.$langs->trans("NoError").'</b></center><br><br>';
if (! count($arrayoferrors) && ! count($arrayofwarnings)) {
print '<center>'.img_picto($langs->trans("OK"),'tick').' <b>'.$langs->trans("NoError").'</b></center><br><br>';
print $langs->trans("NbInsert", $obj->nbinsert).'<br>';
print $langs->trans("NbUpdate", $obj->nbupdate).'<br><br>';
}
else print '<b>'.$langs->trans("NbOfLinesOK",$nbok).'</b><br><br>';
// Show Errors
//var_dump($arrayoferrors);
if (count($arrayoferrors))
@ -1778,7 +1802,7 @@ if ($step == 6 && $datatoimport)
if ($endatlinenb && ($sourcelinenb > $endatlinenb)) continue;
// Run import
$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid);
$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid,$updatekeys);
if (count($obj->errors)) $arrayoferrors[$sourcelinenb]=$obj->errors;
if (count($obj->warnings)) $arrayofwarnings[$sourcelinenb]=$obj->warnings;
@ -1824,7 +1848,9 @@ if ($step == 6 && $datatoimport)
// Show result
print '<br>';
print '<div class="center">';
print $langs->trans("NbOfLinesImported",$nbok).'</b><br><br>';
print $langs->trans("NbOfLinesImported",$nbok).'</b><br>';
print $langs->trans("NbInsert", $obj->nbinsert).'<br>';
print $langs->trans("NbUpdate", $obj->nbupdate).'<br><br>';
print $langs->trans("FileWasImported",$importid).'<br>';
print $langs->trans("YouCanUseImportIdToFindRecord",$importid).'<br>';
print '</div>';

View File

@ -120,3 +120,8 @@ SelectFilterFields=If you want to filter on some values, just input values here.
FilteredFields=Filtered fields
FilteredFieldsValues=Value for filter
FormatControlRule=Format control rule
## imports updates
KeysToUseForUpdates=Key to use for updating data
NbInsert=Number of inserted lines: %s
NbUpdate=Number of updated lines: %s
MultipleRecordFoundWithTheseFilters=Multiple records have been found with these filters: %s