Merge pull request #5826 from atm-maxime/new_modimport_update_option

NEW modimport update option

htdocs/core/modules/import/import_csv.modules.php

@@ -54,6 +54,9 @@ class ImportCsv extends ModeleImports
 
 	var $cacheconvert=array();      // Array to cache list of value found after a convertion
 	var $cachefieldtable=array();   // Array to cache list of value found into fields@tables
+	
+	var $nbinsert = 0; // # of insert done during the import
+	var $nbupdate = 0; // # of update done during the import
 
 
 	/**
@@ -266,9 +269,10 @@ class ImportCsv extends ModeleImports
 	 * @param 	Object	$objimport						Object import (contains objimport->array_import_tables, objimport->array_import_fields, objimport->array_import_convertvalue, ...)
 	 * @param	int		$maxfields						Max number of fields to use
 	 * @param	string	$importid						Import key
+	 * @param	array	$updatekeys						Array of keys to use to try to do update
 	 * @return	int										<0 if KO, >0 if OK
 	 */
-	function import_insert($arrayrecord,$array_match_file_to_database,$objimport,$maxfields,$importid)
+	function import_insert($arrayrecord,$array_match_file_to_database,$objimport,$maxfields,$importid,$updatekeys)
 	{
 		global $langs,$conf,$user;
         global $thirdparty_static;    	// Specific to thirdparty import
@@ -299,13 +303,15 @@ class ImportCsv extends ModeleImports
 		else
 		{
 			$last_insert_id_array = array(); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id (eg: extrafields fk_object will be set with the last inserted object's id)
+			$updatedone = false;
+			$insertdone = false;
 			// For each table to insert, me make a separate insert
 			foreach($objimport->array_import_tables[0] as $alias => $tablename)
 			{
 				// Build sql request
 				$sql='';
-				$listfields='';
+				$listfields=array();
-				$listvalues='';
+				$listvalues=array();
 				$i=0;
 				$errorforthistable=0;
 
@@ -539,36 +545,35 @@ class ImportCsv extends ModeleImports
 						}
 
 						// Define $listfields and $listvalues to build SQL request
-						if ($listfields) { $listfields.=', '; $listvalues.=', '; }
+						$listfields[] = $fieldname;
-						$listfields.=$fieldname;
 
 						// Note: arrayrecord (and 'type') is filled with ->import_read_record called by import.php page before calling import_insert
-						if (empty($newval) && $arrayrecord[($key-1)]['type'] < 0)       $listvalues.=($newval=='0'?$newval:"null");
+						if (empty($newval) && $arrayrecord[($key-1)]['type'] < 0)		 $listvalues[] = ($newval=='0'?$newval:"null");
-						elseif (empty($newval) && $arrayrecord[($key-1)]['type'] == 0) $listvalues.="''";
+						elseif (empty($newval) && $arrayrecord[($key-1)]['type'] == 0)	 $listvalues[] = "''";
-						else															 $listvalues.="'".$this->db->escape($newval)."'";
+						else															 $listvalues[] = "'".$this->db->escape($newval)."'";
 					}
 					$i++;
 				}
 
 				// We add hidden fields (but only if there is at least one field to add into table)
-				if ($listfields && is_array($objimport->array_import_fieldshidden[0]))
+				if (!empty($listfields) && is_array($objimport->array_import_fieldshidden[0]))
 				{
     				// Loop on each hidden fields to add them into listfields/listvalues
 				    foreach($objimport->array_import_fieldshidden[0] as $key => $val)
     				{
     				    if (! preg_match('/^'.preg_quote($alias).'\./', $key)) continue;    // Not a field of current table
-    				    if ($listfields) { $listfields.=', '; $listvalues.=', '; }
     				    if ($val == 'user->id')
     				    {
-    				        $listfields.=preg_replace('/^'.preg_quote($alias).'\./','',$key);
+    				        $listfields[] = preg_replace('/^'.preg_quote($alias).'\./','',$key);
-    				        $listvalues.=$user->id;
+    				        $listvalues[] = $user->id;
     				    }
     				    elseif (preg_match('/^lastrowid-/',$val))
     				    {
     				        $tmp=explode('-',$val);
     				        $lastinsertid=(isset($last_insert_id_array[$tmp[1]]))?$last_insert_id_array[$tmp[1]]:0;
-    				        $listfields.=preg_replace('/^'.preg_quote($alias).'\./','',$key);
+							$keyfield = preg_replace('/^'.preg_quote($alias).'\./','',$key);
-                            $listvalues.=$lastinsertid;
+    				        $listfields[] = $keyfield;
+                            $listvalues[] = $lastinsertid;
     				        //print $key."-".$val."-".$listfields."-".$listvalues."<br>";exit;
     				    }
     				}
@@ -578,47 +583,114 @@ class ImportCsv extends ModeleImports
 				// If no error for this $alias/$tablename, we have a complete $listfields and $listvalues that are defined
 				if (! $errorforthistable)
 				{
-				    //print "$alias/$tablename/$listfields/$listvalues<br>";
+					//print "$alias/$tablename/$listfields/$listvalues<br>";
-					if ($listfields)
+					if (!empty($listfields))
 					{
-					    //var_dump($objimport->array_import_convertvalue); exit;
+						$updatedone = false;
-
+						$insertdone = false;
-						// Build SQL request
+						if(!empty($updatekeys)) {
-						if (empty($tablewithentity_cache[$tablename]))
+							// We do SELECT to get the rowid, if we already have the rowid, it's to be used below for related tables (extrafields)
-						{
+							if(empty($lastinsertid)) {
-							$sql ='INSERT INTO '.$tablename.'('.$listfields.', import_key';
+								$sqlSelect = 'SELECT rowid FROM '.$tablename;
-							if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$objimport->array_import_tables_creator[0][$alias];
+								
-							$sql.=') VALUES('.$listvalues.", '".$importid."'";
+								$data = array_combine($listfields, $listvalues);
-						}
+								$where = array();
-						else
+								$filters = array();
-						{
+								foreach ($updatekeys as $key) {
-							$sql ='INSERT INTO '.$tablename.'('.$listfields.', import_key, entity';
+									$col = $objimport->array_import_updatekeys[0][$key];
-							if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$objimport->array_import_tables_creator[0][$alias];
+									$key=preg_replace('/^.*\./i','',$key);
-							$sql.=') VALUES('.$listvalues.", '".$importid."', ".$conf->entity ;
+									$where[] = $key.' = '.$data[$key];
-						}
+									$filters[] = $col.' = '.$data[$key];
-						if (! empty($objimport->array_import_tables_creator[0][$alias])) $sql.=', '.$user->id;
+								}
-						$sql.=')';
+								$sqlSelect.= ' WHERE '.implode(' AND ', $where);
-						dol_syslog("import_csv.modules", LOG_DEBUG);
+								
-
+								$resql=$this->db->query($sqlSelect);
-						//print '> '.join(',',$arrayrecord);
+								if($resql) {
-						//print 'sql='.$sql;
+									$res = $this->db->fetch_object($resql);
-						//print '<br>'."\n";
+									if($resql->num_rows == 1) {
-
+										$lastinsertid = $res->rowid;
-						// Run insert request
+										$last_insert_id_array[$tablename] = $lastinsertid;
-						if ($sql)
+									} else if($resql->num_rows > 1) {
-						{
+										$this->errors[$error]['lib']=$langs->trans('MultipleRecordFoundWithTheseFilters', implode($filters, ', '));
-							$resql=$this->db->query($sql);
+										$this->errors[$error]['type']='SQL';
-							$last_insert_id_array[$tablename] = $this->db->last_insert_id($tablename); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id. This must be done just after the INSERT request, else we risk losing the id (because another sql query will be issued somewhere in Dolibarr).
+										$error++;
-							if ($resql)
+									} else {
-							{
+										// No record found with filters, insert will be tried below
-								//print '.';
+									}
+								}
+								else
+								{
+									//print 'E';
+									$this->errors[$error]['lib']=$this->db->lasterror();
+									$this->errors[$error]['type']='SQL';
+									$error++;
+								}
 							}
-							else
+							
+							if(!empty($lastinsertid)) {
+								// Build SQL UPDATE request
+								$sqlstart = 'UPDATE '.$tablename;
+								
+								$data = array_combine($listfields, $listvalues);
+								$set = array();
+								foreach ($data as $key => $val) {
+									$set[] = $key.' = '.$val;
+								}
+								$sqlstart.= ' SET '.implode(', ', $set);
+								
+								if(empty($keyfield)) $keyfield = 'rowid';
+								$sqlend = ' WHERE '.$keyfield.' = '.$lastinsertid;
+								
+								$sql = $sqlstart.$sqlend;
+								
+								// Run update request
+								$resql=$this->db->query($sql);
+								if($resql) {
+									// No error, update has been done. $this->db->db->affected_rows can be 0 if data hasn't changed
+									$updatedone = true;
+								}
+								else
+								{
+									//print 'E';
+									$this->errors[$error]['lib']=$this->db->lasterror();
+									$this->errors[$error]['type']='SQL';
+									$error++;
+								}
+							}
+						}
+
+						// Update not done, we do insert
+						if(!$error && !$updatedone) {
+							// Build SQL INSERT request
+							$sqlstart = 'INSERT INTO '.$tablename.'('.implode(', ', $listfields).', import_key';
+							$sqlend = ') VALUES('.implode(', ', $listvalues).", '".$importid."'";
+							if (! empty($tablewithentity_cache[$tablename])) {
+								$sqlstart.= ', entity';
+								$sqlend.= ', '.$conf->entity;
+							} 
+							if (! empty($objimport->array_import_tables_creator[0][$alias])) {
+								$sqlstart.= ', '.$objimport->array_import_tables_creator[0][$alias];
+								$sqlend.=', '.$user->id;
+							}
+							$sql = $sqlstart.$sqlend.')';
+							dol_syslog("import_csv.modules", LOG_DEBUG);
+							
+							// Run insert request
+							if ($sql)
 							{
-								//print 'E';
+								$resql=$this->db->query($sql);
-								$this->errors[$error]['lib']=$this->db->lasterror();
+								$last_insert_id_array[$tablename] = $this->db->last_insert_id($tablename); // store the last inserted auto_increment id for each table, so that dependent tables can be inserted with the appropriate id. This must be done just after the INSERT request, else we risk losing the id (because another sql query will be issued somewhere in Dolibarr).
-								$this->errors[$error]['type']='SQL';
+								if ($resql)
-								$error++;
+								{
+									$insertdone = true;
+								}
+								else
+								{
+									//print 'E';
+									$this->errors[$error]['lib']=$this->db->lasterror();
+									$this->errors[$error]['type']='SQL';
+									$error++;
+								}
 							}
 						}
 					}
@@ -630,6 +702,9 @@ class ImportCsv extends ModeleImports
 
 			    if ($error) break;
 			}
+
+			if($updatedone) $this->nbupdate++;
+			if($insertdone) $this->nbinsert++;
 		}
 
 		return 1;

htdocs/core/modules/modProduct.class.php

@@ -248,6 +248,7 @@ class modProduct extends DolibarrModules
 		$this->import_regex_array[$r]=array('p.ref'=>'[^ ]','p.tosell'=>'^[0|1]$','p.tobuy'=>'^[0|1]$','p.fk_product_type'=>'^[0|1]$','p.datec'=>'^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]$','p.recuperableonly'=>'^[0|1]$');
 		$import_sample=array('p.ref'=>"PREF123456",'p.label'=>"My product",'p.description'=>"This is a description example for record",'p.note'=>"Some note",'p.price'=>"100",'p.price_ttc'=>"110",'p.tva_tx'=>'10','p.tosell'=>"0 or 1",'p.tobuy'=>"0 or 1",'p.fk_product_type'=>"0 for product/1 for service",'p.finished'=>'','p.duration'=>"1y",'p.datec'=>'2008-12-31','p.recuperableonly'=>'0 or 1');
 		$this->import_examplevalues_array[$r]=array_merge($import_sample,$import_extrafield_sample);
+		$this->import_updatekeys_array[$r]=array('p.ref'=>'Ref','p.barcode'=>'BarCode');
 
 		if (! empty($conf->fournisseur->enabled))
 		{
@@ -276,6 +277,7 @@ class modProduct extends DolibarrModules
 					'sp.unitprice'=>'50',
 					'sp.remise_percent'=>'0'
 			);
+			$this->import_updatekeys_array[$r]=array('sp.fk_product'=>'ProductOrService','sp.ref_fourn'=>'SupplierRef');
 		}
 
 		if (! empty($conf->global->PRODUIT_MULTIPRICES))

htdocs/core/modules/modSociete.class.php

@@ -354,6 +354,7 @@ class modSociete extends DolibarrModules
 		//$this->import_convertvalue_array[$r]=array('s.fk_soc'=>array('rule'=>'lastrowid',table='t');
 		$this->import_regex_array[$r]=array('s.status'=>'^[0|1]','s.client'=>'^[0|1|2|3]','s.fournisseur'=>'^[0|1]','s.fk_typent'=>'id@'.MAIN_DB_PREFIX.'c_typent','s.datec'=>'^[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]( [0-9][0-9]:[0-9][0-9]:[0-9][0-9])?$');
 		$this->import_examplevalues_array[$r]=array('s.nom'=>"MyBigCompany",'s.status'=>"0 (closed) or 1 (active)",'s.client'=>'0 (no customer no prospect)/1 (customer)/2 (prospect)/3 (customer and prospect)','s.fournisseur'=>'0 or 1','s.datec'=>dol_print_date(dol_now(),'%Y-%m-%d'),'s.code_client'=>'CU01-0001 or empty or "auto"','s.code_fournisseur'=>'SU01-0001 or empty or "auto"','s.address'=>"61 jump street",'s.zip'=>"123456",'s.town'=>"Big town",'s.fk_pays'=>'US, FR, DE...','s.phone'=>"0101010101",'s.fax'=>"0101010102",'s.url'=>"http://mycompany.com",'s.email'=>"test@mycompany.com",'s.siret'=>"",'s.siren'=>"",'s.ape'=>"",'s.idprof4'=>"",'s.idprof5'=>"",'s.idprof6'=>"",'s.tva_intra'=>"FR0123456789",'s.capital'=>"10000",'s.note_private'=>"This is an example of private note for record",'s.note_public'=>"This is an example of public note for record",'s.fk_typent'=>"2",'s.fk_effectif'=>"3","s.fk_forme_juridique"=>"1",'s.fk_prospectlevel'=>'PL_MEDIUM','s.fk_stcomm'=>'0','s.default_lang'=>'en_US','s.barcode'=>'123456789','s.datec'=>"2015-01-01 or 2015-01-01 12:30:00");
+		$this->import_updatekeys_array[$r]=array('s.nom'=>'Name','s.code_client'=>'CustomerCode','s.code_fournisseur'=>'SupplierCode','s.code_compta'=>'CustomerAccountancyCode','s.code_compta_fournisseur'=>'SupplierAccountancyCode');
 
 		// Import list of contact and attributes
 		$r++;

htdocs/imports/class/import.class.php

@@ -38,6 +38,7 @@ class Import
 	var $array_import_fieldshidden;
 	var $array_import_entities;
 	var $array_import_regex;
+	var $array_import_updatekeys;
 	var $array_import_examplevalues;
 	var $array_import_convertvalue;
 	var $array_import_run_sql_after;
@@ -153,6 +154,8 @@ class Import
 						$this->array_import_entities[$i]=$module->import_entities_array[$r];
 						// Tableau des alias a exporter (cle=champ, valeur=alias)
 						$this->array_import_regex[$i]=$module->import_regex_array[$r];
+						// Array of columns allowed as UPDATE options
+						$this->array_import_updatekeys[$i]=$module->import_updatekeys_array[$r];
 						// Array of examples
 						$this->array_import_examplevalues[$i]=$module->import_examplevalues_array[$r];
 						// Tableau des regles de conversion d'une valeur depuis une autre source (cle=champ, valeur=tableau des regles)

htdocs/imports/import.php

@@ -76,7 +76,8 @@ $import_name		= GETPOST('import_name');
 $hexa				= GETPOST('hexa');
 $importmodelid		= GETPOST('importmodelid');
 $excludefirstline	= (GETPOST('excludefirstline') ? GETPOST('excludefirstline') : 1);
-$endatlinenb     	= (GETPOST('endatlinenb') ? GETPOST('endatlinenb') : '');
+$endatlinenb		= (GETPOST('endatlinenb') ? GETPOST('endatlinenb') : '');
+$updatekeys			= (GETPOST('updatekeys') ? GETPOST('updatekeys') : array());
 $separator			= (GETPOST('separator') ? GETPOST('separator') : (! empty($conf->global->IMPORT_CSV_SEPARATOR_TO_USE)?$conf->global->IMPORT_CSV_SEPARATOR_TO_USE:','));
 $enclosure			= (GETPOST('enclosure') ? GETPOST('enclosure') : '"');
 
@@ -1184,8 +1185,9 @@ if ($step == 5 && $datatoimport)
 
 	$param='&leftmenu=import&format='.$format.'&datatoimport='.$datatoimport.'&filetoimport='.urlencode($filetoimport).'&nboflines='.$nboflines.'&separator='.urlencode($separator).'&enclosure='.urlencode($enclosure);
 	$param2 = $param;  // $param2 = $param without excludefirstline and endatlinenb
-	if ($excludefirstline) $param.='&excludefirstline='.$excludefirstline;
+	if ($excludefirstline)		$param.='&excludefirstline='.$excludefirstline;
-	if ($endatlinenb)      $param.='&endatlinenb='.$endatlinenb;
+	if ($endatlinenb)			$param.='&endatlinenb='.$endatlinenb;
+	if (!empty($updatekeys))	$param.='&updatekeys[]='.implode('&updatekeys[]=', $updatekeys);
 	
 	llxHeader('',$langs->trans("NewImport"),'EN:Module_Imports_En|FR:Module_Imports|ES:Módulo_Importaciones');
 
@@ -1291,6 +1293,24 @@ if ($step == 5 && $datatoimport)
 	}
 	print '</td></tr>';
 	
+	print '<tr><td>';
+	print $langs->trans("KeysToUseForUpdates");
+	print '</td><td>';
+	if($action=='launchsimu') {
+		print $form->multiselectarray('updatekeysbis', $objimport->array_import_updatekeys[0], $updatekeys, 0, 0, '', 1, '', 'disabled');
+		foreach($updatekeys as $val) {
+			print '<input type="hidden" name="updatekeys[]" value="'.$val.'">';
+		}
+		print ' &nbsp; <a href="'.$_SERVER["PHP_SELF"].'?step=5'.$param.'">'.$langs->trans("Modify").'</a>';
+	} else {
+		print $form->multiselectarray('updatekeys', $objimport->array_import_updatekeys[0], $updatekeys, 0, 0, '', 1, '80%');
+		print $form->textwithpicto("", $langs->trans("SelectColumnsOfYourFileForUpdateAttempt"));
+	}
+	/*echo '<pre>';
+	print_r($objimport->array_import_updatekeys);
+	echo '</pre>';*/
+	print '</td></tr>';
+	
 	print '</table>';
 
 	print '<br>';
@@ -1424,7 +1444,7 @@ if ($step == 5 && $datatoimport)
                 if ($endatlinenb && ($sourcelinenb > $endatlinenb)) continue;
                 
                 // Run import
-                $result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid);
+                $result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid,$updatekeys);
                 
                 if (count($obj->errors))   $arrayoferrors[$sourcelinenb]=$obj->errors;
                 if (count($obj->warnings)) $arrayofwarnings[$sourcelinenb]=$obj->warnings;
@@ -1460,9 +1480,13 @@ if ($step == 5 && $datatoimport)
         $db->rollback();    // We force rollback because this was just a simulation.
 
         // Show OK
-        if (! count($arrayoferrors) && ! count($arrayofwarnings)) print '<center>'.img_picto($langs->trans("OK"),'tick').' <b>'.$langs->trans("NoError").'</b></center><br><br>';
+        if (! count($arrayoferrors) && ! count($arrayofwarnings)) {
+        	print '<center>'.img_picto($langs->trans("OK"),'tick').' <b>'.$langs->trans("NoError").'</b></center><br><br>';
+			print $langs->trans("NbInsert", $obj->nbinsert).'<br>';
+			print $langs->trans("NbUpdate", $obj->nbupdate).'<br><br>';
+		}
         else print '<b>'.$langs->trans("NbOfLinesOK",$nbok).'</b><br><br>';
-        
+
         // Show Errors
         //var_dump($arrayoferrors);
         if (count($arrayoferrors))
@@ -1778,7 +1802,7 @@ if ($step == 6 && $datatoimport)
 			if ($endatlinenb && ($sourcelinenb > $endatlinenb)) continue;
 
 			// Run import
-			$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid);
+			$result=$obj->import_insert($arrayrecord,$array_match_file_to_database,$objimport,count($fieldssource),$importid,$updatekeys);
 			
 			if (count($obj->errors))   $arrayoferrors[$sourcelinenb]=$obj->errors;
 			if (count($obj->warnings))	$arrayofwarnings[$sourcelinenb]=$obj->warnings;
@@ -1824,7 +1848,9 @@ if ($step == 6 && $datatoimport)
 	// Show result
 	print '<br>';
 	print '<div class="center">';
-	print $langs->trans("NbOfLinesImported",$nbok).'</b><br><br>';
+	print $langs->trans("NbOfLinesImported",$nbok).'</b><br>';
+	print $langs->trans("NbInsert", $obj->nbinsert).'<br>';
+	print $langs->trans("NbUpdate", $obj->nbupdate).'<br><br>';
 	print $langs->trans("FileWasImported",$importid).'<br>';
 	print $langs->trans("YouCanUseImportIdToFindRecord",$importid).'<br>';
 	print '</div>';

htdocs/langs/en_US/exports.lang

@@ -120,3 +120,8 @@ SelectFilterFields=If you want to filter on some values, just input values here.
 FilteredFields=Filtered fields
 FilteredFieldsValues=Value for filter
 FormatControlRule=Format control rule
+## imports updates
+KeysToUseForUpdates=Key to use for updating data
+NbInsert=Number of inserted lines: %s
+NbUpdate=Number of updated lines: %s
+MultipleRecordFoundWithTheseFilters=Multiple records have been found with these filters: %s