From 0674d062f8b0b3f07219ae674b138846f5fc39e5 Mon Sep 17 00:00:00 2001 From: lmarcouiller Date: Fri, 15 Apr 2022 11:55:24 +0200 Subject: [PATCH] select on source fields import --- htdocs/imports/import.php | 583 +++++++++++++++++++++++++++++++- htdocs/langs/en_US/exports.lang | 1 + 2 files changed, 580 insertions(+), 4 deletions(-) diff --git a/htdocs/imports/import.php b/htdocs/imports/import.php index 01f152f1ea5..b3303402d22 100644 --- a/htdocs/imports/import.php +++ b/htdocs/imports/import.php @@ -35,6 +35,8 @@ require_once DOL_DOCUMENT_ROOT.'/core/lib/import.lib.php'; // Load translation files required by the page $langs->loadLangs(array('exports', 'compta', 'errors')); +$selectimport = true; + // Security check $result = restrictedArea($user, 'import'); @@ -136,7 +138,7 @@ $step = (GETPOST('step') ? GETPOST('step') : 1); $import_name = GETPOST('import_name'); $hexa = GETPOST('hexa'); $importmodelid = GETPOST('importmodelid'); -$excludefirstline = (GETPOST('excludefirstline') ? GETPOST('excludefirstline') : 1); +$excludefirstline = (GETPOST('excludefirstline') ? GETPOST('excludefirstline') : 2); $endatlinenb = (GETPOST('endatlinenb') ? GETPOST('endatlinenb') : ''); $updatekeys = (GETPOST('updatekeys', 'array') ? GETPOST('updatekeys', 'array') : array()); $separator = (GETPOST('separator', 'nohtml') ? GETPOST('separator', 'nohtml') : (!empty($conf->global->IMPORT_CSV_SEPARATOR_TO_USE) ? $conf->global->IMPORT_CSV_SEPARATOR_TO_USE : ',')); @@ -763,7 +765,7 @@ if ($step == 3 && $datatoimport) { // STEP 4: Page to make matching between source file and database fields -if ($step == 4 && $datatoimport) { +if ($step == 4 && $datatoimport && !$selectimport) { $model = $format; $list = $objmodelimport->liste_modeles($db); @@ -1315,6 +1317,565 @@ if ($step == 4 && $datatoimport) { } } +if ($step == 4 && $datatoimport && $selectimport) { + $model = $format; + $list = $objmodelimport->liste_modeles($db); + + // Create classe to use for import + $dir = DOL_DOCUMENT_ROOT."/core/modules/import/"; + $file = "import_".$model.".modules.php"; + $classname = "Import".ucfirst($model); + require_once $dir.$file; + $obj = new $classname($db, $datatoimport); + if ($model == 'csv') { + $obj->separator = $separator_used; + $obj->enclosure = $enclosure; + } + if ($model == 'xlsx') { + if (!preg_match('/\.xlsx$/i', $filetoimport)) { + $langs->load("errors"); + $param = '&datatoimport='.$datatoimport.'&format='.$format; + setEventMessages($langs->trans("ErrorFileMustHaveFormat", $model), null, 'errors'); + header("Location: ".$_SERVER["PHP_SELF"].'?step=3'.$param.'&filetoimport='.urlencode($relativepath)); + exit; + } + } + + if (GETPOST('update')) { + $array_match_file_to_database = array(); + } + + // Load source fields in input file + $fieldssource = array(); + $result = $obj->import_open_file($conf->import->dir_temp.'/'.$filetoimport, $langs); + if ($result >= 0) { + // Read first line + $arrayrecord = $obj->import_read_record(); + // Put into array fieldssource starting with 1. + $i = 1; + foreach ($arrayrecord as $key => $val) { + $fieldssource[$i]['example1'] = dol_trunc($val['val'], 24); + $i++; + } + $obj->import_close_file(); + } + + // Load targets fields in database + $fieldstarget = $objimport->array_import_fields[0]; + $fieldstarget_tmp = array(); + + foreach ($fieldstarget as $key => $label) { + $isrequired = preg_match('/\*$/', $label); + if (!empty($isrequired)) { + $newlabel = substr($label, 0, -1); + $fieldstarget_tmp[$key] = array("label"=>$newlabel,"required"=>true); + } else { + $fieldstarget_tmp[$key] = array("label"=>$label,"required"=>false); + } + $fieldstarget_tmp[$key]["imported"] = true; + } + $fieldstarget = $fieldstarget_tmp; + $maxpos = max(count($fieldssource), count($fieldstarget)); + + //var_dump($array_match_file_to_database); + + // Is it a first time in page (if yes, we must initialize array_match_file_to_database) + if (count($array_match_file_to_database) == 0) { + // This is first input in screen, we need to define + // $array_match_file_to_database + // $serialized_array_match_file_to_database + // $_SESSION["dol_array_match_file_to_database"] + $pos = 1; + $num = count($fieldssource); + while ($pos <= $num) { + if ($num >= 1 && $pos <= $num) { + $posbis = 1; + foreach ($fieldstarget as $key => $val) { + if ($posbis < $pos) { + $posbis++; + continue; + } + // We found the key of targets that is at position pos + $array_match_file_to_database[$pos] = $key; + if ($serialized_array_match_file_to_database) { + $serialized_array_match_file_to_database .= ','; + } + $serialized_array_match_file_to_database .= ($pos.'='.$key); + break; + } + } + $pos++; + } + // Save the match array in session. We now will use the array in session. + $_SESSION["dol_array_match_file_to_database"] = $serialized_array_match_file_to_database; + } + $array_match_database_to_file = array_flip($array_match_file_to_database); + + //print $serialized_array_match_file_to_database; + //print $_SESSION["dol_array_match_file_to_database"]; + //var_dump($array_match_file_to_database);exit; + + // Now $array_match_file_to_database contains fieldnb(1,2,3...)=>fielddatabase(key in $array_match_file_to_database) + + $param = '&format='.$format.'&datatoimport='.urlencode($datatoimport).'&filetoimport='.urlencode($filetoimport); + if ($excludefirstline) { + $param .= '&excludefirstline='.urlencode($excludefirstline); + } + if ($endatlinenb) { + $param .= '&endatlinenb='.urlencode($endatlinenb); + } + if ($separator) { + $param .= '&separator='.urlencode($separator); + } + if ($enclosure) { + $param .= '&enclosure='.urlencode($enclosure); + } + + llxHeader('', $langs->trans("NewImport"), $help_url); + + $head = import_prepare_head($param, 4); + + print dol_get_fiche_head($head, 'step4', '', -2); + + print '
'; + print '
'; + + print ''; + + // Module + print ''; + print ''; + + // Lot de donnees a importer + print ''; + print ''; + + print '
'.$langs->trans("Module").''; + $titleofmodule = $objimport->array_import_module[0]['module']->getName(); + // Special cas for import common to module/services + if (in_array($objimport->array_import_code[0], array('produit_supplierprices', 'produit_multiprice', 'produit_languages'))) { + $titleofmodule = $langs->trans("ProductOrService"); + } + print $titleofmodule; + print '
'.$langs->trans("DatasetToImport").''; + $entity = preg_replace('/:.*$/', '', $objimport->array_import_icon[0]); + $entityicon = strtolower(!empty($entitytoicon[$entity]) ? $entitytoicon[$entity] : $entity); + print img_object($objimport->array_import_module[0]['module']->getName(), $entityicon).' '; + print $objimport->array_import_label[0]; + print '
'; + print '
'; + + print load_fiche_titre($langs->trans("InformationOnSourceFile"), '', 'file-export'); + + print '
'; + print '
'; + print ''; + + // Source file format + print ''; + print ''; + + // Separator and enclosure + if ($model == 'csv') { + print ''; + print ''; + } + + // File to import + print ''; + print ''; + + print '
'.$langs->trans("SourceFileFormat").''; + $text = $objmodelimport->getDriverDescForKey($format); + print $form->textwithpicto($objmodelimport->getDriverLabelForKey($format), $text); + print '
'.$langs->trans("CsvOptions").''; + print '
'; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print $langs->trans("Separator").' : '; + print ''; + print '    '.$langs->trans("Enclosure").' : '; + print ' '; + print ''; + print '
'; + print '
'.$langs->trans("FileToImport").''; + $modulepart = 'import'; + $relativepath = GETPOST('filetoimport'); + print ''; + print img_mime($file, '', 'pictofixedwidth'); + print $filetoimport; + print ''; + print '
'; + print '
'; + + print dol_get_fiche_end(); + + print '
'."\n"; + + + // List of source fields + print ''."\n"; + print '
'; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + + print '
'; + print ''; + $s = $langs->trans("SelectImportFields", '{s1}'); + $s = str_replace('{s1}', img_picto('', 'grip_title', '', false, 0, 0, '', '', 0), $s); + print $s; + print ' '; + $htmlother->select_import_model($importmodelid, 'importmodelid', $datatoimport, 1, $user->id); + print ''; + print '
'; + print '
'; + + // Title of array with fields + print '
'; // You can use div-table-responsive-no-min if you dont need reserved height for your table + print ''; + print ''; + print ''; + print ''; + print ''; + + //var_dump($array_match_file_to_database); + + print ''; + + // List of not imported fields + print ''; + + print ''; + print ''; + + print '
'.$langs->trans("FieldsInSourceFile").''.$langs->trans("FieldsInTargetDatabase").'
'; + + $fieldsplaced = array(); + $valforsourcefieldnb = array(); + $listofkeys = array(); + foreach ($array_match_file_to_database as $key => $val) { + $listofkeys[$key] = 1; + } + + print "\n\n"; + print '
'."\n"; + + // List of source fields + $var = true; + $lefti = 1; + foreach ($array_match_file_to_database as $key => $val) { + $var = !$var; + show_elem($fieldssource, $key, $val, $var); // key is field number in source file + //print '> '.$lefti.'-'.$key.'-'.$val; + $listofkeys[$key] = 1; + $fieldsplaced[$key] = 1; + $valforsourcefieldnb[$lefti] = $key; + $lefti++; + + if ($lefti > count($fieldstarget)) { + break; // Other fields are in the not imported area + } + } + //var_dump($valforsourcefieldnb); + + // Complete source fields from count($fieldssource)+1 to count($fieldstarget) + $more = 1; + $num = count($fieldssource); + while ($lefti <= $num) { + $var = !$var; + $newkey = getnewkey($fieldssource, $listofkeys); + show_elem($fieldssource, $newkey, '', $var); // key start after field number in source file + //print '> '.$lefti.'-'.$newkey; + $listofkeys[$key] = 1; + $lefti++; + $more++; + } + + print "
\n"; + print "\n"; + + + print '		'; + + // List of target fields + $height = '24px'; //needs px for css height attribute below + $i = 0; + $mandatoryfieldshavesource = true; + + print ''; + + foreach ($fieldstarget as $code => $line) { + print ''; + $entity = (!empty($objimport->array_import_entities[0][$code]) ? $objimport->array_import_entities[0][$code] : $objimport->array_import_icon[0]); + + $tablealias = preg_replace('/(\..*)$/i', '', $code); + $tablename = $objimport->array_import_tables[0][$tablealias]; + + $entityicon = !empty($entitytoicon[$entity]) ? $entitytoicon[$entity] : $entity; // $entityicon must string name of picto of the field like 'project', 'company', 'contact', 'modulename', ... + $entitylang = $entitytolang[$entity] ? $entitytolang[$entity] : $objimport->array_import_label[0]; // $entitylang must be a translation key to describe object the field is related to, like 'Company', 'Contact', 'MyModyle', ... + + print ''; + print '"; + print ''; + } + print '
=>'.img_object('', $entityicon).' '.$langs->trans($entitylang).''; + print ''; + //print ajax_combobox("select_".$newlabel); + print "'; + $filecolumn = !empty($array_match_database_to_file[$code])?$array_match_database_to_file[$code]:0; + // Source field info + $htmltext = ''.$langs->trans("FieldSource").'
'; + if ($filecolumn > count($fieldssource)) { + $htmltext .= $langs->trans("DataComeFromNoWhere").'
'; + } else { + if (empty($objimport->array_import_convertvalue[0][$code])) { // If source file does not need convertion + $filecolumntoshow = $filecolumn; + $htmltext .= $langs->trans("DataComeFromFileFieldNb", $filecolumntoshow).'
'; + } else { + if ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromref') { + $htmltext .= $langs->trans("DataComeFromIdFoundFromRef", $filecolumn, $langs->transnoentitiesnoconv($entitylang)).'
'; + } + if ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromcodeid') { + $htmltext .= $langs->trans("DataComeFromIdFoundFromCodeId", $filecolumn, $langs->transnoentitiesnoconv($objimport->array_import_convertvalue[0][$code]['dict'])).'
'; + } + } + } + // Source required + $htmltext .= $langs->trans("SourceRequired").': '.yn($line["label"]).'
'; + $example = !empty($objimport->array_import_examplevalues[0][$code])?$objimport->array_import_examplevalues[0][$code]:""; + // Example + if (empty($objimport->array_import_convertvalue[0][$code])) { // If source file does not need convertion + if ($example) { + $htmltext .= $langs->trans("SourceExample").': '.$example.'
'; + } + } else { + if ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromref') { + $htmltext .= $langs->trans("SourceExample").': '.$langs->transnoentitiesnoconv("ExampleAnyRefFoundIntoElement", $entitylang).($example ? ' ('.$langs->transnoentitiesnoconv("Example").': '.$example.')' : '').'
'; + } elseif ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromcodeid') { + $htmltext .= $langs->trans("SourceExample").': '.$langs->trans("ExampleAnyCodeOrIdFoundIntoDictionary", $langs->transnoentitiesnoconv($objimport->array_import_convertvalue[0][$code]['dict'])).($example ? ' ('.$langs->transnoentitiesnoconv("Example").': '.$example.')' : '').'
'; + } elseif ($example) { + $htmltext .= $langs->trans("SourceExample").': '.$example.'
'; + } + } + // Format control rule + if (!empty($objimport->array_import_regex[0][$code])) { + $htmltext .= $langs->trans("FormatControlRule").': '.$objimport->array_import_regex[0][$code].'
'; + } + $htmltext .= '
'; + // Target field info + $htmltext .= ''.$langs->trans("FieldTarget").'
'; + if (empty($objimport->array_import_convertvalue[0][$code])) { // If source file does not need convertion + $htmltext .= $langs->trans("DataIsInsertedInto").'
'; + } else { + if ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromref') { + $htmltext .= $langs->trans("DataIDSourceIsInsertedInto").'
'; + } + if ($objimport->array_import_convertvalue[0][$code]['rule'] == 'fetchidfromcodeid') { + $htmltext .= $langs->trans("DataCodeIDSourceIsInsertedInto").'
'; + } + } + $htmltext .= $langs->trans("FieldTitle").": ".$langs->trans($line["label"])."
"; + $htmltext .= $langs->trans("Table")." -> ".$langs->trans("Field").': '.$tablename." -> ".preg_replace('/^.*\./', '', $code)."
"; + print $form->textwithpicto($more, $htmltext); + print '
'; + + print '
'.$langs->trans("NotImportedFields").'
'; + + print "\n\n"; + print '\n"; + print "\n"; + + print ''; + $i = 0; + while ($i < $nbofnotimportedfields) { + // Print empty cells + show_elem('', '', 'none', $var, 'nostyle'); + $i++; + } + print '
'; + print '
'; + + + if ($conf->use_javascript_ajax) { + print ''."\n"; + } + + /* + * Action bar + */ + print '
'; + + if (count($array_match_file_to_database)) { + if ($mandatoryfieldshavesource) { + print ''.$langs->trans("NextStep").''; + } else { + print ''.$langs->trans("NextStep").''; + } + } + + print '
'; + + + // Area for profils import + if (count($array_match_file_to_database)) { + print '
'."\n"; + print ''."\n"; + print '
'.$langs->trans("SaveImportModel").'
'; + + print '
'; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + + print '
'; // You can use div-table-responsive-no-min if you dont need reserved height for your table + print ''; + print ''; + print ''; + print ''; + print ''; + print ''; + + print ''; + print ''; + print ''; + print ''; + + // List of existing import profils + $sql = "SELECT rowid, label, fk_user, entity"; + $sql .= " FROM ".MAIN_DB_PREFIX."import_model"; + $sql .= " WHERE type = '".$db->escape($datatoimport)."'"; + if (empty($conf->global->EXPORTS_SHARE_MODELS)) { // EXPORTS_SHARE_MODELS means all templates are visible, whatever is owner. + $sql .= " AND fk_user IN (0, ".((int) $user->id).")"; + } + $sql .= " ORDER BY rowid"; + + $resql = $db->query($sql); + if ($resql) { + $num = $db->num_rows($resql); + + $tmpuser = new user($db); + + $i = 0; + while ($i < $num) { + $obj = $db->fetch_object($resql); + + print ''; + print ''; + print ''; + $i++; + } + } else { + dol_print_error($db); + } + + print '
'.$langs->trans("ImportModelName").''.$langs->trans("Visibility").'
'; + $arrayvisibility = array('private'=>$langs->trans("Private"), 'all'=>$langs->trans("Everybody")); + print $form->selectarray('visibility', $arrayvisibility, 'private'); + print ''; + print ''; + print '
'; + print $obj->label; + print ''; + if (empty($obj->fk_user)) { + print $langs->trans("Everybody"); + } else { + $tmpuser->fetch($obj->fk_user); + print $tmpuser->getNomUrl(1); + } + print ''; + print 'rowid.'&filetoimport='.urlencode($filetoimport).'">'; + print img_delete(); + print ''; + print '
'; + print '
'; + + print '
'; + } +} // STEP 5: Summary of choices and launch simulation if ($step == 5 && $datatoimport) { @@ -1471,6 +2032,20 @@ if ($step == 5 && $datatoimport) { if ($action == 'launchsimu') { print '   '.$langs->trans("Modify").''; } + if ($excludefirstline == 2) { + print $form->textwithpicto("", $langs->trans("WarningFirstImportedLine", $excludefirstline), 1, 'warning', "warningexcludefirstline"); + print ''; + } print ''; // Keys for data UPDATE (not INSERT of new data) @@ -2121,7 +2696,7 @@ function show_elem($fieldssource, $pos, $key, $var, $nostyle = '') if ($pos && $pos > count($fieldssource)) { // No fields print ''; print ''; - print img_picto(($pos > 0 ? $langs->trans("MoveField", $pos) : ''), 'grip_title', 'class="boxhandle" style="cursor:move;"'); + //print img_picto(($pos > 0 ? $langs->trans("MoveField", $pos) : ''), 'grip_title', 'class="boxhandle" style="cursor:move;"'); print ''; print ''; print $langs->trans("NoFields"); @@ -2141,7 +2716,7 @@ function show_elem($fieldssource, $pos, $key, $var, $nostyle = '') print ''; print ''; // The image must have the class 'boxhandle' beause it's value used in DOM draggable objects to define the area used to catch the full object - print img_picto($langs->trans("MoveField", $pos), 'grip_title', 'class="boxhandle" style="cursor:move;"'); + //print img_picto($langs->trans("MoveField", $pos), 'grip_title', 'class="boxhandle" style="cursor:move;"'); print ''; print ''; print $langs->trans("Field").' '.$pos; diff --git a/htdocs/langs/en_US/exports.lang b/htdocs/langs/en_US/exports.lang index f2f2d2cf587..a20741472b7 100644 --- a/htdocs/langs/en_US/exports.lang +++ b/htdocs/langs/en_US/exports.lang @@ -135,3 +135,4 @@ NbInsert=Number of inserted lines: %s NbUpdate=Number of updated lines: %s MultipleRecordFoundWithTheseFilters=Multiple records have been found with these filters: %s StocksWithBatch=Stocks and location (warehouse) of products with batch/serial number +WarningFirstImportedLine=The first line(s) will not be imported with the current selection