0011842: import full related record data
authorPhilipp Schüle <p.schuele@metaways.de>
Mon, 9 May 2016 10:47:30 +0000 (12:47 +0200)
committerPhilipp Schüle <p.schuele@metaways.de>
Mon, 9 May 2016 11:37:48 +0000 (13:37 +0200)
* allows to import all related record (scalar) fields

side effects:
* setRelations(): use existing relations if only relation id is missing
* makes deleteLinkedRelations public (to be able to
 use it in test case cleanup)
* show relations in CRM lead duplicate conflict panel
* improve plugin failure message

https://forge.tine20.org/view.php?id=11842

Change-Id: Ic9e01d3d5797ff009f5bffbb02ef6f7c34be2cd6
Reviewed-on: http://gerrit.tine20.com/customers/3131
Tested-by: Jenkins CI (http://ci.tine20.com/)
Reviewed-by: Philipp Schüle <p.schuele@metaways.de>
tests/tine20/Crm/Import/CsvTest.php
tests/tine20/TestCase.php
tine20/Crm/js/Model.js
tine20/Tinebase/Controller/Record/Abstract.php
tine20/Tinebase/Import/Abstract.php
tine20/Tinebase/Import/Csv/Abstract.php
tine20/Tinebase/Pluggable/Abstract.php
tine20/Tinebase/Record/RecordSet.php
tine20/Tinebase/Relations.php

index 564254d..fef619f 100644 (file)
@@ -60,10 +60,11 @@ class Crm_Import_CsvTest extends ImportTestCase
      * @param        $importFilename
      * @param string $definitionName
      * @param bool   $dryrun
+     * @param string $duplicateResolveStrategy
      * @return array
      * @throws Tinebase_Exception_NotFound
      */
-    protected function _importHelper($importFilename, $definitionName = 'crm_tine_import_csv', $dryrun = true)
+    protected function _importHelper($importFilename, $definitionName = 'crm_tine_import_csv', $dryrun = true, $duplicateResolveStrategy = null)
     {
         $this->_testNeedsTransaction();
 
@@ -76,6 +77,10 @@ class Crm_Import_CsvTest extends ImportTestCase
             'dryrun' => $dryrun,
         );
 
+        if ($duplicateResolveStrategy) {
+            $options['duplicateResolveStrategy'] = $duplicateResolveStrategy;
+        }
+
         $result = $this->_doImport($options, $definitionName);
 
         return $result;
index d78fec6..6ae7211 100644 (file)
@@ -17,6 +17,8 @@ require_once __DIR__ . DIRECTORY_SEPARATOR . 'TestHelper.php';
  * Abstract test class
  * 
  * @package     Tests
+ *
+ * TODO separation of concerns: split into multiple classes/traits with cleanup / fixture / ... functionality
  */
 abstract class TestCase extends PHPUnit_Framework_TestCase
 {
@@ -235,8 +237,6 @@ abstract class TestCase extends PHPUnit_Framework_TestCase
      */
     protected function _deleteUsers()
     {
-
-
         foreach ($this->_usernamesToDelete as $username) {
             try {
                 if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__
@@ -251,6 +251,24 @@ abstract class TestCase extends PHPUnit_Framework_TestCase
     }
 
     /**
+     * removes records and their relations
+     *
+     * @param Tinebase_Record_RecordSet $records
+     */
+    protected function _deleteRecordRelations($records, $modelsToDelete = array(), $typesToDelete = array())
+    {
+        $controller = Tinebase_Core::getApplicationInstance($records->getRecordClassName());
+
+        if (! method_exists($controller, 'deleteLinkedRelations')) {
+            return;
+        }
+
+        foreach ($records as $record) {
+            $controller->deleteLinkedRelations($record, $modelsToDelete, $typesToDelete);
+        }
+    }
+
+    /**
      * get personal container
      * 
      * @param string $applicationName
index 98907bd..8989c92 100644 (file)
@@ -38,7 +38,7 @@ Tine.Crm.Model.Lead = Tine.Tinebase.data.Record.create(Tine.Tinebase.Model.gener
         {name: 'customer', omitDuplicateResolving: true, label: 'Customer', group: 'Relationen'},
         {name: 'partner', omitDuplicateResolving: true, label: 'Partner', group: 'Relationen'},
         {name: 'tasks', omitDuplicateResolving: true},
-        {name: 'relations', omitDuplicateResolving: true},
+        {name: 'relations', label: 'Relationen', group: 'Relationen'},
         {name: 'products', omitDuplicateResolving: true, label: 'Products', group: 'Relationen'},
         {name: 'tags', label: 'Tags'},
         {name: 'notes', omitDuplicateResolving: true},
index bddfb1c..1a19aa9 100644 (file)
@@ -1583,7 +1583,8 @@ abstract class Tinebase_Controller_Record_Abstract
         }
         
         if ($_record->has('relations')) {
-            $this->_deleteLinkedRelations($_record);
+            // TODO check if this needs to be done, as we might already deleting this "from the other side"
+            $this->deleteLinkedRelations($_record);
         }
 
         if ($_record->has('attachments') && Tinebase_Core::isFilesystemAvailable()) {
@@ -1594,30 +1595,36 @@ abstract class Tinebase_Controller_Record_Abstract
     /**
      * delete linked relations
      * 
-     * @param Tinebase_Record_Interface $_record
-     * 
-     * TODO check if this needs to be done, as we might already deleting this "from the other side"
+     * @param Tinebase_Record_Interface $record
+     * @param array $modelsToDelete
+     * @param array $typesToDelete
      */
-    protected function _deleteLinkedRelations(Tinebase_Record_Interface $_record)
+    public function deleteLinkedRelations(Tinebase_Record_Interface $record, $modelsToDelete = array(), $typesToDelete = array())
     {
-        $relations = Tinebase_Relations::getInstance()->getRelations($this->_modelName, $this->_getBackendType(), $_record->getId());
-        if (empty($relations)) {
+        $relations = isset($record->relations) && $record->relations instanceof Tinebase_Record_RecordSet
+            ? $record->relations
+            : Tinebase_Relations::getInstance()->getRelations($this->_modelName, $this->_getBackendType(), $record->getId());
+
+        if (count($relations) === 0) {
             return;
         }
 
-        // remove relations
-        Tinebase_Relations::getInstance()->setRelations($this->_modelName, $this->_getBackendType(), $_record->getId(), array());
+        // unset record relations
+        Tinebase_Relations::getInstance()->setRelations($this->_modelName, $this->_getBackendType(), $record->getId(), array());
 
-        if (empty($this->_relatedObjectsToDelete)) {
+        if (empty($modelsToDelete)) {
+            $modelsToDelete = $this->_relatedObjectsToDelete;
+        }
+        if (empty($modelsToDelete) && empty($typesToDelete)) {
             return;
         }
         
         // remove related objects
         Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Deleting all '
-                . implode(',', $this->_relatedObjectsToDelete) . ' relations.');
+            . implode(',', $modelsToDelete) . ' relations.');
 
         foreach ($relations as $relation) {
-            if (in_array($relation->related_model, $this->_relatedObjectsToDelete)) {
+            if (in_array($relation->related_model, $modelsToDelete) || in_array($relation->type, $typesToDelete)) {
                 list($appName, $i, $itemName) = explode('_', $relation->related_model);
                 $appController = Tinebase_Core::getApplicationInstance($appName, $itemName);
 
index 6197060..d62b7a0 100644 (file)
@@ -593,67 +593,102 @@ abstract class Tinebase_Import_Abstract implements Tinebase_Import_Interface
         }
         
         $values = (isset($field['separator'])) ? $this->_splitBySeparator($field['separator'], $fieldValue): array($fieldValue);
-        
+
         $relations = array();
-        
         foreach ($values as $value) {
-            // check if related record exists
-            $controller = Tinebase_Core::getApplicationInstance($field['related_model']);
-            $filterModel = $field['related_model'] . 'Filter';
-            $operator = isset($field['operator']) ? $field['operator'] : 'equals';
+            $relation = $this->_getRelationForValue($value, $field, $data);
+            $relations[] = $relation;
+        }
 
-            $filterValueToAdd = '';
-            if (isset($field['filterValueAdd']) && isset($data[$field['filterValueAdd']])) {
-                if ($field['filter'] === 'query') {
-                    $filterValueToAdd = ' ' . $data[$field['filterValueAdd']];
-                } else {
-                    if (Tinebase_Core::isLogLevel(Zend_Log::WARN)) {
-                        Tinebase_Core::getLogger()->warn(__METHOD__ . '::' . __LINE__
-                            . ' "filterValueAdd" Currently only working for query filter');
-                    }
-                }
+        // TODO how do we handle this with multiple relations/values?
+        if (isset($field['targetField']) && isset($field['targetFieldData']) && count($relations) > 0) {
+            $this->_setTargetFieldFromRelation($field, $data, $relations[0]);
+        }
+        
+        return $relations;
+    }
+
+    protected function _setTargetFieldFromRelation($field, &$data, $relation)
+    {
+        $unreplaced = $targetField = $field['targetFieldData'];
+        $recordArray = $relation['related_record'];
+        foreach ($recordArray as $key => $value) {
+            if (preg_match('/' . preg_quote($key) . '/', $targetField) && is_scalar($value)) {
+                $targetField = preg_replace('/' . preg_quote($key) . '/', $value, $targetField);
+                $unreplaced = preg_replace('/^[, ]*' . preg_quote($key) . '/', '', $unreplaced);
             }
+        }
 
-            $filter = new $filterModel(array(
-                array('field' => $field['filter'], 'operator' => $operator, 'value' => $value . $filterValueToAdd)
-            ));
-            $result = $controller->search($filter);
-            if (count($result) > 0) {
-                $record = $result->getFirstRecord()->toArray();
+        // remove unreplaced stuff
+        $targetField = str_replace($unreplaced, '', $targetField);
+
+        // finally set the target field value
+        $data[$field['targetField']] = trim($targetField);
+    }
+
+    protected function _getRelationForValue($value, $field, $data)
+    {
+        // check if related record exists
+        $controller = Tinebase_Core::getApplicationInstance($field['related_model']);
+        $filterModel = $field['related_model'] . 'Filter';
+        $operator = isset($field['operator']) ? $field['operator'] : 'equals';
+
+        $filterValueToAdd = '';
+        if (isset($field['filterValueAdd']) && isset($data[$field['filterValueAdd']])) {
+            if ($field['filter'] === 'query') {
+                $filterValueToAdd = ' ' . $data[$field['filterValueAdd']];
             } else {
-                // create new related record
-                $record = array(
-                    (isset($field['related_field']) ? $field['related_field'] : $field['filter']) => $value
-                );
-                if (! empty($filterValueToAdd)) {
-                    $record[str_replace($field['destination'] . '_', '', $field['filterValueAdd'])] = trim($filterValueToAdd);
+                if (Tinebase_Core::isLogLevel(Zend_Log::WARN)) {
+                    Tinebase_Core::getLogger()->warn(__METHOD__ . '::' . __LINE__
+                        . ' "filterValueAdd" Currently only working for query filter');
                 }
             }
+        }
+
+        $filter = new $filterModel(array(
+            array('field' => $field['filter'], 'operator' => $operator, 'value' => $value . $filterValueToAdd)
+        ));
+        $result = $controller->search($filter, null, /* $_getRelations */ true);
+        $relation = $this->_getRelationData($result->getFirstRecord(), $field, $data, $value);
+
+        return $relation;
+    }
+
+    protected function _getRelationData($record, $field, $data, $value)
+    {
+        $relationType = $field['destination'];
+        $relation = array(
+            'type'          => $relationType,
+            'related_model' => $field['related_model'],
+            // TODO move this to product (modelconfig?)
+            'remark'        => $relationType == 'PRODUCT' ? array('quantity' => 1) : null,
+        );
 
-            $relation = array(
-                'type'  => $field['destination'], 
-                'related_record' => $record,
-                // TODO move this to product (modelconfig?)
-                'remark' => $field['destination'] == 'PRODUCT' ? array('quantity' => 1) : null
+        if ($record) {
+            $relation['related_id'] = $record->getId();
+            $recordArray = $record->toArray();
+        } else {
+            // create new related record
+            $recordArray = array(
+                (isset($field['related_field']) ? $field['related_field'] : $field['filter']) => $value
             );
-            
-            $relations[] = $relation;
+            if (! empty($filterValueToAdd)) {
+                $recordArray[str_replace($relationType . '_', '', $field['filterValueAdd'])] = trim($filterValueToAdd);
+            }
         }
 
-        if (isset($field['targetField'])&& isset($field['targetFieldData']) && isset($record)) {
-            $unreplaced = $targetField = $field['targetFieldData'];
-            foreach ($record as $key => $value) {
-                if (preg_match('/' . preg_quote($key) . '/', $targetField) && is_scalar($value)) {
-                    $targetField = preg_replace('/' . preg_quote($key) . '/', $value, $targetField);
-                    $unreplaced = preg_replace('/^[, ]*' . preg_quote($key) . '/', '', $unreplaced);
-                }
+        // add more data for this relation if available
+        foreach ($data as $key => $value) {
+            $regex = '/^' . preg_quote($relationType) . '_/';
+            if (preg_match($regex, $key)) {
+                $relatedField = preg_replace($regex, '', $key);
+                $recordArray[$relatedField] = trim($value);
             }
-            // remove unreplaced stuff
-            $targetField = str_replace($unreplaced, '', $targetField);
-            $data[$field['targetField']] = $targetField;
         }
-        
-        return $relations;
+
+        $relation['related_record'] = $recordArray;
+
+        return $relation;
     }
 
     /**
index fb00a89..df6cc19 100644 (file)
@@ -205,7 +205,10 @@ abstract class Tinebase_Import_Csv_Abstract extends Tinebase_Import_Abstract
                 foreach ($destinations as $destination) {
                     if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__
                         . ' destination ' . $destination);
-                    $data[$destination] = $values[$i++];
+                    if (isset($values[$i])) {
+                        $data[$destination] = trim($values[$i]);
+                    }
+                    $i++;
                 }
             } else {
                 $data[$field['destination']] = $value;
index 086e00c..e2e083e 100644 (file)
@@ -80,7 +80,7 @@ abstract class Tinebase_Pluggable_Abstract
             ), $args);
         } else {
             throw new Tinebase_Exception(
-                'Plugin ' . $method . ' was not found in haystack');
+                'Plugin ' . $method . ' was not found in haystack of class ' . get_class($this));
         }
     }
 }
index e20420b..64ab3cf 100644 (file)
@@ -494,6 +494,8 @@ class Tinebase_Record_RecordSet implements IteratorAggregate, Countable, ArrayAc
      *
      * @param array $_properties
      * @return $this
+     *
+     * TODO implement
      */
     public function addIndices(array $_properties)
     {
index 0b7e709..a2c7740 100644 (file)
@@ -99,13 +99,13 @@ class Tinebase_Relations
         // compute relations to add/delete
         $currentRelations = $this->getRelations($_model, $_backend, $_id, NULL, array(), $_ignoreACL);
         $currentIds   = $currentRelations->getArrayOfIds();
-        $relationsIds = $relations->getArrayOfIds();
+        $relationsIds = $this->_getRelationIds($relations, $currentRelations);
         
         $toAdd = $relations->getIdLessIndexes();
         $toDel = array_diff($currentIds, $relationsIds);
         $toUpdate = array_intersect($currentIds, $relationsIds);
 
-        // prevent two empty related_id s of the same relation type
+        // prevent two empty related_ids of the same relation type
         $emptyRelatedId = array();
         foreach ($toAdd as $idx) {
             if (empty($relations[$idx]->related_id)) {
@@ -174,7 +174,44 @@ class Tinebase_Relations
             }
         }
     }
-    
+
+    /**
+     * appends missing relation ids if related records + type match
+     *
+     * @param $relations
+     * @param $currentRelations
+     * @return mixed
+     */
+    protected function _getRelationIds($relations, $currentRelations)
+    {
+        $clonedRelations = clone $relations;
+
+        if (count($currentRelations) > 0) {
+            foreach ($clonedRelations as $relation) {
+                if ($relation->getId()) {
+                    continue;
+                }
+
+                // if relation has no id, maybe we have the same relation already in current relations
+                $subset = $currentRelations->filter('own_id', $relation->own_id)
+                    ->filter('related_id', $relation->related_id)
+                    ->filter('type', $relation->type);
+
+                if (count($subset) === 1) {
+                    // remove and add to make sure index is updated in record set
+                    $relations->removeRecord($relation);
+                    $relation->setId($subset->getFirstRecord()->getId());
+                    $relations->addRecord($relation);
+                    //$result[] = $subset->getFirstRecord()->getId();
+                }
+            }
+        }
+
+        $result = $relations->getArrayOfIds();
+
+        return $result;
+    }
+
     /**
      * returns the constraints config for the given models and their mirrored values (seen from the other side
      *