0012106: improve import performance for duplicates
[tine20] / tests / tine20 / Addressbook / Import / CsvTest.php
1 <?php
2 /**
3  * Tine 2.0 - http://www.tine20.org
4  * 
5  * @package     Addressbook
6  * @license     http://www.gnu.org/licenses/agpl.html
7  * @copyright   Copyright (c) 2008-2015 Metaways Infosystems GmbH (http://www.metaways.de)
8  * @author      Philipp Schüle <p.schuele@metaways.de>
9  */
10
11 /**
12  * Test class for Addressbook_Import_Csv
13  */
14 class Addressbook_Import_CsvTest extends ImportTestCase
15 {
16     protected $_deletePersonalContacts = false;
17
18     protected $_importerClassName = 'Addressbook_Import_Csv';
19     protected $_exporterClassName = 'Addressbook_Export_Csv';
20     protected $_modelName = 'Addressbook_Model_Contact';
21
22     /**
23      * Sets up the fixture.
24      * This method is called before a test is executed.
25      *
26      * @access protected
27      */
28     protected function setUp()
29     {
30         // always resolve customfields
31         Addressbook_Controller_Contact::getInstance()->resolveCustomfields(TRUE);
32     }
33
34     /**
35      * Tears down the fixture
36      * This method is called after a test is executed.
37      *
38      * @access protected
39      */
40     protected function tearDown()
41     {
42         // cleanup
43         if (file_exists($this->_filename) && $this->_deleteImportFile) {
44             unlink($this->_filename);
45         }
46         
47         if ($this->_deletePersonalContacts) {
48             Addressbook_Controller_Contact::getInstance()->deleteByFilter(new Addressbook_Model_ContactFilter(array(array(
49                 'field' => 'container_id', 'operator' => 'equals', 'value' => Addressbook_Controller_Contact::getInstance()->getDefaultAddressbook()->getId()
50             ))));
51         }
52         
53         Addressbook_Controller_Contact::getInstance()->duplicateCheckFields(Addressbook_Config::getInstance()->get(Addressbook_Config::CONTACT_DUP_FIELDS));
54     }
55     
56     /**
57      * test import duplicate data
58      *
59      * @return array
60      */
61     public function testImportDuplicates()
62     {
63         $internalContainer = Tinebase_Container::getInstance()->getContainerByName('Addressbook', 'Internal Contacts', Tinebase_Model_Container::TYPE_SHARED);
64         $options = array(
65             'container_id'  => $internalContainer->getId(),
66         );
67         $result = $this->_doImport($options, 'adb_tine_import_csv', new Addressbook_Model_ContactFilter(array(
68             array('field' => 'container_id', 'operator' => 'equals', 'value' => $internalContainer->getId()),
69         )));
70
71         $this->assertGreaterThan(0, $result['duplicatecount'], 'no duplicates.');
72         $this->assertTrue($result['exceptions'] instanceof Tinebase_Record_RecordSet);
73
74         return $result;
75     }
76     
77     /**
78      * test import data
79      */
80     public function testImportSalutation()
81     {
82         $myContact = Addressbook_Controller_Contact::getInstance()->getContactByUserId(Tinebase_Core::getUser()->getId());
83         $salutation = Addressbook_Config::getInstance()->get(Addressbook_Config::CONTACT_SALUTATION)->records->getFirstRecord()->value;
84         $myContact->salutation = $salutation;
85         Addressbook_Controller_Contact::getInstance()->update($myContact);
86         
87         $result = $this->testImportDuplicates();
88         
89         $found = FALSE;
90         foreach ($result['exceptions'] as $exception) {
91             if ($exception['exception']['clientRecord']['email'] === Tinebase_Core::getUser()->accountEmailAddress) {
92                 $found = TRUE;
93                 $this->assertTrue(isset($exception['exception']['clientRecord']['salutation']), 'no salutation found: ' . print_r($exception['exception']['clientRecord'], TRUE));
94                 $this->assertEquals($salutation, $exception['exception']['clientRecord']['salutation']);
95                 break;
96             }
97         }
98         
99         $this->assertTrue($found,
100             'did not find user ' . Tinebase_Core::getUser()->accountFullName . ' in import exceptions: '
101             . print_r($result['exceptions']->toArray(), true));
102     }
103
104     /**
105      * test import umlaut
106      * 
107      * @see 0006936: detect import file encoding
108      */
109     public function testImportUmlaut()
110     {
111         $myContact = Addressbook_Controller_Contact::getInstance()->getContactByUserId(Tinebase_Core::getUser()->getId());
112         $myContact->org_name = 'Übel leckerer Äppler';
113         Addressbook_Controller_Contact::getInstance()->update($myContact);
114         
115         $result = $this->testImportDuplicates();
116         
117         $found = FALSE;
118         foreach ($result['exceptions'] as $exception) {
119             $record = $exception['exception']['clientRecord'];
120             if ($record['email'] === Tinebase_Core::getUser()->accountEmailAddress) {
121                 $found = TRUE;
122                 $this->assertEquals($myContact->org_name, $record['org_name']);
123             }
124         }
125         
126         $this->assertTrue($found);
127     }
128     
129     /**
130      * import google contacts
131      */
132     public function testImportGoogleContacts()
133     {
134         $this->_filename = dirname(__FILE__) . '/files/google_contacts.csv';
135         $this->_deleteImportFile = FALSE;
136         
137         $result = $this->_doImport(array('dryrun' => TRUE), 'adb_google_import_csv');
138         
139         $this->assertEquals(5, $result['totalcount']);
140         $this->assertEquals('Niedersachsen Ring 22', $result['results'][4]->adr_one_street);
141         $this->assertEquals('abc@here.de', $result['results'][3]->email);
142         $this->assertEquals('+49227913452', $result['results'][0]->tel_work);
143     }
144     
145     /**
146      * test import of a customfield
147      */
148     public function testImportCustomField()
149     {
150         $this->_createCustomField();
151         
152         // create/get new import/export definition with customfield
153         $filename = dirname(__FILE__) . '/files/adb_google_import_csv_test.xml';
154         $applicationId = Tinebase_Application::getInstance()->getApplicationByName('Addressbook')->getId();
155         $definition = Tinebase_ImportExportDefinition::getInstance()->getFromFile($filename, $applicationId);
156         
157         $this->_filename = dirname(__FILE__) . '/files/google_contacts.csv';
158         $this->_deleteImportFile = FALSE;
159         
160         $result = $this->_doImport(array(), $definition);
161         $this->_deletePersonalContacts = TRUE;
162         $this->assertEquals(5, $result['totalcount']);
163         
164         $contacts = Addressbook_Controller_Contact::getInstance()->search(new Addressbook_Model_ContactFilter(array(
165             array('field' => 'container_id', 'operator' => 'equals', 'value' => Addressbook_Controller_Contact::getInstance()->getDefaultAddressbook()->getId()),
166             array('field' => 'n_given', 'operator' => 'equals', 'value' => 'Ando'),
167         )));
168         $this->assertEquals(1, count($contacts));
169         $ando = $contacts->getFirstRecord();
170         $this->assertEquals(array('Yomi Name' => 'yomi'), $ando->customfields);
171     }
172     
173     /**
174      * testExportAndImportWithCustomField
175      * 
176      * @see 0006230: add customfields to csv export
177      */
178     public function testExportAndImportWithCustomField()
179     {
180         $customField = $this->_createCustomField();
181         $ownContact = Addressbook_Controller_Contact::getInstance()->getContactByUserId(Tinebase_Core::getUser()->getId());
182         $cfValue = array($customField->name => 'testing');
183         $ownContact->customfields = $cfValue;
184         Addressbook_Controller_Contact::getInstance()->update($ownContact);
185         
186         $definition = Tinebase_ImportExportDefinition::getInstance()->getByName('adb_tine_import_csv');
187         $definition->plugin_options = preg_replace('/<\/mapping>/',
188             '<field>
189                 <source>Yomi Name</source>
190                 <destination>Yomi Name</destination>
191             </field></mapping>', $definition->plugin_options);
192         $result = $this->_doImport(array(), $definition, new Addressbook_Model_ContactFilter(array(
193             array('field' => 'id', 'operator' => 'equals', 'value' => $ownContact->getId()),
194         )));
195         $this->assertGreaterThan(0, $result['duplicatecount'], 'no duplicates.');
196         $this->assertTrue($result['exceptions'] instanceof Tinebase_Record_RecordSet);
197
198         $exceptionArray = $result['exceptions']->toArray();
199         $this->assertTrue(isset($exceptionArray[0]['exception']['clientRecord']['customfields']),
200             'could not find customfields in client record: ' . print_r($exceptionArray[0]['exception']['clientRecord'], TRUE));
201         $this->assertEquals('testing', $exceptionArray[0]['exception']['clientRecord']['customfields'][$customField->name],
202             'could not find cf value in client record: ' . print_r($exceptionArray[0]['exception']['clientRecord'], TRUE));
203     }
204     
205     /**
206      * testImportWithUmlautsWin1252
207      * 
208      * @see 0006534: import of contacts with umlaut as first char fails
209      */
210     public function testImportWithUmlautsWin1252()
211     {
212         $definition = $this->_getDefinitionFromFile('adb_import_csv_win1252.xml');
213         
214         $this->_filename = dirname(__FILE__) . '/files/importtest_win1252.csv';
215         $this->_deleteImportFile = FALSE;
216         
217         $result = $this->_doImport(array(), $definition);
218         $this->_deletePersonalContacts = TRUE;
219         
220         $this->assertEquals(4, $result['totalcount']);
221         $this->assertEquals('Üglü, ÖzdemirÖ', $result['results'][2]->n_fileas, 'Umlauts were not imported correctly: ' . print_r($result['results'][2]->toArray(), TRUE));
222     }
223     
224     /**
225      * returns import definition from file
226      * 
227      * @param string $filename
228      * @return Tinebase_Model_ImportExportDefinition
229      */
230     protected function _getDefinitionFromFile($filename, $path = null)
231     {
232         $filename = ($path ? $path : dirname(__FILE__) . '/files/') . $filename;
233         $applicationId = Tinebase_Application::getInstance()->getApplicationByName('Addressbook')->getId();
234         $definition = Tinebase_ImportExportDefinition::getInstance()->getFromFile($filename, $applicationId);
235         
236         return $definition;
237     }
238     
239     /**
240     * get custom field record
241     * 
242     * @param string $name
243     * @return Tinebase_Model_CustomField_Config
244     */
245     protected function _createCustomField($name = 'Yomi Name')
246     {
247         $cfData = new Tinebase_Model_CustomField_Config(array(
248             'application_id'    => Tinebase_Application::getInstance()->getApplicationByName('Addressbook')->getId(),
249             'name'              => $name,
250             'model'             => 'Addressbook_Model_Contact',
251             'definition'        => array(
252                 'label' => Tinebase_Record_Abstract::generateUID(),
253                 'type'  => 'string',
254                 'uiconfig' => array(
255                     'xtype'  => Tinebase_Record_Abstract::generateUID(),
256                     'length' => 10,
257                     'group'  => 'unittest',
258                     'order'  => 100,
259                 )
260             )
261         ));
262         
263         try {
264             $result = Tinebase_CustomField::getInstance()->addCustomField($cfData);
265         } catch (Zend_Db_Statement_Exception $zdse) {
266             // customfield already exists
267             $cfs = Tinebase_CustomField::getInstance()->getCustomFieldsForApplication('Addressbook');
268             $result = $cfs->filter('name', $name)->getFirstRecord();
269         }
270         
271         return $result;
272     }
273
274     /**
275      * testImportDuplicateResolve
276      * 
277      * @see 0009316: add duplicate resolving to cli import
278      */
279     public function testImportDuplicateResolve()
280     {
281         $definition = $this->_getDefinitionFromFile('adb_import_csv_duplicate.xml');
282         
283         $this->_filename = dirname(__FILE__) . '/files/import_duplicate_1.csv';
284         $this->_deleteImportFile = FALSE;
285         
286         $this->_doImport(array(), $definition);
287         $this->_deletePersonalContacts = TRUE;
288
289         $this->_filename = dirname(__FILE__) . '/files/import_duplicate_2.csv';
290         
291         $result = $this->_doImport(array(), $definition);
292         
293         $this->assertEquals(1, $result['updatecount'], 'should have updated 1 contact');
294         $this->assertEquals('joerg@home.com', $result['results'][0]->email_home, 'duplicates resolving did not work: ' . print_r($result['results']->toArray(), true));
295         $this->assertEquals('Jörg', $result['results'][0]->n_given, 'wrong encoding: ' . print_r($result['results']->toArray(), true));
296     }
297
298     /**
299      * testImportLxOffice
300      */
301     public function testImportLxOffice()
302     {
303         $options = array(
304             'container_id'  => Addressbook_Controller_Contact::getInstance()->getDefaultAddressbook()->getId(),
305         );
306         
307         // add duplicate field "customernumber"
308         Addressbook_Controller_Contact::getInstance()->duplicateCheckFields(array(
309             array('email'),
310             array('customernumber')
311         ));
312         
313         $this->_createCustomField('customernumber');
314         
315         $definition = $this->_getDefinitionFromFile('adb_lxoffice_import_csv.xml',
316             dirname(dirname(dirname(dirname(dirname(__FILE__))))) . '/tine20/Addressbook/Import/definitions/');
317         
318         $this->_filename = dirname(__FILE__) . '/files/importtest_lxoffice1.csv';
319         $this->_deleteImportFile = FALSE;
320         
321         $result = $this->_doImport($options, $definition);
322         $this->_deletePersonalContacts = TRUE;
323         
324         $this->assertEquals(3, $result['totalcount'], print_r($result['results']->toArray(), true));
325         
326         $contacts = $result['results'];
327         $berger = $contacts->getFirstRecord();
328         $this->assertEquals(array('customernumber' => '73029'), $berger->customfields, print_r($berger->toArray(), true));
329         
330         $this->_filename = dirname(__FILE__) . '/files/importtest_lxoffice2.csv';
331         
332         $result = $this->_doImport($options, $definition);
333         
334         $this->assertEquals(5, count($result['results']));
335         $this->assertEquals(2, $result['updatecount'], 'should have updated 3 contacts');
336         $this->assertEquals(3, $result['totalcount'], 'should have added 3 contacts');
337         $this->assertEquals('Straßbough', $result['results'][1]['adr_one_locality'],
338                 'should have changed the locality of contact #2: ' . print_r($result['results'][1]->toArray(), true));
339         $this->assertEquals('Gartencenter Röhr & Vater', $result['results'][3]['n_family']);
340
341         // TODO this should be researched, imho the relation should not trigger an update of the record
342 //        $this->assertEquals(1, $result['results'][3]['seq'], 'Wolfer has been updated - relations changed');
343 //        $this->assertEquals('Weixdorf DD', $result['results'][0]['adr_one_locality'], 'locality should persist');
344 //        $this->assertEquals('Gartencenter Röhr & Vater', $result['results'][4]['n_fileas']);
345 //        $this->assertEquals('Straßback', $result['results'][5]['adr_one_locality']);
346     }
347
348     public function testSplitField()
349     {
350         $definition = $this->_getDefinitionFromFile('adb_import_csv_split.xml');
351
352         $this->_filename = dirname(__FILE__) . '/files/import_split.csv';
353         $this->_deleteImportFile = FALSE;
354
355         $result = $this->_doImport(array('dryrun' => true), $definition);
356
357         $this->assertEquals(1, $result['totalcount'], print_r($result, true));
358         $importedRecord = $result['results']->getFirstRecord();
359
360         $this->assertEquals('21222', $importedRecord->adr_one_postalcode, print_r($importedRecord->toArray(), true));
361         $this->assertEquals('Käln', $importedRecord->adr_one_locality, print_r($importedRecord->toArray(), true));
362     }
363
364     /**
365      * @see 0011354: keep both records if duplicates are within current import file
366      */
367     public function testImportDuplicateInImport()
368     {
369         $definition = $this->_getDefinitionFromFile('adb_import_csv_split.xml');
370
371         $this->_filename = dirname(__FILE__) . '/files/import_split_duplicate.csv';
372         $this->_deletePersonalContacts = TRUE;
373         $this->_deleteImportFile = FALSE;
374
375         $result = $this->_doImport(array('dryrun' => false), $definition);
376
377         $this->assertEquals(2, $result['totalcount'], print_r($result, true));
378         $this->assertEquals(2, count(array_unique($result['results']->getArrayOfIds())));
379     }
380 }