0009910: updateFlags is using too much memory
[tine20] / tine20 / Felamimail / Controller / Cache / Message.php
1 <?php
2 /**
3  * Tine 2.0
4  *
5  * @package     Felamimail
6  * @subpackage  Controller
7  * @license     http://www.gnu.org/licenses/agpl.html AGPL Version 3
8  * @author      Philipp Schüle <p.schuele@metaways.de>
9  * @copyright   Copyright (c) 2009-2014 Metaways Infosystems GmbH (http://www.metaways.de)
10  */
11
12 /**
13  * cache controller for Felamimail messages
14  *
15  * @package     Felamimail
16  * @subpackage  Controller
17  */
18 class Felamimail_Controller_Cache_Message extends Felamimail_Controller_Message
19 {
20     /**
21      * number of imported messages in one caching step
22      *
23      * @var integer
24      */
25     protected $_importCountPerStep = 50;
26     
27     /**
28      * number of fetched messages for one step of flag sync
29      *
30      * @var integer
31      */
32     protected $_flagSyncCountPerStep = 500;
33     
34     /**
35      * max size of message to cache body for
36      * 
37      * @var integer
38      */
39     protected $_maxMessageSizeToCacheBody = 2097152;
40     
41     /**
42      * initial cache status (used by updateCache and helper funcs)
43      * 
44      * @var string
45      */
46     protected $_initialCacheStatus = NULL;
47
48     /**
49      * message sequence in cache (used by updateCache and helper funcs)
50      * 
51      * @var integer
52      */
53     protected $_cacheMessageSequence = NULL;
54
55     /**
56      * message sequence on imap server (used by updateCache and helper funcs)
57      * 
58      * @var integer
59      */
60     protected $_imapMessageSequence = NULL;
61
62     /**
63      * start of cache update in seconds+microseconds/unix timestamp (used by updateCache and helper funcs)
64      * 
65      * @var float
66      */
67     protected $_timeStart = NULL;
68     
69     /**
70      * time elapsed in seconds (used by updateCache and helper funcs)
71      * 
72      * @var integer
73      */
74     protected $_timeElapsed = 0;
75
76     /**
77      * time for update in seconds (used by updateCache and helper funcs)
78      * 
79      * @var integer
80      */
81     protected $_availableUpdateTime = 0;
82     
83     /**
84      * holds the instance of the singleton
85      *
86      * @var Felamimail_Controller_Cache_Message
87      */
88     private static $_instance = NULL;
89     
90     /**
91      * the constructor
92      *
93      * don't use the constructor. use the singleton
94      */
95     private function __construct() {
96         $this->_backend = new Felamimail_Backend_Cache_Sql_Message();
97     }
98     
99     /**
100      * don't clone. Use the singleton.
101      *
102      */
103     private function __clone() 
104     {
105     }
106     
107     /**
108      * the singleton pattern
109      *
110      * @return Felamimail_Controller_Cache_Message
111      */
112     public static function getInstance() 
113     {
114         if (self::$_instance === NULL) {
115             self::$_instance = new Felamimail_Controller_Cache_Message();
116         }
117         
118         return self::$_instance;
119     }
120     
121     /**
122     * get folder status and return all folders where something needs to be done
123     *
124     * @param Felamimail_Model_FolderFilter  $_filter
125     * @return Tinebase_Record_RecordSet
126     */
127     public function getFolderStatus(Felamimail_Model_FolderFilter $_filter)
128     {
129         $this->_availableUpdateTime = NULL;
130         
131         // add user account ids to filter and use the folder backend to search as the folder controller has some special handling in its search function
132         $accountIdFilter = $_filter->createFilter(array('field' => 'account_id', 'operator' => 'in', 'value' => Felamimail_Controller_Account::getInstance()->search()->getArrayOfIds()));
133         $_filter->addFilter($accountIdFilter);
134         $folderBackend = new Felamimail_Backend_Folder();
135         $folders = $folderBackend->search($_filter);
136         
137         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ .  ' ' . print_r($_filter->toArray(), TRUE));
138         if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ .  " Checking status of " . count($folders) . ' folders.');
139         
140         $result = new Tinebase_Record_RecordSet('Felamimail_Model_Folder');
141         foreach ($folders as $folder) {
142             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  ' Checking folder ' . $folder->globalname);
143             
144             if ($this->_doNotUpdateCache($folder, FALSE)) {
145                 continue;
146             }
147             
148             $imap = Felamimail_Backend_ImapFactory::factory($folder->account_id);
149             
150             try {
151                 $folder = Felamimail_Controller_Cache_Folder::getInstance()->getIMAPFolderCounter($folder);
152             } catch (Zend_Mail_Storage_Exception $zmse) {
153                 if (Tinebase_Core::isLogLevel(Zend_Log::NOTICE)) Tinebase_Core::getLogger()->notice(__METHOD__ . '::' . __LINE__ .  ' ' . $zmse->getMessage());
154                 if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ 
155                     . ' Removing folder and contained messages from cache.');
156                 Felamimail_Controller_Message::getInstance()->deleteByFolder($folder);
157                 Felamimail_Controller_Cache_Folder::getInstance()->delete($folder->getId());
158                 continue;
159             }
160             
161             if ($this->_cacheIsInvalid($folder) || $this->_messagesInCacheButNotOnIMAP($folder)) {
162                 $result->addRecord($folder);
163                 continue;
164             }
165             
166             if ($folder->imap_totalcount > 0) {
167                 try {
168                     $this->_updateMessageSequence($folder, $imap);
169                 } catch (Felamimail_Exception_IMAPMessageNotFound $feimnf) {
170                     $result->addRecord($folder);
171                     continue;
172                 }
173                 
174                 if ($this->_messagesDeletedOnIMAP($folder) || $this->_messagesToBeAddedToCache($folder) || $this->_messagesMissingFromCache($folder) ) {
175                     $result->addRecord($folder);
176                     continue;
177                 }
178             }
179         }
180
181         if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ .  " Found " . count($result) . ' folders that need an update.');
182         
183         return $result;
184     }
185     
186     /**
187      * returns true on uidvalidity mismatch
188      * 
189      * @param Felamimail_Model_Folder $_folder
190      * @return boolean
191      * 
192      * @todo remove int casting when http://forge.tine20.org/mantisbt/view.php?id=5764 is resolved
193      */
194     protected function _cacheIsInvalid($_folder)
195     {
196         return (isset($_folder->cache_uidvalidity) && (int) $_folder->imap_uidvalidity !== (int) $_folder->cache_uidvalidity);
197     }
198     
199     /**
200      * returns true if there are messages in cache but not in folder on IMAP
201      * 
202      * @param Felamimail_Model_Folder $_folder
203      * @return boolean
204      */
205     protected function _messagesInCacheButNotOnIMAP($_folder)
206     {
207         return ($_folder->imap_totalcount == 0 && $_folder->cache_totalcount > 0);
208     }
209     
210     /**
211      * returns true if there are messages deleted on IMAP but not in cache
212      * 
213      * @param Felamimail_Model_Folder $_folder
214      * @return boolean
215      */
216     protected function _messagesDeletedOnIMAP($_folder)
217     {
218         return ($_folder->imap_totalcount > 0 && $this->_cacheMessageSequence > $this->_imapMessageSequence);
219     }
220     
221     /**
222      * returns true if there are new messages on IMAP
223      * 
224      * @param Felamimail_Model_Folder $_folder
225      * @return boolean
226      */
227     protected function _messagesToBeAddedToCache($_folder)
228     {
229         return ($_folder->imap_totalcount > 0 && $this->_imapMessageSequence < $_folder->imap_totalcount);
230     }
231     
232     /**
233      * returns true if there are messages on IMAP that are missing from the cache
234      * 
235      * @param Felamimail_Model_Folder $_folder
236      * @return boolean
237      */
238     protected function _messagesMissingFromCache($_folder)
239     {
240         return ($_folder->imap_totalcount > 0 && $_folder->cache_totalcount < $_folder->imap_totalcount);
241     }
242     
243     /**
244      * update message cache
245      * 
246      * @param string $_folder
247      * @param integer $_time in seconds
248      * @param integer $_updateFlagFactor 1 = update flags every time, x = update flags roughly each xth run (10 by default)
249      * @return Felamimail_Model_Folder folder status (in cache)
250      * @throws Felamimail_Exception
251      */
252     public function updateCache($_folder, $_time = 10, $_updateFlagFactor = 10)
253     {
254         $oldMaxExcecutionTime = Tinebase_Core::setExecutionLifeTime(300); // 5 minutes
255         
256         // always read folder from database
257         $folder = Felamimail_Controller_Folder::getInstance()->get($_folder);
258         
259         if ($this->_doNotUpdateCache($folder)) {
260             return $folder;
261         }
262         
263         $imap = Felamimail_Backend_ImapFactory::factory($folder->account_id);
264         
265         $this->_availableUpdateTime = $_time;
266        
267         try { 
268             $this->_expungeCacheFolder($folder, $imap);
269         } catch (Felamimail_Exception_IMAPFolderNotFound $feifnf) {
270             return $folder;
271         }
272         
273         $this->_initUpdate($folder);
274         $this->_updateMessageSequence($folder, $imap);
275         $this->_deleteMessagesInCache($folder, $imap);
276         $this->_addMessagesToCache($folder, $imap);
277         $this->_checkForMissingMessages($folder, $imap);
278         $this->_updateFolderStatus($folder);
279         
280         if ($folder->supports_condstore || rand(1, $_updateFlagFactor) == 1) {
281             $folder = $this->updateFlags($folder);
282         }
283         
284         $this->_updateFolderQuota($folder, $imap);
285         
286         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
287             . ' Folder status of ' . $folder->globalname . ' after updateCache(): ' . $folder->cache_status);
288         
289         // reset max execution time to old value
290         Tinebase_Core::setExecutionLifeTime($oldMaxExcecutionTime);
291         
292         return $folder;
293     }
294     
295     /**
296      * checks if cache update should not commence / fencing
297      * 
298      * @param Felamimail_Model_Folder $_folder
299      * @param boolean $_lockFolder
300      * @return boolean
301      */
302     protected function _doNotUpdateCache(Felamimail_Model_Folder $_folder, $_lockFolder = TRUE)
303     {
304         if ($_folder->is_selectable == false) {
305             // nothing to be done
306             return FALSE;
307         }
308         
309         if (Felamimail_Controller_Cache_Folder::getInstance()->updateAllowed($_folder, $_lockFolder) !== true) {
310             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " update of folder {$_folder->globalname} currently not allowed. do nothing!");
311             return FALSE;
312         }
313     }
314     
315     /**
316      * expunge cache folder
317      * 
318      * @param Felamimail_Model_Folder $_folder
319      * @param Felamimail_Backend_ImapProxy $_imap
320      * @throws Felamimail_Exception_IMAPFolderNotFound
321      */
322     protected function _expungeCacheFolder(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
323     {
324         try {
325             $_imap->expunge(Felamimail_Model_Folder::encodeFolderName($_folder->globalname));
326         } catch (Zend_Mail_Storage_Exception $zmse) {
327             Tinebase_Exception::log($zmse);
328             
329             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
330                 . ' Marking folder as not selectable: ' . print_r($_folder->toArray(), true));
331             
332             // mark folder as not selectable + finish cache update
333             $_folder->is_selectable = 0;
334             $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_COMPLETE;
335             $_folder = Felamimail_Controller_Folder::getInstance()->update($_folder);
336             
337             // @todo check if folder is really deleted?
338             //Felamimail_Controller_Cache_Folder::getInstance()->delete($_folder->getId());
339             
340             throw new Felamimail_Exception_IMAPFolderNotFound('Folder not found / is not selectable: ' . $_folder->globalname);
341         }
342     }
343     
344     /**
345      * init cache update process
346      * 
347      * @param Felamimail_Model_Folder $_folder
348      * @return void
349      */
350     protected function _initUpdate(Felamimail_Model_Folder $_folder)
351     {
352         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " status of folder {$_folder->globalname}: {$_folder->cache_status}");
353         
354         $this->_initialCacheStatus = $_folder->cache_status;
355         
356         // reset cache counter when transitioning from Felamimail_Model_Folder::CACHE_STATUS_COMPLETE or 
357         if ($_folder->cache_status == Felamimail_Model_Folder::CACHE_STATUS_COMPLETE || $_folder->cache_status == Felamimail_Model_Folder::CACHE_STATUS_EMPTY) {
358             $_folder->cache_job_actions_est = 0;
359             $_folder->cache_job_actions_done     = 0;
360             $_folder->cache_job_startuid         = 0;
361         }
362         
363         $_folder = Felamimail_Controller_Cache_Folder::getInstance()->getIMAPFolderCounter($_folder);
364         
365         if ($this->_cacheIsInvalid($_folder)) {
366             if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' uidvalidity changed => clear cache: ' . print_r($_folder->toArray(), TRUE));
367             $_folder = $this->clear($_folder);
368         }
369         
370         if ($this->_messagesInCacheButNotOnIMAP($_folder)) {
371             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " folder is empty on imap server => clear cache of folder {$_folder->globalname}");
372             $_folder = $this->clear($_folder);
373         }
374         
375         $_folder->cache_status    = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
376         $_folder->cache_timestamp = Tinebase_DateTime::now();
377         
378         $this->_timeStart = microtime(true);
379     }
380     
381     /**
382      * at which sequence is the message with the highest messageUid (cache + imap)?
383      * 
384      * @param Felamimail_Model_Folder $_folder
385      * @param Felamimail_Backend_ImapProxy $_imap
386      * @param boolean $_updateFolder
387      * @throws Felamimail_Exception
388      * @throws Felamimail_Exception_IMAPMessageNotFound
389      */
390     protected function _updateMessageSequence(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap, $_updateFolder = TRUE)
391     {
392         if ($_folder->imap_totalcount > 0) {
393             $transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
394         
395             $lastFailedUid   = null;
396             $messageSequence = null;
397             $decrementMessagesCounter = 0;
398             $decrementUnreadCounter   = 0;
399             
400             while ($messageSequence === null) {
401                 $latestMessageUidArray = $this->_getLatestMessageUid($_folder);
402                 
403                 if (is_array($latestMessageUidArray)) {
404                     $latestMessageId  = key($latestMessageUidArray);
405                     $latestMessageUid = current($latestMessageUidArray);
406                     
407                     if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " $latestMessageId  $latestMessageUid");
408                     
409                     if ($latestMessageUid === $lastFailedUid) {
410                         throw new Felamimail_Exception('Failed to delete invalid messageuid from cache');
411                     }
412                     
413                     if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " Check messageUid {$latestMessageUid} in folder " . $_folder->globalname);
414                     
415                     try {
416                         $this->_imapMessageSequence  = $_imap->resolveMessageUid($latestMessageUid);
417                         $this->_cacheMessageSequence = $_folder->cache_totalcount;
418                         $messageSequence             = $this->_imapMessageSequence + 1;
419                     } catch (Zend_Mail_Protocol_Exception $zmpe) {
420                         if (! $_updateFolder) {
421                             throw new Felamimail_Exception_IMAPMessageNotFound('Message not found on IMAP');
422                         }
423                         
424                         // message does not exist on imap server anymore, remove from local cache
425                         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . " messageUid {$latestMessageUid} not found => remove from cache");
426                         
427                         $lastFailedUid = $latestMessageUid;
428                         
429                         $latestMessage = $this->_backend->get($latestMessageId);
430                         $this->_backend->delete($latestMessage);
431                         
432                         $decrementMessagesCounter++;
433                         if (! $latestMessage->hasSeenFlag()) {
434                             $decrementUnreadCounter++;
435                         }
436                     }
437                 } else {
438                     $this->_imapMessageSequence = 0;
439                     $messageSequence = 1;
440                 }
441                 
442                 if (! $this->_timeLeft()) {
443                     $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_INCOMPLETE;
444                     break;
445                 }
446             }
447             
448             Tinebase_TransactionManager::getInstance()->commitTransaction($transactionId);
449             
450             if ($decrementMessagesCounter > 0 || $decrementUnreadCounter > 0) {
451                 Felamimail_Controller_Folder::getInstance()->updateFolderCounter($_folder, array(
452                     'cache_totalcount'  => "-$decrementMessagesCounter",
453                     'cache_unreadcount' => "-$decrementUnreadCounter",
454                 ));
455             }
456         }
457         
458         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
459             . " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount} cache sequence: $this->_cacheMessageSequence imap sequence: $this->_imapMessageSequence");
460     }
461     
462     /**
463      * get message with highest messageUid from cache 
464      * 
465      * @param  mixed  $_folderId
466      * @return Felamimail_Model_Message
467      */
468     protected function _getLatestMessageUid($_folderId) 
469     {
470         $folderId = ($_folderId instanceof Felamimail_Model_Folder) ? $_folderId->getId() : $_folderId;
471         
472         $filter = new Felamimail_Model_MessageFilter(array(
473             array(
474                 'field'    => 'folder_id', 
475                 'operator' => 'equals', 
476                 'value'    => $folderId
477             )
478         ));
479         $pagination = new Tinebase_Model_Pagination(array(
480             'limit' => 1,
481             'sort'  => 'messageuid',
482             'dir'   => 'DESC'
483         ));
484         
485         $result = $this->_backend->searchMessageUids($filter, $pagination);
486         
487         if (count($result) === 0) {
488             return null;
489         }
490         
491         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' Got last message uid: ' . print_r($result, TRUE));
492         
493         return $result;
494     }
495     
496     /**
497      * do we have time left for update (updates elapsed time)?
498      * 
499      * @return boolean
500      */
501     protected function _timeLeft()
502     {
503         if ($this->_availableUpdateTime === NULL) {
504             // "infinite" time
505             return TRUE;
506         }
507         
508         $this->_timeElapsed = round(((microtime(true)) - $this->_timeStart));
509         return ($this->_timeElapsed < $this->_availableUpdateTime);
510     }
511     
512     /**
513      * delete messages in cache
514      * 
515      *   - if the latest message on the cache has a different sequence number then on the imap server
516      *     then some messages before the latest message(from the cache) got deleted
517      *     we need to remove them from local cache first
518      *     
519      *   - $folder->cache_totalcount equals to the message sequence of the last message in the cache
520      * 
521      * @param Felamimail_Model_Folder $_folder
522      * @param Felamimail_Backend_ImapProxy $_imap
523      */
524     protected function _deleteMessagesInCache(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
525     {
526         if ($this->_messagesDeletedOnIMAP($_folder)) {
527
528             $messagesToRemoveFromCache = $this->_cacheMessageSequence - $this->_imapMessageSequence;
529             
530             if ($this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_COMPLETE || $this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_EMPTY) {
531                 $_folder->cache_job_actions_est += $messagesToRemoveFromCache;
532             }        
533             
534             $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_INCOMPLETE;
535             
536             if ($this->_timeElapsed < $this->_availableUpdateTime) {
537             
538                 $begin = $_folder->cache_job_startuid > 0 ? $_folder->cache_job_startuid : $_folder->cache_totalcount;
539                 
540                 $firstMessageSequence = 0;
541                  
542                 if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " $messagesToRemoveFromCache message to remove from cache. starting at $begin");
543                 
544                 for ($i=$begin; $i > 0; $i -= $this->_importCountPerStep) {
545                     $firstMessageSequence = ($i-$this->_importCountPerStep) >= 0 ? $i-$this->_importCountPerStep : 0;
546                     if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " fetching from $firstMessageSequence");
547                     $cachedMessageUids = $this->_getCachedMessageUidsChunked($_folder, $firstMessageSequence);
548
549                     // $cachedMessageUids can be empty if we fetch the last chunk
550                     if (count($cachedMessageUids) > 0) {
551                         $messageUidsOnImapServer = $_imap->messageUidExists($cachedMessageUids);
552                         
553                         $difference = array_diff($cachedMessageUids, $messageUidsOnImapServer);
554                         $removedMessages = $this->_deleteMessagesByIdAndUpdateCounters(array_keys($difference), $_folder);
555                         $messagesToRemoveFromCache -= $removedMessages;
556                         
557                         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
558                             . " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount} messages to remove: $messagesToRemoveFromCache");
559                         
560                         if ($messagesToRemoveFromCache <= 0) {
561                             $_folder->cache_job_startuid = 0;
562                             $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
563                             break;
564                         }
565                     }
566                     
567                     if (! $this->_timeLeft()) {
568                         $_folder->cache_job_startuid = $i;
569                         break;
570                     }
571                 }
572                 
573                 if ($firstMessageSequence === 0) {
574                     $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
575                 }
576             }
577         }
578         
579         $this->_cacheMessageSequence = $_folder->cache_totalcount;
580         
581         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount} cache sequence: $this->_cacheMessageSequence imap sequence: $this->_imapMessageSequence");
582                 
583         Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Folder cache status: ' . $_folder->cache_status);
584         Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Folder cache actions to be done yet: ' . ($_folder->cache_job_actions_est - $_folder->cache_job_actions_done));
585     }
586     
587     /**
588      * delete messages from cache
589      * 
590      * @param array $_ids
591      * @param Felamimail_Model_Folder $_folder
592      * @return integer number of removed messages
593      */
594     protected function _deleteMessagesByIdAndUpdateCounters($_ids, Felamimail_Model_Folder $_folder)
595     {
596         if (count($_ids) == 0) {
597             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' No messages to delete.');
598             return 0;
599         }
600         
601         $decrementMessagesCounter = 0;
602         $decrementUnreadCounter   = 0;
603         
604         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  
605             ' Delete ' . count($_ids) . ' messages'
606         );
607         
608         $messagesToBeDeleted = $this->_backend->getMultiple($_ids);
609         
610         $transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
611         
612         foreach ($messagesToBeDeleted as $messageToBeDeleted) {
613             $this->_backend->delete($messageToBeDeleted);
614             
615             $_folder->cache_job_actions_done++;
616             $decrementMessagesCounter++;
617             if (! $messageToBeDeleted->hasSeenFlag()) {
618                 $decrementUnreadCounter++;
619             }
620         }
621         
622         Tinebase_TransactionManager::getInstance()->commitTransaction($transactionId);
623         
624         $_folder = Felamimail_Controller_Folder::getInstance()->updateFolderCounter($_folder, array(
625             'cache_totalcount'  => "-$decrementMessagesCounter",
626             'cache_unreadcount' => "-$decrementUnreadCounter",
627         ));
628         
629         return $decrementMessagesCounter;
630     }
631     
632     /**
633      * get message with highest messageUid from cache 
634      * 
635      * @param  mixed  $_folderId
636      * @return array
637      */
638     protected function _getCachedMessageUidsChunked($_folderId, $_firstMessageSequnce) 
639     {
640         $folderId = ($_folderId instanceof Felamimail_Model_Folder) ? $_folderId->getId() : $_folderId;
641         
642         $filter = new Felamimail_Model_MessageFilter(array(
643             array(
644                 'field'    => 'folder_id', 
645                 'operator' => 'equals', 
646                 'value'    => $folderId
647             )
648         ));
649         $pagination = new Tinebase_Model_Pagination(array(
650             'start' => $_firstMessageSequnce,
651             'limit' => $this->_importCountPerStep,
652             'sort'  => 'messageuid',
653             'dir'   => 'ASC'
654         ));
655         
656         $result = $this->_backend->searchMessageUids($filter, $pagination);
657         
658         return $result;
659     }
660     
661     /**
662      * add new messages to cache
663      * 
664      * @param Felamimail_Model_Folder $_folder
665      * @param Felamimail_Backend_ImapProxy $_imap
666      * 
667      * @todo split into smaller parts
668      */
669     protected function _addMessagesToCache(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
670     {
671         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
672             .  " cache sequence: {$this->_imapMessageSequence} / imap count: {$_folder->imap_totalcount}");
673     
674         if ($this->_messagesToBeAddedToCache($_folder)) {
675             
676             if ($this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_COMPLETE || $this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_EMPTY) {
677                 $_folder->cache_job_actions_est += ($_folder->imap_totalcount - $this->_imapMessageSequence);
678             }
679             
680             $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_INCOMPLETE;
681             
682             if ($this->_fetchAndAddMessages($_folder, $_imap)) {
683                 $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
684             }
685         }
686         
687         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
688             . " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount} cache sequence: $this->_cacheMessageSequence imap sequence: $this->_imapMessageSequence");
689     }
690     
691     /**
692      * fetch messages from imap server and add them to cache until timelimit is reached or all messages have been fetched
693      * 
694      * @param Felamimail_Model_Folder $_folder
695      * @param Felamimail_Backend_ImapProxy $_imap
696      * @return boolean finished
697      */
698     protected function _fetchAndAddMessages(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
699     {
700         $messageSequenceStart = $this->_imapMessageSequence + 1;
701         
702         // add new messages
703         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
704             . " Retrieve message from $messageSequenceStart to {$_folder->imap_totalcount}");
705         
706         while ($messageSequenceStart <= $_folder->imap_totalcount) {
707             if (! $this->_timeLeft()) {
708                 return FALSE;
709             }
710             
711             $messageSequenceEnd = (($_folder->imap_totalcount - $messageSequenceStart) > $this->_importCountPerStep ) 
712                 ? $messageSequenceStart+$this->_importCountPerStep : $_folder->imap_totalcount;
713             
714             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
715                 .  " Fetch message from $messageSequenceStart to $messageSequenceEnd $this->_timeElapsed / $this->_availableUpdateTime");
716             
717             try {
718                 $messages = $_imap->getSummary($messageSequenceStart, $messageSequenceEnd, false);
719             } catch (Zend_Mail_Protocol_Exception $zmpe) {
720                 // imap server might have gone away during update
721                 Tinebase_Core::getLogger()->notice(__METHOD__ . '::' . __LINE__ 
722                     . ' IMAP protocol error during message fetching: ' . $zmpe->getMessage());
723                 return FALSE;
724             }
725
726             $this->_addMessagesToCacheAndIncreaseCounters($messages, $_folder);
727             
728             $messageSequenceStart = $messageSequenceEnd + 1;
729             
730             Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Folder cache status: ' . $_folder->cache_status);
731         }
732         
733         return ($messageSequenceEnd == $_folder->imap_totalcount);
734     }
735     
736     /**
737      * add imap messages to cache and increase counters
738      * 
739      * @param array $_messages
740      * @param Felamimail_Model_Folder $_folder
741      * @return Felamimail_Model_Folder
742      */
743     protected function _addMessagesToCacheAndIncreaseCounters($_messages, $_folder)
744     {
745         $incrementMessagesCounter = 0;
746         $incrementUnreadCounter   = 0;
747         
748         $transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
749         
750         foreach ($_messages as $uid => $message) {
751             if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__
752                 .  " Add message $uid to cache");
753             if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__
754                 .  ' ' . print_r($message, TRUE));
755             $addedMessage = $this->addMessage($message, $_folder, false);
756             
757             if ($addedMessage) {
758                 $_folder->cache_job_actions_done++;
759                 $incrementMessagesCounter++;
760                 if (! $addedMessage->hasSeenFlag()) {
761                     $incrementUnreadCounter++;
762                 }
763             }
764         }
765         
766         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . " Added $incrementMessagesCounter ($incrementUnreadCounter) new (unread) messages to cache.");
767         
768         Tinebase_TransactionManager::getInstance()->commitTransaction($transactionId);
769         
770         $_folder = Felamimail_Controller_Folder::getInstance()->updateFolderCounter($_folder, array(
771             'cache_totalcount'  => "+$incrementMessagesCounter",
772             'cache_unreadcount' => "+$incrementUnreadCounter",
773         ));
774     }
775     
776     /**
777      * maybe there are some messages missing before $this->_imapMessageSequence
778      * 
779      * @param Felamimail_Model_Folder $_folder
780      * @param Felamimail_Backend_ImapProxy $_imap
781      */
782     protected function _checkForMissingMessages(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
783     {
784         if ($this->_messagesMissingFromCache($_folder)) {
785             
786             if ($this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_COMPLETE || $this->_initialCacheStatus == Felamimail_Model_Folder::CACHE_STATUS_EMPTY) {
787                 $_folder->cache_job_actions_est += ($_folder->imap_totalcount - $_folder->cache_totalcount);
788             }
789             
790             $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_INCOMPLETE;
791             
792             if ($this->_timeLeft()) {
793                 // add missing messages
794                 if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " Retrieve message from {$_folder->imap_totalcount} to 1");
795                 
796                 $begin = $_folder->cache_job_lowestuid > 0 ? $_folder->cache_job_lowestuid : $this->_imapMessageSequence;
797                 
798                 for ($i = $begin; $i > 0; $i -= $this->_importCountPerStep) {
799                     
800                     $messageSequenceStart = (($i - $this->_importCountPerStep) > 0 ) ? $i - $this->_importCountPerStep : 1;
801                     
802                     if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .  " Fetch message from $messageSequenceStart to $i $this->_timeElapsed / $this->_availableUpdateTime");
803                     
804                     $messageUidsOnImapServer = $_imap->resolveMessageSequence($messageSequenceStart, $i);
805                     
806                     $missingUids = $this->_getMissingMessageUids($_folder, $messageUidsOnImapServer);
807                     
808                     if (count($missingUids) != 0) {
809                         $messages = $_imap->getSummary($missingUids);
810                         $this->_addMessagesToCacheAndIncreaseCounters($messages, $_folder);
811                     }
812                     
813                     if ($_folder->cache_totalcount == $_folder->imap_totalcount || $messageSequenceStart == 1) {
814                         $_folder->cache_job_lowestuid = 0;
815                         $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
816                         break;
817                     }
818                     
819                     if (! $this->_timeLeft()) {
820                         $_folder->cache_job_lowestuid = $messageSequenceStart;
821                         break;
822                     }
823                     Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Folder cache status: ' . $_folder->cache_status);
824                 }
825                 
826                 if (defined('messageSequenceStart') && $messageSequenceStart === 1) {
827                     $_folder->cache_status = Felamimail_Model_Folder::CACHE_STATUS_UPDATING;
828                 }
829             }
830         }
831         
832         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount} cache sequence: $this->_cacheMessageSequence imap sequence: $this->_imapMessageSequence");
833                 
834         Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Folder cache status: ' . $_folder->cache_status);
835     }
836     
837     /**
838      * add one message to cache
839      * 
840      * @param  array                    $_message
841      * @param  Felamimail_Model_Folder  $_folder
842      * @param  bool                     $_updateFolderCounter
843      * @return Felamimail_Model_Message|bool
844      */
845     public function addMessage(array $_message, Felamimail_Model_Folder $_folder, $_updateFolderCounter = true)
846     {
847         if (! (isset($_message['header']) || array_key_exists('header', $_message)) || ! is_array($_message['header'])) {
848             if (Tinebase_Core::isLogLevel(Zend_Log::NOTICE)) Tinebase_Core::getLogger()->notice(__METHOD__ . '::' . __LINE__ . ' Email uid ' . $_message['uid'] . ' has no headers. Skipping ...');
849             return FALSE;
850         }
851         
852         $messageToCache = $this->_createMessageToCache($_message, $_folder);
853         $cachedMessage = $this->_addMessageToCache($messageToCache);
854         
855         if ($cachedMessage !== FALSE) {
856             $this->_saveMessageInTinebaseCache($cachedMessage, $_folder, $_message);
857             
858             if ($_updateFolderCounter == TRUE) {
859                 Felamimail_Controller_Folder::getInstance()->updateFolderCounter($_folder, array(
860                     'cache_totalcount'  => '+1',
861                     'cache_unreadcount' => (! $messageToCache->hasSeenFlag())   ? '+1' : '+0',
862                 ));
863             }
864         }
865         
866         return $cachedMessage;
867     }
868     
869     /**
870      * create new message for the cache
871      * 
872      * @param array $_message
873      * @param Felamimail_Model_Folder $_folder
874      * @return Felamimail_Model_Message
875      */
876     protected function _createMessageToCache(array $_message, Felamimail_Model_Folder $_folder)
877     {
878         $message = new Felamimail_Model_Message(array(
879             'account_id'    => $_folder->account_id,
880             'messageuid'    => $_message['uid'],
881             'folder_id'     => $_folder->getId(),
882             'timestamp'     => Tinebase_DateTime::now(),
883             'received'      => Felamimail_Message::convertDate($_message['received']),
884             'size'          => $_message['size'],
885             'flags'         => $_message['flags'],
886         ));
887
888         $message->parseStructure($_message['structure']);
889         $message->parseHeaders($_message['header']);
890         $message->parseBodyParts();
891         
892         $attachments = $this->getAttachments($message);
893         $message->has_attachment = (count($attachments) > 0) ? true : false;
894         
895         return $message;
896     }
897
898     /**
899      * add message to cache backend
900      * 
901      * @param Felamimail_Model_Message $_message
902      * @return Felamimail_Model_Message|bool
903      */
904     protected function _addMessageToCache(Felamimail_Model_Message $_message)
905     {
906         $_message->from_email = substr($_message->from_email, 0, 254);\r
907         $_message->from_name  = substr($_message->from_name,  0, 254);
908         
909         try {
910             $result = $this->_backend->create($_message);
911         } catch (Zend_Db_Statement_Exception $zdse) {
912             if (Tinebase_Core::isLogLevel(Zend_Log::NOTICE)) Tinebase_Core::getLogger()->notice(
913                 __METHOD__ . '::' . __LINE__ . ' failed to add message to cache: ' . $zdse->getMessage());
914
915             $result = FALSE;
916         }
917         
918         return $result;
919     }
920     
921     /**
922      * save message in tinebase cache
923      * - only cache message headers if received during the last day
924      * 
925      * @param Felamimail_Model_Message $_message
926      * @param Felamimail_Model_Folder $_folder
927      * @param array $_messageData
928      * 
929      * @todo do we need the headers in the Tinebase cache?
930      */
931     protected function _saveMessageInTinebaseCache(Felamimail_Model_Message $_message, Felamimail_Model_Folder $_folder, $_messageData)
932     {
933         if (! $_message->received->isLater(Tinebase_DateTime::now()->subDay(3))) {
934             return;
935         }
936         
937         $memory = (function_exists('memory_get_peak_usage')) ? memory_get_peak_usage(true) : memory_get_usage(true);
938         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
939             . ' caching message ' . $_message->getId() . ' / memory usage: ' . $memory/1024/1024 . ' MBytes');
940         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' ' . print_r($_message->toArray(), TRUE));
941         
942         $cacheId = 'getMessageHeaders' . $_message->getId();
943         Tinebase_Core::getCache()->save($_messageData['header'], $cacheId, array('getMessageHeaders'));
944     
945         // prefetch body to cache
946         if (Felamimail_Config::getInstance()->get(Felamimail_Config::CACHE_EMAIL_BODY, TRUE) && $_message->size < $this->_maxMessageSizeToCacheBody) {
947             $account = Felamimail_Controller_Account::getInstance()->get($_folder->account_id);
948             $mimeType = ($account->display_format == Felamimail_Model_Account::DISPLAY_HTML || $account->display_format == Felamimail_Model_Account::DISPLAY_CONTENT_TYPE)
949                 ? Zend_Mime::TYPE_HTML
950                 : Zend_Mime::TYPE_TEXT;
951             Felamimail_Controller_Message::getInstance()->getMessageBody($_message, null, $mimeType, $account);
952         }
953     }
954     
955     /**
956      * update folder status and counters
957      * 
958      * @param Felamimail_Model_Folder $_folder
959      */
960     protected function _updateFolderStatus(Felamimail_Model_Folder $_folder)
961     {
962         if ($_folder->cache_status == Felamimail_Model_Folder::CACHE_STATUS_UPDATING) {
963             $_folder->cache_status               = Felamimail_Model_Folder::CACHE_STATUS_COMPLETE;
964             $_folder->cache_job_actions_est      = 0;
965             $_folder->cache_job_actions_done     = 0;
966             $_folder->cache_job_lowestuid        = 0;
967             $_folder->cache_job_startuid         = 0;
968         }
969         
970         if ($_folder->cache_status == Felamimail_Model_Folder::CACHE_STATUS_COMPLETE) {
971             $this->_checkAndUpdateFolderCounts($_folder);
972         }
973         
974         $_folder = Felamimail_Controller_Folder::getInstance()->update($_folder);
975         
976         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' Folder values after import of all messages: ' . print_r($_folder->toArray(), TRUE));
977     }
978     
979     /**
980      * check and update mismatching folder counts (totalcount + unreadcount)
981      * 
982      * @param Felamimail_Model_Folder $_folder
983      */
984     protected function _checkAndUpdateFolderCounts(Felamimail_Model_Folder $_folder)
985     {
986         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Checking foldercounts.');
987         
988         $updatedCounters = Felamimail_Controller_Cache_Folder::getInstance()->getCacheFolderCounter($_folder);
989         
990         if ($this->_countMismatch($_folder, $updatedCounters)) {
991             if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ .
992                 ' something went wrong while in/decrementing counters => recalculate cache counters by counting rows in database.' .
993                 " Cache status cache total count: {$_folder->cache_totalcount} imap total count: {$_folder->imap_totalcount}");
994                         
995             Felamimail_Controller_Folder::getInstance()->updateFolderCounter($_folder, $updatedCounters);
996         }
997         
998         if ($updatedCounters['cache_totalcount'] != $_folder->imap_totalcount) {
999             if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ 
1000                 . ' There are still messages missing in the cache: setting status to INCOMPLETE');
1001             
1002             $_folder->cache_status == Felamimail_Model_Folder::CACHE_STATUS_INCOMPLETE;
1003         }
1004     }
1005     
1006     /**
1007      * returns true if one if these counts mismatch: 
1008      *     - imap_totalcount/cache_totalcount
1009      *  - $_updatedCounters_totalcount/cache_totalcount
1010      *  - $_updatedCounters_unreadcount/cache_unreadcount
1011      * 
1012      * @param Felamimail_Model_Folder $_folder
1013      * @param array $_updatedCounters
1014      * @return boolean
1015      */
1016     protected function _countMismatch($_folder, $_updatedCounters)
1017     {
1018         return ($_folder->cache_totalcount != $_folder->imap_totalcount
1019             || $_updatedCounters['cache_totalcount'] != $_folder->cache_totalcount 
1020             || $_updatedCounters['cache_unreadcount'] != $_folder->cache_unreadcount
1021         );
1022     }
1023     
1024     /**
1025      * get uids missing from cache
1026      * 
1027      * @param  mixed  $_folderId
1028      * @param  array $_messageUids
1029      * @return array
1030      */
1031     protected function _getMissingMessageUids($_folderId, array $_messageUids) 
1032     {
1033         $folderId = ($_folderId instanceof Felamimail_Model_Folder) ? $_folderId->getId() : $_folderId;
1034         
1035         $filter = new Felamimail_Model_MessageFilter(array(
1036             array(
1037                 'field'    => 'folder_id', 
1038                 'operator' => 'equals', 
1039                 'value'    => $folderId
1040             ),
1041             array(
1042                 'field'    => 'messageuid', 
1043                 'operator' => 'in', 
1044                 'value'    => $_messageUids
1045             )
1046         ));
1047         
1048         $messageUidsInCache = $this->_backend->search($filter, NULL, array('messageuid'));
1049         
1050         $result = array_diff($_messageUids, array_keys($messageUidsInCache));
1051         
1052         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' ' . print_r($result, TRUE));
1053         
1054         return $result;
1055     }
1056     
1057     /**
1058      * remove all cached messages for this folder and reset folder values / folder status is updated in the database
1059      *
1060      * @param string|Felamimail_Model_Folder $_folder
1061      * @return Felamimail_Model_Folder
1062      */
1063     public function clear($_folder)
1064     {
1065         $folder = ($_folder instanceof Felamimail_Model_Folder) ? $_folder : Felamimail_Controller_Folder::getInstance()->get($_folder);
1066         
1067         Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__ . ' Clearing cache of ' . $folder->globalname);
1068         
1069         $this->deleteByFolder($folder);
1070         
1071         $folder->cache_timestamp        = Tinebase_DateTime::now();
1072         $folder->cache_status           = Felamimail_Model_Folder::CACHE_STATUS_EMPTY;
1073         $folder->cache_job_actions_est = 0;
1074         $folder->cache_job_actions_done = 0;
1075         
1076         Felamimail_Controller_Folder::getInstance()->updateFolderCounter($folder, array(
1077             'cache_totalcount'  => 0,
1078             'cache_recentcount' => 0,
1079             'cache_unreadcount' => 0
1080         ));
1081         
1082         $folder = Felamimail_Controller_Folder::getInstance()->update($folder);
1083         
1084         return $folder;
1085     }
1086     
1087     /**
1088      * update/synchronize flags
1089      * 
1090      * @param string|Felamimail_Model_Folder $_folder
1091      * @param integer $_time
1092      * @return Felamimail_Model_Folder
1093      * 
1094      * @todo only get flags of current batch of messages from imap?
1095      * @todo add status/progress to start at later messages when this is called next time?
1096      */
1097     public function updateFlags($_folder, $_time = 60)
1098     {
1099         // always read folder from database
1100         $folder  = Felamimail_Controller_Folder::getInstance()->get($_folder);
1101         
1102         if ($folder->cache_status !== Felamimail_Model_Folder::CACHE_STATUS_COMPLETE) {
1103             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . 
1104                 ' Do not update flags of incomplete folder ' . $folder->globalname
1105             );
1106             return $folder;
1107         }
1108         
1109         if ($this->_availableUpdateTime == 0) {
1110             $this->_availableUpdateTime = $_time;
1111             $this->_timeStart = microtime(true);
1112         }
1113         
1114         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . 
1115             ' Updating flags of folder ' . $folder->globalname .
1116             ' / start time: ' . Tinebase_DateTime::now()->toString() .
1117             ' / available seconds: ' . ($this->_availableUpdateTime - $this->_timeElapsed)
1118         );
1119         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . 
1120             ' Folder: ' . print_r($folder->toArray(), true));
1121         
1122         $imap = Felamimail_Backend_ImapFactory::factory($folder->account_id);
1123         
1124         // switch to folder (read-only)
1125         $imap->examineFolder(Felamimail_Model_Folder::encodeFolderName($folder->globalname));
1126         
1127         if ($folder->supports_condstore) {
1128             $this->_updateCondstoreFlags($imap, $folder);
1129         } else {
1130             $this->_updateAllFlags($imap, $folder);
1131         }
1132         
1133         $updatedCounters = Felamimail_Controller_Cache_Folder::getInstance()->getCacheFolderCounter($folder);
1134         $folder = Felamimail_Controller_Folder::getInstance()->updateFolderCounter($folder, array(
1135             'cache_unreadcount' => $updatedCounters['cache_unreadcount'],
1136         ));
1137         
1138         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
1139             . ' New unreadcount after flags update: ' . $updatedCounters['cache_unreadcount']);
1140         
1141         return $folder;
1142     }
1143     
1144     /**
1145      * update folder flags using condstore
1146      * 
1147      * @param Felamimail_Backend_ImapProxy $imap
1148      * @param Felamimail_Model_Folder $folder
1149      */
1150     protected function _updateCondstoreFlags($imap, Felamimail_Model_Folder $folder)
1151     {
1152         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .
1153             ' Folder supports condstore, fetching flags since last mod seq ' . $folder->imap_lastmodseq);
1154         
1155         $flags = $imap->getChangedFlags($folder->imap_lastmodseq);
1156         
1157         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__
1158             . ' got ' . count($flags) . ' changed flags');
1159         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ 
1160             . ' Flags: ' . print_r($flags, true));
1161         
1162         if (! empty($flags)) {
1163             
1164             if (count($flags) <= $this->_flagSyncCountPerStep) {
1165                 $filter = new Felamimail_Model_MessageFilter(array(
1166                     array(
1167                         'field' => 'account_id', 'operator' => 'equals', 'value' => $folder->account_id
1168                     ),
1169                     array(
1170                         'field' => 'folder_id',  'operator' => 'equals', 'value' => $folder->getId()
1171                     ),
1172                     array(
1173                         'field' => 'messageuid', 'operator' => 'in', 'value' => array_keys($flags)
1174                     )
1175                 ));
1176                 $messages = $this->_backend->search($filter);
1177                 if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
1178                     . ' got ' . count($messages) . ' messages.');
1179                 
1180                 $this->_setFlagsOnCache($flags, $folder, $messages, false);
1181             } else {
1182                 if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .
1183                     ' Got too many changed flags. Maybe this is the initial load of the cache. Just updating last mod seq ...');
1184             }
1185             
1186             foreach ($flags as $flag) {
1187                 if ($folder->imap_lastmodseq < $flag['modseq']) {
1188                     $folder->imap_lastmodseq = $flag['modseq'];
1189                 }
1190             }
1191             
1192             if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .
1193                 ' Got ' . count($flags) . ' changed flags and updated last mod seq to ' . $folder->imap_lastmodseq);
1194             
1195             $folder = Felamimail_Controller_Folder::getInstance()->update($folder);
1196         }
1197     }
1198     
1199     /**
1200      * update all flags of folder
1201      * 
1202      * @param Felamimail_Backend_ImapProxy $imap
1203      * @param Felamimail_Model_Folder $folder
1204      */
1205     protected function _updateAllFlags($imap, Felamimail_Model_Folder $folder)
1206     {
1207         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ .
1208             ' Get all flags for folder');
1209         
1210         $flags = $imap->getFlags(1, INF);
1211         
1212         for ($i = $folder->cache_totalcount; $i > 0; $i -= $this->_flagSyncCountPerStep) {
1213             $firstMessageSequence = ($i - $this->_flagSyncCountPerStep) >= 0 ? $i - $this->_flagSyncCountPerStep : 0;
1214             $messagesWithFlags = $this->_backend->getFlagsForFolder($folder->getId(), $firstMessageSequence, $this->_flagSyncCountPerStep);
1215             $this->_setFlagsOnCache($flags, $folder, $messagesWithFlags);
1216         
1217             if(! $this->_timeLeft()) {
1218                 break;
1219             }
1220         }
1221     }
1222     
1223     /**
1224      * set flags on cache if different
1225      * 
1226      * @param array $flags
1227      * @param Felamimail_Model_Folder $_folderId
1228      * @param Tinebase_Record_RecordSet $messages
1229      * @param boolean $checkDiff
1230      */
1231     protected function _setFlagsOnCache($flags, $folder, $messages, $checkDiff = true)
1232     {
1233         $transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
1234         $supportedFlags = array_keys(Felamimail_Controller_Message_Flags::getInstance()->getSupportedFlags(FALSE));
1235         
1236         $updateCount = 0;
1237         foreach ($messages as $cachedMessage) {
1238             if (isset($flags[$cachedMessage->messageuid]) || array_key_exists($cachedMessage->messageuid, $flags)) {
1239                 $newFlags = array_intersect($flags[$cachedMessage->messageuid]['flags'], $supportedFlags);
1240                 
1241                 if ($checkDiff) {
1242                     $cachedFlags = array_intersect($cachedMessage->flags, $supportedFlags);
1243                     $diff1 = array_diff($cachedFlags, $newFlags);
1244                     $diff2 = array_diff($newFlags, $cachedFlags);
1245                 }
1246                 
1247                 if (! $checkDiff || count($diff1) > 0 || count($diff2) > 0) {
1248                     try {
1249                         $this->_backend->setFlags(array($cachedMessage->getId()), $newFlags, $folder->getId());
1250                         $updateCount++;
1251                     } catch (Zend_Db_Statement_Exception $zdse) {
1252                         if (Tinebase_Core::isLogLevel(Zend_Log::NOTICE)) Tinebase_Core::getLogger()->notice(__METHOD__ . '::' . __LINE__ 
1253                             . ' Could not update flags, maybe message was deleted or is not in the cache yet.');
1254                         Tinebase_Exception::log($zdse);
1255                     }
1256                 }
1257             }
1258         }
1259         
1260         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ . ' Updated ' . $updateCount . ' messages.');
1261         
1262         Tinebase_TransactionManager::getInstance()->commitTransaction($transactionId);
1263     }
1264     
1265     /**
1266      * update folder quota (check if server supports QUOTA first)
1267      * 
1268      * @param Felamimail_Model_Folder $_folder
1269      * @param Felamimail_Backend_ImapProxy $_imap
1270      */
1271     protected function _updateFolderQuota(Felamimail_Model_Folder $_folder, Felamimail_Backend_ImapProxy $_imap)
1272     {
1273         // only do it for INBOX
1274         if ($_folder->localname !== 'INBOX') {
1275             return;
1276         }
1277         
1278         $account = Felamimail_Controller_Account::getInstance()->get($_folder->account_id);
1279         if (! $account->hasCapability('QUOTA')) {
1280             if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
1281                 . ' Account ' . $account->name . ' has no QUOTA capability');
1282             return;
1283         }
1284         
1285         if (Tinebase_Core::isLogLevel(Zend_Log::DEBUG)) Tinebase_Core::getLogger()->debug(__METHOD__ . '::' . __LINE__ 
1286             . ' Getting quota for INBOX ' . $_folder->getId());
1287             
1288         // get quota and save in folder
1289         $quota = $_imap->getQuota($_folder->localname);
1290         
1291         if (! empty($quota) && isset($quota['STORAGE'])) {
1292             $_folder->quota_usage = $quota['STORAGE']['usage'];
1293             $_folder->quota_limit = $quota['STORAGE']['limit'];
1294         } else {
1295             $_folder->quota_usage = 0;
1296             $_folder->quota_limit = 0;
1297         }
1298         
1299         if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__ . ' ' . print_r($quota, TRUE));
1300     }
1301     
1302     /**
1303      * fetch message summary from IMAP server
1304      * 
1305      * @param string $messageUid
1306      * @param string $accountId
1307      * @param string $folderId
1308      * @return array
1309      */
1310     public function getMessageSummary($messageUid, $accountId, $folderId = NULL)
1311     {
1312         $imap = Felamimail_Backend_ImapFactory::factory($accountId);
1313         
1314         if ($folderId !== NULL) {
1315             try {
1316                 $folder = Felamimail_Controller_Folder::getInstance()->get($folderId);
1317                 $imap->selectFolder(Felamimail_Model_Folder::encodeFolderName($folder->globalname));
1318             } catch (Exception $e) {
1319                 if (Tinebase_Core::isLogLevel(Zend_Log::WARN)) Tinebase_Core::getLogger()->warn(__METHOD__ . '::' . __LINE__ 
1320                     . ' Could not select folder ' . $folder->globalname . ': ' . $e->getMessage());
1321             }
1322         }
1323         
1324         $summary = $imap->getSummary($messageUid, NULL, TRUE);
1325         
1326         return $summary;
1327     }
1328 }