0012896: recursively sum up file sizes as folder size
authorPaul Mehrer <p.mehrer@metaways.de>
Tue, 28 Mar 2017 11:03:12 +0000 (13:03 +0200)
committerPhilipp Schüle <p.schuele@metaways.de>
Fri, 31 Mar 2017 09:58:23 +0000 (11:58 +0200)
folders have a size that contains all sizes of subnodes
folders have a revision size, that contains all revision sizes of subnodes

Change-Id: I2282d5dd9b37806c2a5d383502a0e6d65c9289ff
Reviewed-on: http://gerrit.tine20.com/customers/4441
Reviewed-by: Philipp Schüle <p.schuele@metaways.de>
Tested-by: Philipp Schüle <p.schuele@metaways.de>
19 files changed:
tests/tine20/Tinebase/FileSystemTest.php
tine20/Filemanager/Controller/Node.php
tine20/Setup/essentials.xml
tine20/Tinebase/ActionQueue/Worker.php
tine20/Tinebase/Config.php
tine20/Tinebase/Config/Abstract.php
tine20/Tinebase/Core.php
tine20/Tinebase/FileSystem.php
tine20/Tinebase/Frontend/Cli.php
tine20/Tinebase/Fulltext/Indexer.php
tine20/Tinebase/Fulltext/TextExtract.php
tine20/Tinebase/Model/Tree/FileObject.php
tine20/Tinebase/Model/Tree/Node.php
tine20/Tinebase/Model/Tree/Node/Filter.php
tine20/Tinebase/Setup/Update/Release10.php
tine20/Tinebase/Setup/setup.xml
tine20/Tinebase/Tree/FileObject.php
tine20/Tinebase/Tree/Node.php
tine20/worker.php

index 236e724..4b036da 100644 (file)
@@ -36,7 +36,12 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
     protected $_backend;
 
     protected $_oldModLog;
-    
+    protected $_oldIndexContent;
+
+    protected $_rmDir = array();
+
+    protected $_transactionId = null;
+
     /**
      * Runs the test methods of this class.
      *
@@ -61,9 +66,11 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
             $this->markTestSkipped('filesystem base path not found');
         }
 
-        $this->_oldModLog = Tinebase_Core::getConfig()->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE};
-        
-        Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
+        $this->_rmDir = array();
+        $this->_oldModLog = Tinebase_Core::getConfig()->{Tinebase_Config::FILESYSTEM}->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE};
+        $this->_oldIndexContent = Tinebase_Core::getConfig()->{Tinebase_Config::FILESYSTEM}->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT};
+
+        $this->_transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
         
         $this->_controller = new Tinebase_FileSystem();
         $this->_basePath   = '/' . Tinebase_Application::getInstance()->getApplicationByName('Tinebase')->getId() . '/folders/' . Tinebase_Model_Container::TYPE_SHARED;
@@ -79,11 +86,26 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
      */
     protected function tearDown()
     {
+        $fsConfig = Tinebase_Core::getConfig()->get(Tinebase_Config::FILESYSTEM);
+        $fsConfig->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE} = $this->_oldModLog;
+        $fsConfig->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT} = $this->_oldIndexContent;
+        Tinebase_Core::getConfig()->set(Tinebase_Config::FILESYSTEM, $fsConfig);
+        Tinebase_FileSystem::getInstance()->resetBackends();
+
         Tinebase_TransactionManager::getInstance()->rollBack();
         Tinebase_FileSystem::getInstance()->clearStatCache();
+
+        if (!empty($this->_rmDir)) {
+            try {
+                foreach($this->_rmDir as $rmDir) {
+                    Tinebase_FileSystem::getInstance()->rmdir($rmDir, true);
+                }
+            } catch (Exception $e) {
+            }
+            Tinebase_FileSystem::getInstance()->clearStatCache();
+        }
+
         Tinebase_FileSystem::getInstance()->clearDeletedFilesFromFilesystem();
-        Tinebase_Core::getConfig()->set(Tinebase_Config::FILESYSTEM_MODLOGACTIVE, $this->_oldModLog);
-        Tinebase_FileSystem::getInstance()->resetBackends();
     }
     
     public function testMkdir()
@@ -261,9 +283,10 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
         $testDir  = $this->testMkdir();
         $testFile = 'phpunit.txt';
         $testPath = $testDir . '/' . $testFile;
-        
+
         $basePathNode = $this->_controller->stat($testDir);
-        
+        $this->assertEquals(1, $basePathNode->revision);
+
         $handle = $this->_controller->fopen($testPath, 'x');
         
         $this->assertEquals('resource', gettype($handle), 'opening file failed');
@@ -273,18 +296,26 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
         $this->assertEquals(7, $written);
         
         $this->_controller->fclose($handle);
-        
+
         $children = $this->_controller->scanDir($testDir)->name;
-        
+
+        $updatedBasePathNode = $this->_controller->stat($testDir);
+
         $this->assertContains($testFile, $children);
-        $this->assertNotEquals($basePathNode->hash, $this->_controller->stat($testDir)->hash);
+        $this->assertEquals(1, $updatedBasePathNode->revision);
+        $this->assertNotEquals($basePathNode->hash, $updatedBasePathNode->hash);
         
         return $testPath;
     }
 
-    public function testRevisionSizeFile()
+    public function testModLogAndIndexContent()
     {
-        Tinebase_Core::getConfig()->set(Tinebase_Config::FILESYSTEM_MODLOGACTIVE, true);
+        $this->_rmDir[] = $this->_basePath . '/PHPUNIT';
+
+        $fsConfig = Tinebase_Core::getConfig()->get(Tinebase_Config::FILESYSTEM);
+        $fsConfig->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE} = true;
+        $fsConfig->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT} = true;
+        Tinebase_Core::getConfig()->set(Tinebase_Config::FILESYSTEM, $fsConfig);
         $this->_controller->resetBackends();
 
         $testPath = $this->testCreateFile();
@@ -293,14 +324,52 @@ class Tinebase_FileSystemTest extends PHPUnit_Framework_TestCase
         $this->assertEquals(7, $node->size);
         $this->assertEquals(7, $node->revision_size);
 
+        Tinebase_TransactionManager::getInstance()->commitTransaction($this->_transactionId);
+
+        $this->_transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
+
+        $filter = new Tinebase_Model_Tree_Node_Filter(array(
+            array('field' => 'id',          'operator' => 'equals',     'value' => $node->getId()),
+            array('field' => 'content',     'operator' => 'contains',   'value' => 'phpunit'),
+        ));
+        $result = $this->_controller->search($filter);
+        $this->assertEquals(1, $result->count(), 'didn\'t find file');
+
+
+        $filter = new Tinebase_Model_Tree_Node_Filter(array(
+            array('field' => 'id',          'operator' => 'equals',     'value' => $node->getId()),
+            array('field' => 'content',     'operator' => 'contains',   'value' => 'shooo'),
+        ));
+        $result = $this->_controller->search($filter);
+        $this->assertEquals(0, $result->count(), 'did find file where non should be found');
+
+
         $handle = $this->_controller->fopen($testPath, 'w');
-        $written = fwrite($handle, '12345');
+        $written = fwrite($handle, 'abcde');
         $this->assertEquals(5, $written);
         $this->_controller->fclose($handle);
 
         $node = $this->_controller->stat($testPath);
         $this->assertEquals(5, $node->size);
         $this->assertEquals(12, $node->revision_size);
+
+
+        Tinebase_TransactionManager::getInstance()->commitTransaction($this->_transactionId);
+
+        $filter = new Tinebase_Model_Tree_Node_Filter(array(
+            array('field' => 'id',          'operator' => 'equals',     'value' => $node->getId()),
+            array('field' => 'content',     'operator' => 'contains',   'value' => 'abcde'),
+        ));
+        $result = $this->_controller->search($filter);
+        $this->assertEquals(1, $result->count(), 'didn\'t find file');
+
+
+        $filter = new Tinebase_Model_Tree_Node_Filter(array(
+            array('field' => 'id',          'operator' => 'equals',     'value' => $node->getId()),
+            array('field' => 'content',     'operator' => 'contains',   'value' => 'phpunit'),
+        ));
+        $result = $this->_controller->search($filter);
+        $this->assertEquals(0, $result->count(), 'did find file where non should be found');
     }
     
     /**
index 0329c97..a9810b9 100644 (file)
@@ -668,14 +668,8 @@ class Filemanager_Controller_Node extends Tinebase_Controller_Record_Abstract
         switch ($_type) {
             case Tinebase_Model_Tree_Node::TYPE_FILE:
                 $this->_backend->copyTempfile($_tempFileId, $_statpath);
-
-                // fulltext indexing
-                if (true === Filemanager_Config::getInstance()->get(Filemanager_Config::INDEX_CONTENT)) {
-                    $node = $this->_backend->stat($_statpath);
-                    Tinebase_ActionQueue::getInstance()->queueAction('Filemanager_Controller_Node.indexNode', $node->getId());
-                }
-
                 break;
+
             case Tinebase_Model_Tree_Node::TYPE_FOLDER:
                 $node = $this->_backend->mkdir($_statpath);
                 break;
@@ -683,34 +677,6 @@ class Filemanager_Controller_Node extends Tinebase_Controller_Record_Abstract
 
         return $node !== null ? $node : $this->_backend->stat($_statpath);
     }
-
-    public function indexNode($_nodeId)
-    {
-        /** @var Tinebase_Model_Tree_Node $node */
-        try {
-            $node = $this->_backend->get($_nodeId);
-        } catch(Tinebase_Exception_NotFound $tenf) {
-            // TODO log
-            return;
-        }
-        if (Tinebase_Model_Tree_Node::TYPE_FILE !== $node->type) {
-            // TODO log
-            return;
-        }
-        if ($node->hash === $node->indexed_hash) {
-            // nothing to do
-            return true;
-        }
-
-        $tmpFile = Tinebase_Fulltext_TextExtract::getInstance()->nodeToTempFile($node);
-        Tinebase_Fulltext_Indexer::getInstance()->addFileContentsToIndex($node->getId(), $tmpFile);
-        unlink($tmpFile);
-
-        $node->indexed_hash = $node->hash;
-        Tinebase_FileSystem::getInstance()->update($node);
-
-        return true;
-    }
     
     /**
      * check file existance
index e6c82ca..c8c4a41 100644 (file)
@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <essential>
-    <enviroment name="PHP" version="5.4"/>
+    <enviroment name="PHP" version="5.5"/>
     <extensions>
         <extension name="json" />
         <extension name="gd" />
index 7648e8e..793508b 100755 (executable)
@@ -153,24 +153,25 @@ class Tinebase_ActionQueue_Worker extends Console_Daemon
      * execute the action
      *
      * @param  string  $job
+     * //@ throws Exception
      * @todo make self::EXECUTION_METHOD_EXEC_CLI working
      */
     protected function _executeAction($job)
     {
         // execute in subprocess
-        if ($this->_getConfig()->tine20->executionMethod === self::EXECUTION_METHOD_EXEC_CLI) {
+        /*if ($this->_getConfig()->tine20->executionMethod === self::EXECUTION_METHOD_EXEC_CLI) {
             $output = system('php $paths ./../../tine20.php --method Tinebase.executeQueueJob message=' . escapeshellarg($job), $exitCode );
-            if (exitCode != 0) {
+            if ($exitCode != 0) {
                 throw new Exception('Problem during execution with shell: ' . $output);
             }
 
         // execute in same process
-        } else {
+        } else { */
             Tinebase_Core::initFramework();
     
             Tinebase_Core::set(Tinebase_Core::USER, Tinebase_User::getInstance()->getFullUserById($job['account_id']));
             
             Tinebase_ActionQueue::getInstance()->executeAction($job);
-        }
+        //}
     }
 }
index ae18352..a8f2530 100644 (file)
@@ -487,10 +487,11 @@ class Tinebase_Config extends Tinebase_Config_Abstract
     const FULLTEXT_JAVABIN = 'javaBin';
     const FULLTEXT_TIKAJAR = 'tikaJar';
 
-    /**
-     * @var string
-     */
-    const FILESYSTEM_MODLOGACTIVE = 'fileSystemModLogActive';
+    const FILESYSTEM = 'filesystem';
+    const FILESYSTEM_MODLOGACTIVE = 'modLogActive';
+    const FILESYSTEM_NUMKEEPREVISIONS = 'numKeepRevisions';
+    const FILESYSTEM_MONTHKEEPREVISIONS = 'monthKeepRevisions';
+    const FILESYSTEM_INDEX_CONTENT = 'index_content';
 
     /**
      * (non-PHPdoc)
@@ -1345,17 +1346,65 @@ class Tinebase_Config extends Tinebase_Config_Abstract
             'setBySetupModule'      => FALSE,
             'default'               => FALSE,
         ),
-        self::FILESYSTEM_MODLOGACTIVE => array(
-        //_('Filesystem history')
-            'label'                 => 'Filesystem history',
-        //_('Filesystem keeps history, default is false.')
-            'description'           => 'Filesystem keeps history, default is false.',
-            'type'                  => 'bool',
+        self::FILESYSTEM => array(
+            //_('Filesystem settings')
+            'label'                 => 'Filesystem settings',
+            //_('Filesystem settings.')
+            'description'           => 'Filesystem settings.',
+            'type'                  => 'object',
+            'class'                 => 'Tinebase_Config_Struct',
             'clientRegistryInclude' => TRUE,
             'setByAdminModule'      => FALSE,
             'setBySetupModule'      => FALSE,
-            'default'               => FALSE,
+            'content'               => array(
+                self::FILESYSTEM_MODLOGACTIVE => array(
+                    //_('Filesystem history')
+                    'label'                 => 'Filesystem history',
+                    //_('Filesystem keeps history, default is false.')
+                    'description'           => 'Filesystem keeps history, default is false.',
+                    'type'                  => 'bool',
+                    'clientRegistryInclude' => TRUE,
+                    'setByAdminModule'      => FALSE,
+                    'setBySetupModule'      => FALSE,
+                    'default'               => FALSE,
+                ),
+                self::FILESYSTEM_NUMKEEPREVISIONS => array(
+                    //_('Filesystem number of revisions')
+                    'label'                 => 'Filesystem number of revisions',
+                    //_('Filesystem number of revisions being kept before they are automatically deleted.')
+                    'description'           => 'Filesystem number of revisions being kept before they are automatically deleted.',
+                    'type'                  => 'integer',
+                    'clientRegistryInclude' => TRUE,
+                    'setByAdminModule'      => FALSE,
+                    'setBySetupModule'      => FALSE,
+                    'default'               => 100,
+                ),
+                self::FILESYSTEM_MONTHKEEPREVISIONS => array(
+                    //_('Filesystem months of revisions')
+                    'label'                 => 'Filesystem months of revisions',
+                    //_('Filesystem number of months revisions being kept before they are automatically deleted.')
+                    'description'           => 'Filesystem number of months revisions being kept before they are automatically deleted.',
+                    'type'                  => 'integer',
+                    'clientRegistryInclude' => TRUE,
+                    'setByAdminModule'      => FALSE,
+                    'setBySetupModule'      => FALSE,
+                    'default'               => 60,
+                ),
+                self::FILESYSTEM_INDEX_CONTENT => array(
+                    //_('Filesystem index content')
+                    'label'                 => 'Filesystem index content',
+                    //_('Filesystem index content.')
+                    'description'           => 'Filesystem index content.',
+                    'type'                  => 'boolean',
+                    'clientRegistryInclude' => TRUE,
+                    'setByAdminModule'      => FALSE,
+                    'setBySetupModule'      => FALSE,
+                    'default'               => FALSE,
+                ),
+            ),
+            'default'               => array(),
         ),
+
     );
     
     /**
index a638a8f..7ef6ebb 100644 (file)
@@ -224,6 +224,18 @@ abstract class Tinebase_Config_Abstract implements Tinebase_Config_Interface
         ));
         
         $this->_saveConfig($configRecord);
+
+        if (null !== $this->getConfigFileSection($_name)) {
+            if (is_object($_value)) {
+                if ($_value instanceof Tinebase_Config_Struct) {
+                    $_value = $_value->toArray();
+                } else {
+                    return;
+                }
+            }
+            (isset(self::$_configFileData[$this->_appName]) || array_key_exists($this->_appName, self::$_configFileData)) && (isset(self::$_configFileData[$this->_appName][$_name]) || array_key_exists($_name, self::$_configFileData[$this->_appName])) ? self::$_configFileData[$this->_appName][$_name] = $_value :
+                ((isset(self::$_configFileData[$_name]) || array_key_exists($_name, self::$_configFileData)) ? self::$_configFileData[$_name] = $_value : NULL);
+        }
     }
     
     /**
index 71f1f0b..c4a7693 100644 (file)
@@ -349,6 +349,7 @@ class Tinebase_Core
      * returns an instance of the controller of an application
      *
      * ATTENTION if ever refactored, this is called via ActionQueue with the single parameter 'Tinebase_FOO_User' to get Tinebase_User (triggered in Tinebase_User_Sql::deleteUserInSqlBackend()
+     * Tinebase_FOO_Filesystem
      *
      * @param   string $_applicationName appname / modelname
      * @param   string $_modelName
index 3022de4..43802a7 100644 (file)
@@ -44,6 +44,10 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
      * @var string
      */
     protected $_basePath;
+
+    protected $_modLogActive = false;
+
+    protected $_indexingActive = false;
     
     /**
      * stat cache
@@ -69,9 +73,11 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         }
 
         $config = Tinebase_Core::getConfig();
+        $this->_modLogActive = true === $config->{Tinebase_Config::FILESYSTEM}->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE};
+        $this->_indexingActive = true === $config->{Tinebase_Config::FILESYSTEM}->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT};
 
         $this->_fileObjectBackend  = new Tinebase_Tree_FileObject(null, array(
-            Tinebase_Config::FILESYSTEM_MODLOGACTIVE => true === $config->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE}
+            Tinebase_Config::FILESYSTEM_MODLOGACTIVE => $this->_modLogActive
         ));
 
         $this->_basePath = $config->{Tinebase_Config::FILESDIR};
@@ -93,11 +99,14 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
 
     public function resetBackends()
     {
+        $config = Tinebase_Core::getConfig()->{Tinebase_Config::FILESYSTEM};
+        $this->_modLogActive = true === $config->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE};
+        $this->_indexingActive = true === $config->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT};
+
         $this->_treeNodeBackend = null;
-        $config = Tinebase_Core::getConfig();
 
         $this->_fileObjectBackend  = new Tinebase_Tree_FileObject(null, array(
-            Tinebase_Config::FILESYSTEM_MODLOGACTIVE => true === $config->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE}
+            Tinebase_Config::FILESYSTEM_MODLOGACTIVE => $this->_modLogActive
         ));
     }
     
@@ -155,7 +164,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
      * Get one tree node (by id)
      *
      * @param integer|Tinebase_Record_Interface $_id
-     * @param $_getDeleted get deleted records
+     * @param boolean $_getDeleted get deleted records
      * @return Tinebase_Model_Tree_Node
      */
     public function get($_id, $_getDeleted = FALSE)
@@ -170,10 +179,10 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
     protected function _getTreeNodeBackend()
     {
         if ($this->_treeNodeBackend === null) {
-            $config = Tinebase_Core::getConfig();
+
             $this->_treeNodeBackend    = new Tinebase_Tree_Node(null, /* options */ array(
                 'modelName' => $this->_treeNodeModel,
-                Tinebase_Config::FILESYSTEM_MODLOGACTIVE => true === $config->{Tinebase_Config::FILESYSTEM_MODLOGACTIVE}
+                Tinebase_Config::FILESYSTEM_MODLOGACTIVE => $this->_modLogActive
             ));
         }
 
@@ -329,7 +338,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
     /**
      * close file handle
      * 
-     * @param  handle $handle
+     * @param  resource $handle
      * @return boolean
      */
     public function fclose($handle)
@@ -377,6 +386,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
      */
     protected function _updateFileObject($_id, $_hash, $_hashFile = null)
     {
+        /** @var Tinebase_Model_Tree_FileObject $currentFileObject */
         $currentFileObject = $_id instanceof Tinebase_Record_Abstract ? $_id : $this->_fileObjectBackend->get($_id);
 
         if (! $_hash) {
@@ -425,8 +435,106 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         if (empty($updatedFileObject->size)) {
             $updatedFileObject->size = 0;
         }
-        
-        return $this->_fileObjectBackend->update($updatedFileObject);
+
+        /** @var Tinebase_Model_Tree_FileObject $newFileObject */
+        $newFileObject = $this->_fileObjectBackend->update($updatedFileObject);
+
+        $sizeDiff = ((int)$newFileObject->size) - ((int)$currentFileObject->size);
+        $revisionSizeDiff = (((int)$currentFileObject->revision) === ((int)$newFileObject->revision) ? 0 : $newFileObject->revision_size);
+
+        if ($sizeDiff !== 0 || $revisionSizeDiff > 0) {
+            // update parents with new sizes
+            $objectIds = $this->_getTreeNodeBackend()->getAllFolderNodes($this->_getTreeNodeBackend()->getObjectUsage($newFileObject->getId()))->object_id;
+            if (!empty($objectIds)) {
+                /** @var Tinebase_Model_Tree_FileObject $fileObject */
+                foreach($this->_fileObjectBackend->getMultiple($objectIds) as $fileObject) {
+                    if ($fileObject->getId() === $_id) {
+                        continue;
+                    }
+                    $fileObject->size = ((int)$fileObject->size) + $sizeDiff;
+                    $fileObject->revision_size = ((int)$fileObject->revision_size) + $revisionSizeDiff;
+                    $this->_fileObjectBackend->update($fileObject);
+                }
+            }
+        }
+
+        if (true === Tinebase_Config::getInstance()->get(Tinebase_Config::FILESYSTEM)->{Tinebase_Config::FILESYSTEM_INDEX_CONTENT}) {
+            Tinebase_ActionQueue::getInstance()->queueAction('Tinebase_FOO_FileSystem.indexFileObject', $newFileObject->getId());
+        }
+
+        return $newFileObject;
+    }
+
+    /**
+     * @param string $_objectId
+     * @return bool
+     */
+    public function indexFileObject($_objectId)
+    {
+        /** @var Tinebase_Model_Tree_FileObject $fileObject */
+        try {
+            $fileObject = $this->_fileObjectBackend->get($_objectId);
+        } catch(Tinebase_Exception_NotFound $tenf) {
+            if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__
+                . ' Could not find file object ' . $_objectId);
+            return true;
+        }
+        if (Tinebase_Model_Tree_FileObject::TYPE_FILE !== $fileObject->type) {
+            if (Tinebase_Core::isLogLevel(Zend_Log::WARN)) Tinebase_Core::getLogger()->warn(__METHOD__ . '::' . __LINE__
+                . ' file object ' . $_objectId . ' is not a file: ' . $fileObject->type);
+            return true;
+        }
+        if ($fileObject->hash === $fileObject->indexed_hash) {
+            // nothing to do
+            return true;
+        }
+
+        // we clean up $tmpFile down there in finally
+        if (false === ($tmpFile = Tinebase_Fulltext_TextExtract::getInstance()->fileObjectToTempFile($fileObject))) {
+            return false;
+        }
+
+        $indexedHash = $fileObject->hash;
+
+        $transactionId = Tinebase_TransactionManager::getInstance()->startTransaction(Tinebase_Core::getDb());
+
+        try {
+
+            try {
+                $fileObject = $this->_fileObjectBackend->get($_objectId);
+            } catch(Tinebase_Exception_NotFound $tenf) {
+                if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__
+                    . ' Could not find file object ' . $_objectId);
+                return true;
+            }
+            if (Tinebase_Model_Tree_FileObject::TYPE_FILE !== $fileObject->type) {
+                if (Tinebase_Core::isLogLevel(Zend_Log::WARN)) Tinebase_Core::getLogger()->warn(__METHOD__ . '::' . __LINE__
+                    . ' file object ' . $_objectId . ' is not a file: ' . $fileObject->type);
+                return true;
+            }
+            if ($fileObject->hash === $fileObject->indexed_hash || $indexedHash === $fileObject->indexed_hash) {
+                // nothing to do
+                return true;
+            }
+
+            Tinebase_Fulltext_Indexer::getInstance()->addFileContentsToIndex($fileObject->getId(), $tmpFile);
+
+            $fileObject->indexed_hash = $indexedHash;
+            $this->_fileObjectBackend->update($fileObject);
+
+            Tinebase_TransactionManager::getInstance()->commitTransaction($transactionId);
+
+        } catch (Exception $e) {
+            Tinebase_Exception::log($e);
+            Tinebase_TransactionManager::getInstance()->rollBack();
+
+            return false;
+
+        } finally {
+            unlink($tmpFile);
+        }
+
+        return true;
     }
     
     /**
@@ -442,12 +550,16 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         
         // update nodes stored in local statCache
         $subPath = null;
+        /** @var Tinebase_Model_Tree_Node $node */
         foreach ($parentNodes as $node) {
+            /** @var Tinebase_Model_Tree_FileObject $directoryObject */
             $directoryObject = $updatedNodes->getById($node->object_id);
             
             if ($directoryObject) {
-                $node->revision = $directoryObject->revision;
-                $node->hash     = $directoryObject->hash;
+                $node->revision      = $directoryObject->revision;
+                $node->hash          = $directoryObject->hash;
+                $node->size          = $directoryObject->size;
+                $node->revision_size = $directoryObject->revision_size;
             }
             
             $subPath .= "/" . $node->name;
@@ -460,7 +572,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
      * 
      * @param string $_path
      * @param string $_mode
-     * @return handle
+     * @return resource|boolean
      */
     public function fopen($_path, $_mode)
     {
@@ -611,7 +723,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
      *
      * @param  string  $oldPath
      * @param  string  $newPath
-     * @return Tinebase_Model_Tree_Node
+     * @return Tinebase_Model_Tree_Node|boolean
      */
     public function rename($oldPath, $newPath)
     {
@@ -661,6 +773,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         $currentPath = array();
         $parentNode  = null;
         $pathParts   = $this->_splitPath($path);
+        $node = null;
         
         foreach ($pathParts as $pathPart) {
             $pathPart = trim($pathPart);
@@ -714,12 +827,16 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             }
         }
         
-       $this->_getTreeNodeBackend()->softDelete($node->getId());
+        $this->_getTreeNodeBackend()->delete($node->getId());
         $this->clearStatCache($path);
 
         // delete object only, if no other tree node refers to it
+        // we can use treeNodeBackend property because getTreeNodeBackend was called just above
         if ($this->_treeNodeBackend->getObjectCount($node->object_id) == 0) {
             $this->_fileObjectBackend->softDelete($node->object_id);
+            if (false === $this->_modLogActive && true === $this->_indexingActive) {
+                Tinebase_Fulltext_Indexer::getInstance()->removeFileContentsFromIndex($node->object_id);
+            }
         }
         
         return true;
@@ -745,6 +862,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
     /**
      * @param  string  $path
      * @return Tinebase_Model_Tree_Node
+     * @throws Tinebase_Exception_NotFound
      */
     public function stat($path)
     {
@@ -836,11 +954,15 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             throw new Tinebase_Exception_InvalidArgument('can not unlink directories');
         }
         
-       $this->_getTreeNodeBackend()->softDelete($node->getId());
+       $this->_getTreeNodeBackend()->delete($node->getId());
         
         // delete object only, if no one uses it anymore
+        // we can use treeNodeBackend property because getTreeNodeBackend was called just above
         if ($this->_treeNodeBackend->getObjectCount($node->object_id) == 0) {
             $this->_fileObjectBackend->softDelete($node->object_id);
+            if (false === $this->_modLogActive && true === $this->_indexingActive) {
+                Tinebase_Fulltext_Indexer::getInstance()->removeFileContentsFromIndex($node->object_id);
+            }
         }
     }
     
@@ -918,17 +1040,12 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         return $treeNode;
     }
 
-    public function getRealPathForNode(Tinebase_Model_Tree_Node $_node)
-    {
-        $hashDirectory = $this->_basePath . '/' . substr($_node->hash, 0, 3);
-        return $hashDirectory . '/' . substr($_node->hash, 3);
-    }
-
     /**
      * places contents into a file blob
      * 
-     * @param  stream|string|tempFile $contents
+     * @param  resource $contents
      * @return string hash
+     * @throws Tinebase_Exception_NotImplemented
      */
     public function createFileBlob($contents)
     {
@@ -1224,11 +1341,12 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             . ' Scanning ' . $this->_basePath . ' for deleted files ...');
         
         $deleteCount = 0;
+        /** @var DirectoryIterator $item */
         foreach ($dirIterator as $item) {
             if (!$item->isDir()) {
                 continue;
             }
-            $subDir = $item->getFileName();
+            $subDir = $item->getFilename();
             if ($subDir[0] == '.') continue;
             if (Tinebase_Core::isLogLevel(Zend_Log::TRACE)) Tinebase_Core::getLogger()->trace(__METHOD__ . '::' . __LINE__
                 . ' Checking ' . $subDir);
@@ -1237,7 +1355,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             // loop dirs + check if files in dir are in tree_filerevisions
             foreach ($subDirIterator as $file) {
                 if ($file->isFile()) {
-                    $hash = $subDir . $file->getFileName();
+                    $hash = $subDir . $file->getFilename();
                     $hashsToCheck[] = $hash;
                 }
             }
@@ -1284,13 +1402,17 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
 
             $fileObjects = $this->_fileObjectBackend->search($filter, $pagination);
             foreach ($fileObjects as $fileObject) {
-                if ($fileObject->type == Tinebase_Model_Tree_FileObject::TYPE_FILE && $fileObject->hash && !file_exists($fileObject->getFilesystemPath())) {
+                if ($fileObject->type === Tinebase_Model_Tree_FileObject::TYPE_FILE && $fileObject->hash && !file_exists($fileObject->getFilesystemPath())) {
                     $toDeleteIds[] = $fileObject->getId();
                 }
             }
 
             $start += $limit;
         } while ($fileObjects->count() >= $limit);
+
+        if (count($toDeleteIds) === 0) {
+            return 0;
+        }
         
         $nodeIdsToDelete =$this->_getTreeNodeBackend()->search(new Tinebase_Model_Tree_Node_Filter(array(array(
             'field'     => 'object_id',
@@ -1302,6 +1424,11 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         $deleteCount = $this->_getTreeNodeBackend()->delete($nodeIdsToDelete);
         if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__
             . ' Removed ' . $deleteCount . ' obsolete filenode(s) from the database.');
+
+        $this->_fileObjectBackend->softDelete($toDeleteIds);
+        if (false === $this->_modLogActive && true === $this->_indexingActive) {
+            Tinebase_Fulltext_Indexer::getInstance()->removeFileContentsFromIndex($toDeleteIds);
+        }
         
         return $deleteCount;
     }
@@ -1327,7 +1454,8 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             $tempStream = $tempFile;
         } else if (is_string($tempFile) || is_array($tempFile)) {
             $tempFile = Tinebase_TempFile::getInstance()->getTempFile($tempFile);
-            return $this->copyTempfile($tempFile, $path);
+            $this->copyTempfile($tempFile, $path);
+            return;
         } else if ($tempFile instanceof Tinebase_Model_Tree_Node) {
             if (isset($tempFile->hash)) {
                 $hashFile = $this->_basePath . '/' . substr($tempFile->hash, 0, 3) . '/' . substr($tempFile->hash, 3);
@@ -1335,7 +1463,8 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
             } else if (is_resource($tempFile->stream)) {
                 $tempStream = $tempFile->stream;
             } else {
-                return $this->copyTempfile($tempFile->tempFile, $path);
+                $this->copyTempfile($tempFile->tempFile, $path);
+                return;
             }
         } else if ($tempFile instanceof Tinebase_Model_TempFile) {
             $tempStream = fopen($tempFile->path, 'r');
@@ -1349,7 +1478,7 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
     /**
      * copy stream data to file path
      *
-     * @param  stream  $in
+     * @param  resource  $in
      * @param  string  $path
      * @throws Tinebase_Exception_AccessDenied
      * @throws Tinebase_Exception_UnexpectedValue
@@ -1376,4 +1505,49 @@ class Tinebase_FileSystem implements Tinebase_Controller_Interface
         
         $this->fclose($handle);
     }
+
+    /**
+     * recalculates all revision sizes of file objects of type file only
+     *
+     * on error it still continues and tries to calculate as many revision sizes as possible, but returns false
+     *
+     * @return bool
+     */
+    public function recalculateRevisionSize()
+    {
+        return $this->_fileObjectBackend->recalculateRevisionSize();
+    }
+
+    /**
+     * recalculates all folder sizes
+     *
+     * on error it still continues and tries to calculate as many folder sizes as possible, but returns false
+     *
+     * @return bool
+     */
+    public function recalculateFolderSize()
+    {
+        return $this->_getTreeNodeBackend()->recalculateFolderSize($this->_fileObjectBackend);
+    }
+
+    /**
+     * indexes all not indexed file objects
+     *
+     * on error it still continues and tries to index as many file objects as possible, but returns false
+     *
+     * @return bool
+     */
+    public function checkIndexing()
+    {
+        if (false === $this->_indexingActive) {
+            return true;
+        }
+
+        $success = true;
+        foreach($this->_fileObjectBackend->getNotIndexedObjectIds() as $objectId) {
+            $success = $this->indexFileObject($objectId) && $success;
+        }
+
+        return $success;
+    }
 }
index a906ac4..42f91f4 100644 (file)
@@ -1118,12 +1118,13 @@ class Tinebase_Frontend_Cli extends Tinebase_Frontend_Cli_Abstract
     
     /**
      * clears deleted files from filesystem + database
-     * @return boolean
+     *
+     * @return int
      */
     public function clearDeletedFiles()
     {
         if (! $this->_checkAdminRight()) {
-            return FALSE;
+            return -1;
         }
         
         $this->_addOutputLogWriter();
@@ -1132,6 +1133,40 @@ class Tinebase_Frontend_Cli extends Tinebase_Frontend_Cli_Abstract
 
         return 0;
     }
+
+    /**
+     * recalculates the revision sizes and then the folder sizes
+     *
+     * @return int
+     */
+    public function fileSystemSizeRecalculation()
+    {
+        if (! $this->_checkAdminRight()) {
+            return -1;
+        }
+
+        Tinebase_FileSystem::getInstance()->recalculateRevisionSize();
+
+        Tinebase_FileSystem::getInstance()->recalculateFolderSize();
+
+        return 0;
+    }
+
+    /**
+     * checks if there are not yet indexed file objects and adds them to the index synchronously
+     * that means this can be very time consuming
+     *
+     * @return int
+     */
+    public function fileSystemCheckIndexing()
+    {
+
+        if (! $this->_checkAdminRight()) {
+            return -1;
+        }
+
+        Tinebase_FileSystem::getInstance()->checkIndexing();
+    }
     
     /**
      * repair a table
index c6dc47a..8cd54ad 100644 (file)
@@ -60,6 +60,9 @@ class Tinebase_Fulltext_Indexer
 
     /**
      * constructor
+     *
+     * @throws Tinebase_Exception_UnexpectedValue
+     * @throws Tinebase_Exception_NotImplemented
      */
     private function __construct()
     {
@@ -72,14 +75,32 @@ class Tinebase_Fulltext_Indexer
         }
     }
 
+    /**
+     * @param string $_id
+     * @param string $_fileName
+     *
+     * @throws Tinebase_Exception_InvalidArgument
+     */
     public function addFileContentsToIndex($_id, $_fileName)
     {
         if (false === ($blob = file_get_contents($_fileName))) {
-            throw new Tinebase_Exception_UnexpectedValue('could not get file contents of: ' . $_fileName);
+            throw new Tinebase_Exception_InvalidArgument('could not get file contents of: ' . $_fileName);
         }
 
         $db = Tinebase_Core::getDb();
         $db->delete(SQL_TABLE_PREFIX . 'external_fulltext', $db->quoteInto($db->quoteIdentifier('id') . ' = ?', $_id));
         $db->insert(SQL_TABLE_PREFIX . 'external_fulltext', array('id' => $_id, 'text_data' => $blob));
     }
+
+    /**
+     * @param string|array $_ids
+     */
+    public function removeFileContentsFromIndex($_ids)
+    {
+        if (empty($_ids)) {
+            return;
+        }
+        $db = Tinebase_Core::getDb();
+        $db->delete(SQL_TABLE_PREFIX . 'external_fulltext', $db->quoteInto($db->quoteIdentifier('id') . ' IN (?)', (array)$_ids));
+    }
 }
\ No newline at end of file
index 1155176..6ebdac2 100644 (file)
@@ -57,6 +57,8 @@ class Tinebase_Fulltext_TextExtract
 
     /**
      * constructor
+     *
+     * @throws Tinebase_Exception_UnexpectedValue
      */
     private function __construct()
     {
@@ -68,12 +70,29 @@ class Tinebase_Fulltext_TextExtract
         $this->_tikaJar = escapeshellarg($fulltextConfig->{Tinebase_Config::FULLTEXT_TIKAJAR});
     }
 
-    public function nodeToTempFile(Tinebase_Model_Tree_Node $_node)
+    /**
+     * @param Tinebase_Model_Tree_FileObject $_fileObject
+     * @return bool|string
+     * @throws Tinebase_Exception_InvalidArgument
+     */
+    public function fileObjectToTempFile(Tinebase_Model_Tree_FileObject $_fileObject)
     {
+        if (Tinebase_Model_Tree_FileObject::TYPE_FILE !== $_fileObject->type) {
+            throw new Tinebase_Exception_InvalidArgument('$_fileObject needs to be of type file only!');
+        }
+
         $tempFileName = Tinebase_TempFile::getTempPath();
-        $blobFileName = Tinebase_FileSystem::getInstance()->getRealPathForNode($_node);
+        $blobFileName = $_fileObject->getFilesystemPath();
+
+        exec($this->_javaBin . ' -jar '. $this->_tikaJar . ' -t -eUTF8 '
+            . escapeshellarg($blobFileName) . ' > ' . escapeshellarg($tempFileName), $output, $result);
 
-        exec($this->_javaBin . ' -jar '. $this->_tikaJar . ' -t -eUTF8 ' . escapeshellarg($blobFileName) . ' > ' . escapeshellarg($tempFileName));
+        if ($result !== 0) {
+            if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__
+                . ' tika did not return status 0: ' . print_r($output, true) . ' ' . print_r($result, true));
+            unlink($tempFileName);
+            return false;
+        }
 
         return $tempFileName;
     }
index 1ddf1c6..7266ba0 100644 (file)
  * class to hold data representing one object which can be inserted into the tree
  * 
  * @property  string   name
+ * @property  string   revision
+ * @property  string   description
+ * @property  string   contenttype
+ * @property  integer  size
+ * @property  integer  revision_size
+ * @property  string   hash
+ * @property  string   type
  */
 class Tinebase_Model_Tree_FileObject extends Tinebase_Record_Abstract
 {
@@ -77,7 +84,9 @@ class Tinebase_Model_Tree_FileObject extends Tinebase_Record_Abstract
         'description'           => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'contenttype'           => array(Zend_Filter_Input::ALLOW_EMPTY => true, Zend_Filter_Input::DEFAULT_VALUE => 'application/octet-stream'),
         'size'                  => array(Zend_Filter_Input::ALLOW_EMPTY => true, 'Digits'),
+        'revision_size'         => array(Zend_Filter_Input::ALLOW_EMPTY => true, 'Digits'),
         'hash'                  => array(Zend_Filter_Input::ALLOW_EMPTY => true),
+        'indexed_hash'          => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'type'                  => array(
             'presence' => 'required',
             array('InArray', array(self::TYPE_FOLDER, self::TYPE_FILE))
index eca61f5..9f9670a 100644 (file)
@@ -25,6 +25,7 @@
  * @property    string             size
  * @property    string             revision_size
  * @property    string             type
+ * @property    string             revision
  */
 class Tinebase_Model_Tree_Node extends Tinebase_Record_Abstract
 {
@@ -93,7 +94,6 @@ class Tinebase_Model_Tree_Node extends Tinebase_Record_Abstract
             Zend_Filter_Input::DEFAULT_VALUE => '0',
             array('InArray', array(true, false))
         ),
-        'indexed_hash'   => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         
         'relations' => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'notes' => array(Zend_Filter_Input::ALLOW_EMPTY => true),
@@ -109,6 +109,7 @@ class Tinebase_Model_Tree_Node extends Tinebase_Record_Abstract
         'contenttype'    => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'revision'       => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'hash'           => array(Zend_Filter_Input::ALLOW_EMPTY => true),
+        'indexed_hash'   => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'size'           => array(Zend_Filter_Input::ALLOW_EMPTY => true),
         'revision_size'  => array(Zend_Filter_Input::ALLOW_EMPTY => true),
 
index 591ba3c..343b057 100644 (file)
@@ -99,7 +99,7 @@ class Tinebase_Model_Tree_Node_Filter extends Tinebase_Model_Filter_FilterGroup
         'content'              => array(
             'filter'            => 'Tinebase_Model_Filter_ExternalFullText',
             'options'           => array(
-                'idProperty' => 'id',
+                'idProperty' => 'object_id',
             )
         ),
     );
index 9ebdb11..17585ad 100644 (file)
@@ -398,6 +398,14 @@ class Tinebase_Setup_Update_Release10 extends Setup_Update_Abstract
                     <name>creation_time</name>
                     <type>datetime</type>
                 </field>
+                <index>
+                    <name>id</name>
+                    <primary>true</primary>
+                    <field>
+                        <name>id</name>
+                    </field>
+                </index>
+                <index>
                 <name>path</name>
                     <fulltext>true</fulltext>
                     <field>
@@ -423,30 +431,10 @@ class Tinebase_Setup_Update_Release10 extends Setup_Update_Abstract
 
     /**
      * update to 10.11
-     */
-    public function update_10()
-    {
-        if (!$this->_backend->columnExists('is_deleted', 'tree_nodes')) {
-            $declaration = new Setup_Backend_Schema_Field_Xml('<field>
-                    <name>is_deleted</name>
-                    <type>boolean</type>
-                    <default>false</default>
-                </field>');
-
-            $this->_backend->addCol('tree_nodes', $declaration);
-
-            $this->setTableVersion('tree_nodes', '2');
-        }
-
-        $this->setApplicationVersion('Tinebase', '10.11');
-    }
-
-    /**
-     * update to 10.12
      *
      * create external_fulltext table
      */
-    public function update_11()
+    public function update_10()
     {
         $this->_backend->createTable(new Setup_Backend_Schema_Table_Xml('<table>
             <name>external_fulltext</name>
@@ -481,27 +469,15 @@ class Tinebase_Setup_Update_Release10 extends Setup_Update_Abstract
             </declaration>
         </table>'), 'Tinebase', 'external_fulltext');
 
-        if (!$this->_backend->columnExists('tree_nodes', 'indexed_hash')) {
-            $declaration = new Setup_Backend_Schema_Field_Xml('<field>
-                    <name>indexed_hash</name>
-                    <type>text</type>
-                    <length>40</length>
-                </field>');
-
-            $this->_backend->addCol('tree_nodes', $declaration);
-
-            $this->setTableVersion('tree_nodes', '3');
-        }
-
-        $this->setApplicationVersion('Tinebase', '10.12');
+        $this->setApplicationVersion('Tinebase', '10.11');
     }
 
     /**
-     * update to 10.13
+     * update to 10.12
      *
      * add revision_size to tree_fileobjects
      */
-    public function update_12()
+    public function update_11()
     {
 
         if (!$this->_backend->columnExists('total_size', 'tree_fileobjects')) {
@@ -512,11 +488,21 @@ class Tinebase_Setup_Update_Release10 extends Setup_Update_Abstract
                     <default>0</default>
                 </field>');
 
-            $this->_backend->addCol('tree_fileobjects', $declaration);
+            $query = $this->_backend->addAddCol('', 'tree_fileobjects', $declaration);
+
+            $declaration = new Setup_Backend_Schema_Field_Xml('<field>
+                    <name>indexed_hash</name>
+                    <type>text</type>
+                    <length>40</length>
+                </field>');
+
+            $query = $this->_backend->addAddCol($query, 'tree_fileobjects', $declaration);
+
+            $this->_backend->execQueryVoid($query);
 
             $this->setTableVersion('tree_fileobjects', '4');
         }
 
-        $this->setApplicationVersion('Tinebase', '10.13');
+        $this->setApplicationVersion('Tinebase', '10.12');
     }
 }
index c1aef4c..c78959b 100644 (file)
@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <application>
     <name>Tinebase</name>
-    <version>10.13</version>
+    <version>10.12</version>
     <tables>
         <table>
             <name>applications</name>
                     <notnull>true</notnull>
                     <default>0</default>
                 </field>
+                <field>
+                    <name>indexed_hash</name>
+                    <type>text</type>
+                    <length>40</length>
+                </field>
                 <index>
                     <name>id</name>
                     <primary>true</primary>
 
         <table>
             <name>tree_nodes</name>
-            <version>3</version>
+            <version>1</version>
             <declaration>
                 <field>
                     <name>id</name>
index e4eef51..a040bd9 100644 (file)
@@ -52,6 +52,8 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
      */
     protected $_keepOldRevisions = false;
 
+    protected $_getSelectHook = array();
+
     /**
      * the constructor
      *
@@ -92,6 +94,12 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
                 . $this->_db->quoteIdentifier($this->_tableName . '.revision') . ' = ' . $this->_db->quoteIdentifier($this->_revisionsTableName . '.revision'),
             /* select */ array('hash', 'size')
         );
+
+        if (count($this->_getSelectHook) > 0) {
+            foreach($this->_getSelectHook as $hook) {
+                call_user_func_array($hook, array($select));
+            }
+        }
             
         return $select;
     }        
@@ -113,7 +121,7 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
         
         // lock row
         $stmt = $this->_db->query($select);
-        $queryResult = $stmt->fetchAll();
+        $stmt->fetchAll();
         
         // increase revision
         $where = $this->_db->quoteInto($this->_db->quoteIdentifier('id') . ' = ?', $objectId);
@@ -168,11 +176,14 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
         if ($_mode !== 'create') {
             /** @var Tinebase_Model_Tree_FileObject $currentRecord */
             $currentRecord = $this->get($_record);
-            if ($currentRecord->hash !== NULL) {
+            if ($currentRecord->hash !== NULL && !empty($currentRecord->revision)) {
                 if ($currentRecord->hash === $_record->hash && (int)$currentRecord->size === (int)$_record->size) {
                     return;
                 }
                 $updateRevision = TRUE;
+                if (Tinebase_Model_Tree_FileObject::TYPE_FOLDER === $_record->type) {
+                    $createRevision = FALSE;
+                }
             } else {
                 $createRevision = TRUE;
             }
@@ -187,17 +198,19 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
             'created_by'    => is_object(Tinebase_Core::getUser()) ? Tinebase_Core::getUser()->getId() : null,
             'hash'          => $_record->hash,
             'size'          => $_record->size,
-            'revision'      => $this->_getNextRevision($_record),
+            'revision'      => false === $createRevision && Tinebase_Model_Tree_FileObject::TYPE_FOLDER === $_record->type ? 1 : $this->_getNextRevision($_record),
         );
             
         if ($createRevision) {
             $data['id'] = $_record->getId();
             $this->_db->insert($this->_tablePrefix . 'tree_filerevisions', $data);
 
-            // update total size
-            $this->_db->update($this->_tablePrefix . $this->_tableName,
-                array('revision_size' => new Zend_Db_Expr($this->_db->quoteIdentifier('revision_size') . ' + ' . (int)$_record->size)),
-                $this->_db->quoteInto( $this->_db->quoteIdentifier($this->_tablePrefix . $this->_tableName . '.id') . ' = ?', $_record->getId()) );
+            if (Tinebase_Model_Tree_FileObject::TYPE_FILE === $_record->type) {
+                // update total size
+                $this->_db->update($this->_tablePrefix . $this->_tableName,
+                    array('revision_size' => new Zend_Db_Expr($this->_db->quoteIdentifier('revision_size') . ' + ' . (int)$_record->size)),
+                    $this->_db->quoteInto($this->_db->quoteIdentifier($this->_tablePrefix . $this->_tableName . '.id') . ' = ?', $_record->getId()));
+            }
 
         } elseif ($updateRevision) {
             $where = array(
@@ -205,10 +218,12 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
             );
             $this->_db->update($this->_tablePrefix . 'tree_filerevisions', $data, $where);
 
-            // update total size
-            $this->_db->update($this->_tablePrefix . $this->_tableName,
-                array('revision_size' => $_record->size),
-                $this->_db->quoteInto( $this->_db->quoteIdentifier($this->_tablePrefix . $this->_tableName . '.id') . ' = ?', $_record->getId()) );
+            if (Tinebase_Model_Tree_FileObject::TYPE_FILE === $_record->type && (int)$_record->revision_size !== (int) $_record->size) {
+                // update total size
+                $this->_db->update($this->_tablePrefix . $this->_tableName,
+                    array('revision_size' => $_record->size),
+                    $this->_db->quoteInto($this->_db->quoteIdentifier($this->_tablePrefix . $this->_tableName . '.id') . ' = ?', $_record->getId()));
+            }
         }
     }
 
@@ -267,4 +282,81 @@ class Tinebase_Tree_FileObject extends Tinebase_Backend_Sql_Abstract
         
         return $this->getMultiple($nodes->object_id);
     }
+
+    /**
+     * recalculates all revision sizes of file objects of type file only
+     *
+     * on error it still continues and tries to calculate as many revision sizes as possible, but returns false
+     *
+     * @return bool
+     */
+    public function recalculateRevisionSize()
+    {
+        $success = true;
+
+        // fetch ids only, no transaction
+        $ids = $this->search(new Tinebase_Model_Tree_FileObjectFilter(array(
+                array('field' => 'type', 'operator' => 'equals', 'value' => Tinebase_Model_Tree_FileObject::TYPE_FILE)
+            )), null, true);
+        $transactionManager = Tinebase_TransactionManager::getInstance();
+        $dbExpr = new Zend_Db_Expr('sum(size)');
+
+        foreach($ids as $id) {
+            $transactionId = $transactionManager->startTransaction($this->_db);
+            try {
+                try {
+                    /** @var Tinebase_Model_Tree_FileObject $record */
+                    $record = $this->get($id);
+                } catch (Tinebase_Exception_NotFound $tenf) {
+                    $transactionManager->commitTransaction($transactionId);
+                    continue;
+                }
+
+                $stmt = $this->_db->query($this->_db->select()->from($this->_tablePrefix . $this->_revisionsTableName, array($dbExpr))
+                    ->where('id = ?', $id));
+                if (($row = $stmt->fetch(Zend_Db::FETCH_NUM)) && ((int)$row[0]) !== ((int)$record->revision_size)) {
+
+                    if (Tinebase_Core::isLogLevel(Zend_Log::INFO)) Tinebase_Core::getLogger()->info(__METHOD__ . '::' . __LINE__
+                        . ' revision size mismatch on ' . $id . ': ' . $row[0] .' != ' . $record->revision_size);
+
+                    $record->revision_size = $row[0];
+                    $this->update($record);
+                }
+
+                $transactionManager->commitTransaction($transactionId);
+
+            // this shouldn't happen
+            } catch (Exception $e) {
+                $transactionManager->rollBack();
+                Tinebase_Exception::log($e);
+                $success = false;
+            }
+        }
+
+        return $success;
+    }
+
+    /**
+     * @param Zend_Db_Select $_select
+     */
+    protected function addNotIndexedWhere(Zend_Db_Select $_select)
+    {
+        $_select->where($this->_db->quoteIdentifier($this->_tableName . '.indexedHash') . ' <> ' . $this->_db->quoteIdentifier($this->_revisionsTableName . '.hash'));
+    }
+
+    /**
+     * @return array
+     */
+    public function getNotIndexedObjectIds()
+    {
+        $this->_getSelectHook = array(array($this, 'addNotIndexedWhere'));
+
+        $fileObjects = $this->search(new Tinebase_Model_Tree_FileObjectFilter(
+                array('field' => 'type', 'operator' => 'equals', 'value' => Tinebase_Model_Tree_FileObject::TYPE_FILE)
+            ), null, true);
+
+        $this->_getSelectHook = array();
+
+        return $fileObjects;
+    }
 }
index 62e95d7..e4709ad 100644 (file)
@@ -51,9 +51,9 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
      */
     public function __construct($_dbAdapter = NULL, $_options = array())
     {
-        if (isset($_options[Tinebase_Config::FILESYSTEM_MODLOGACTIVE]) && true === $_options[Tinebase_Config::FILESYSTEM_MODLOGACTIVE]) {
+        /*if (isset($_options[Tinebase_Config::FILESYSTEM_MODLOGACTIVE]) && true === $_options[Tinebase_Config::FILESYSTEM_MODLOGACTIVE]) {
             $this->_modlogActive = true;
-        }
+        }*/
 
         parent::__construct($_dbAdapter, $_options);
     }
@@ -73,7 +73,7 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
             ->joinLeft(
                 /* table  */ array('tree_fileobjects' => $this->_tablePrefix . 'tree_fileobjects'), 
                 /* on     */ $this->_db->quoteIdentifier($this->_tableName . '.object_id') . ' = ' . $this->_db->quoteIdentifier('tree_fileobjects.id'),
-                /* select */ array('type', 'created_by', 'creation_time', 'last_modified_by', 'last_modified_time', 'revision', 'contenttype', 'revision_size')
+                /* select */ array('type', 'created_by', 'creation_time', 'last_modified_by', 'last_modified_time', 'revision', 'contenttype', 'revision_size', 'indexed_hash')
             )
             ->joinLeft(
                 /* table  */ array('tree_filerevisions' => $this->_tablePrefix . 'tree_filerevisions'), 
@@ -122,6 +122,7 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
      * 
      * @param  string|Tinebase_Model_Tree_Node  $parentId   the id of the parent node
      * @param  string|Tinebase_Model_Tree_Node  $childName  the name of the child node
+     * @throws Tinebase_Exception_NotFound
      * @return Tinebase_Model_Tree_Node
      */
     public function getChild($parentId, $childName)
@@ -171,6 +172,45 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
         
         return $children;
     }
+
+    /**
+     * returns all directory nodes up to the root
+     *
+     * @param Tinebase_Record_RecordSet $_nodes
+     * @param Tinebase_Record_RecordSet $_result
+     * @return Tinebase_Record_RecordSet
+     */
+    public function getAllFolderNodes(Tinebase_Record_RecordSet $_nodes, Tinebase_Record_RecordSet $_result = null)
+    {
+        if (null === $_result) {
+            $_result = new Tinebase_Record_RecordSet('Tinebase_Model_Tree_Node');
+        }
+
+        $ids = array();
+        /** @var Tinebase_Model_Tree_Node $node */
+        foreach($_nodes as $node) {
+            if (Tinebase_Model_Tree_Node::TYPE_FOLDER === $node->type) {
+                $_result->addRecord($node);
+            }
+            if (!empty($node->parent_id)) {
+                $ids[] = $node->parent_id;
+            }
+        }
+
+        if (!empty($ids)) {
+            $searchFilter = new Tinebase_Model_Tree_Node_Filter(array(
+                array(
+                    'field'     => 'id',
+                    'operator'  => 'in',
+                    'value'     => $ids
+                )
+            ));
+            $parents = $this->search($searchFilter);
+            $this->getAllFolderNodes($parents, $_result);
+        }
+
+        return $_result;
+    }
     
     /**
      * @param  string  $path
@@ -191,6 +231,17 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
      */
     public function getObjectCount($_objectId)
     {
+        return $this->getObjectUsage($_objectId)->count();
+    }
+
+    /**
+     * get object usage
+     *
+     * @param string $_objectId
+     * @return Tinebase_Record_RecordSet
+     */
+    public function getObjectUsage($_objectId)
+    {
         $searchFilter = new Tinebase_Model_Tree_Node_Filter(array(
             array(
                 'field'     => 'object_id',
@@ -198,9 +249,7 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
                 'value'     => $_objectId
             )
         ));
-        $result = $this->search($searchFilter);
-        
-        return $result->count();
+        return $this->search($searchFilter);
     }
     
     /**
@@ -208,10 +257,12 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
      * 
      * @param string $_path
      * @return Tinebase_Record_RecordSet
+     * @throws Tinebase_Exception_InvalidArgument
+     * @throws Tinebase_Exception_NotFound
      */
-    public function getPathNodes($path)
+    public function getPathNodes($_path)
     {
-        $pathParts = $this->splitPath($path);
+        $pathParts = $this->splitPath($_path);
         
         if (empty($pathParts)) {
             throw new Tinebase_Exception_InvalidArgument('empty path provided');
@@ -236,7 +287,7 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
             $node = $this->search($searchFilter)->getFirstRecord();
             
             if (!$node) {
-                throw new Tinebase_Exception_NotFound('path: ' . $path . ' not found!');
+                throw new Tinebase_Exception_NotFound('path: ' . $_path . ' not found!');
             }
             
             $pathNodes->addRecord($node);
@@ -281,8 +332,135 @@ class Tinebase_Tree_Node extends Tinebase_Backend_Sql_Abstract
      * @param  string  $_path
      * @return array
      */
-    public function splitPath($path)
+    public function splitPath($_path)
+    {
+        return explode('/', $this->sanitizePath($_path));
+    }
+
+    /**
+     * recalculates all folder sizes
+     *
+     * on error it still continues and tries to calculate as many folder sizes as possible, but returns false
+     *
+     * @param Tinebase_Tree_FileObject $_fileObjectBackend
+     * @return bool
+     */
+    public function recalculateFolderSize(Tinebase_Tree_FileObject $_fileObjectBackend)
     {
-        return explode('/', $this->sanitizePath($path));
+        // no transactions yet
+        // get root node ids
+        $searchFilter = new Tinebase_Model_Tree_Node_Filter(array(
+            array(
+                'field'     => 'parent_id',
+                'operator'  => 'isnull',
+                'value'     => null
+            ), array(
+                'field'     => 'type',
+                'operator'  => 'equals',
+                'value'     => Tinebase_Model_Tree_Node::TYPE_FOLDER
+            )
+        ));
+        return $this->_recalculateFolderSize($_fileObjectBackend, $this->_getIdsOfDeepestFolders($this->search($searchFilter, null, true)));
+    }
+
+    /**
+     * @param Tinebase_Tree_FileObject $_fileObjectBackend
+     * @param array $_folderIds
+     * @param bool
+     */
+    protected function _recalculateFolderSize(Tinebase_Tree_FileObject $_fileObjectBackend, array $_folderIds)
+    {
+        $success = true;
+        $parentIds = array();
+        $transactionManager = Tinebase_TransactionManager::getInstance();
+
+        foreach($_folderIds as $id) {
+            $transactionId = $transactionManager->startTransaction($this->_db);
+
+            try {
+                try {
+                    /** @var Tinebase_Model_Tree_Node $record */
+                    $record = $this->get($id);
+                } catch (Tinebase_Exception_NotFound $tenf) {
+                    $transactionManager->commitTransaction($transactionId);
+                    continue;
+                }
+
+                if (!empty($record->parent_id) && !isset($parentIds[$record->parent_id])) {
+                    $parentIds[$record->parent_id] = $record->parent_id;
+                }
+
+                $childrenNodes = $this->getChildren($id);
+                $size = 0;
+                $revision_size = 0;
+
+                /** @var Tinebase_Model_Tree_Node $child */
+                foreach($childrenNodes as $child) {
+                    $size += ((int)$child->size);
+                    $revision_size += ((int)$child->revision_size);
+                }
+
+                if ($size !== ((int)$record->size) || $revision_size !== ((int)$record->revision_size)) {
+                    /** @var Tinebase_Model_Tree_FileObject $fileObject */
+                    $fileObject = $_fileObjectBackend->get($record->object_id);
+                    $fileObject->size = $size;
+                    $fileObject->revision_size = $revision_size;
+                    $_fileObjectBackend->update($fileObject);
+                }
+
+                $transactionManager->commitTransaction($transactionId);
+
+            // this shouldn't happen
+            } catch (Exception $e) {
+                $transactionManager->rollBack();
+                Tinebase_Exception::log($e);
+                $success = false;
+            }
+        }
+
+        if (!empty($parentIds)) {
+            $success = $this->_recalculateFolderSize($_fileObjectBackend, $parentIds) && $success;
+        }
+
+        return $success;
+    }
+
+    /**
+     * returns ids of folders that do not have any sub folders
+     *
+     * @param array $_folderIds
+     * @return array
+     */
+    protected function _getIdsOfDeepestFolders(array $_folderIds)
+    {
+        $result = array();
+        $subFolderIds = array();
+        foreach($_folderIds as $folderId) {
+            // children folders
+            $searchFilter = new Tinebase_Model_Tree_Node_Filter(array(
+                array(
+                    'field'     => 'parent_id',
+                    'operator'  => 'equals',
+                    'value'     => $folderId
+                ), array(
+                    'field'     => 'type',
+                    'operator'  => 'equals',
+                    'value'     => Tinebase_Model_Tree_Node::TYPE_FOLDER
+                )
+            ));
+            $nodeIds = $this->search($searchFilter, null, true);
+            if (empty($nodeIds)) {
+                // no children, this is a result
+                $result[] = $folderId;
+            } else {
+                $subFolderIds = array_merge($subFolderIds, $nodeIds);
+            }
+        }
+
+        if (!empty($subFolderIds)) {
+            $result = array_merge($result, $this->_getIdsOfDeepestFolders($subFolderIds));
+        }
+
+        return $result;
     }
 }
index 3186f45..f2569d1 100644 (file)
@@ -1,11 +1,11 @@
 <?php
 /**
- * this is the general file any request should be routed trough
+ * this is action queue worker daemon
  *
  * @package     Tinebase
  * @license     http://www.gnu.org/licenses/agpl.html AGPL Version 3
  * @author      Lars Kneschke <l.kneschke@metaways.de>
- * @copyright   Copyright (c) 2013-2013 Metaways Infosystems GmbH (http://www.metaways.de)
+ * @copyright   Copyright (c) 2013-2017 Metaways Infosystems GmbH (http://www.metaways.de)
  */
 
 if (PHP_SAPI != 'cli') {