From 3e1d5cbe7600ecb35629fa631f48df4daadf8928 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Sun, 5 Jul 2015 17:36:23 +0200 Subject: [PATCH 01/21] Fix md5 calls --- clustering/dfs/ezpostsgresqlbackend.php | 46 +- .../ezpostsgresqlbackend.php.MD5_incapsulato | 1921 ++++++++++++++++ ...gresqlbackend.php.MD5_incapsulato_corretto | 1921 ++++++++++++++++ .../dfs/ezpostsgresqlbackend.php.originale | 1910 ++++++++++++++++ ...esqlbackend.php_ante_modifiche_luca_22_mag | 1924 +++++++++++++++++ sql/postgresql/cluster_dfs_schema.sql | 1 + 6 files changed, 7707 insertions(+), 16 deletions(-) create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.originale create mode 100644 clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index e89d353..6be86e5 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -784,7 +784,8 @@ public function _rename( $srcFilePath, $dstFilePath ) $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); // Mark entry for update to lock it - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { // @todo Throw an exception @@ -799,9 +800,11 @@ public function _rename( $srcFilePath, $dstFilePath ) // Create a new meta-data entry for the new file to make foreign keys happy. $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - "WHERE name_hash=MD5($srcFilePathStr)"; + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); @@ -816,7 +819,8 @@ public function _rename( $srcFilePath, $dstFilePath ) } // Remove old entry - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); @@ -1162,7 +1166,7 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ $stmt = $this->db->query( $query ); if ( !$stmt ) { - $this->_error( $query, $fname, $error ); + $this->_error( $query, $stmt, $fname, $error ); eZDebug::accumulatorStop( 'postgresql_cluster_query' ); // @todo Throw an exception return false; @@ -1410,7 +1414,8 @@ protected function _quote( $value ) **/ protected function _md5( $value ) { - return "MD5(" . $this->_quote( $value ) . ")"; + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); } /** @@ -1494,6 +1499,9 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . "VALUES(" . implode( ', ', $insertData ) . ")"; + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; + try { $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); } catch( PDOException $e ) { @@ -1558,7 +1566,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) // no rename: the .generating entry is just deleted if ( $rename === false ) { - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); $this->dfsbackend->delete( $generatingFilePath ); return true; } @@ -1569,7 +1578,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_begin( $fname ); // both files are locked for update - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); @@ -1577,7 +1587,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); // the original file does not exist: we move the generating file - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); if ( $stmt->rowCount() == 0 ) { $metaData = $generatingMetaData; @@ -1598,7 +1609,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_rollback( $fname ); throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } // the original file exists: we move the generating data to this file // and update it @@ -1612,12 +1624,14 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $mtime = $generatingMetaData['mtime']; $filesize = $generatingMetaData['size']; - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } $this->_commit( $fname ); @@ -1652,7 +1666,7 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi if ( !$stmt ) { // @todo Throw an exception - $this->_error( $query, $fname ); + $this->_error( $query, $stmt, $fname ); return false; } $numRows = $stmt->rowCount(); @@ -1778,7 +1792,7 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) { $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - if ( count( $scopes ) == 0 ) + if ( count( $scopes ) == 0 || $scopes == false ) throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); $scopeString = $this->_sqlList( $scopes ); @@ -1839,7 +1853,7 @@ public function deleteCacheFiles( $limit ) /** * DB connexion handle - * @var handle + * @var PDO */ public $db = null; @@ -1907,4 +1921,4 @@ public function deleteCacheFiles( $limit ) * @var int */ const ERROR_UNIQUE_VIOLATION = 23505; -} \ No newline at end of file +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato new file mode 100644 index 0000000..577a717 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato @@ -0,0 +1,1921 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto new file mode 100644 index 0000000..73927bd --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto @@ -0,0 +1,1921 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePathStr ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePathStr ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this-> md5( $value ) ; + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.originale b/clustering/dfs/ezpostsgresqlbackend.php.originale new file mode 100644 index 0000000..e89d353 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.originale @@ -0,0 +1,1910 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + "WHERE name_hash=MD5($srcFilePathStr)"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + return "MD5(" . $this->_quote( $value ) . ")"; + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} \ No newline at end of file diff --git a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag new file mode 100644 index 0000000..feaf076 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag @@ -0,0 +1,1924 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 || $scopes == false ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/sql/postgresql/cluster_dfs_schema.sql b/sql/postgresql/cluster_dfs_schema.sql index aabdca4..46cc92e 100644 --- a/sql/postgresql/cluster_dfs_schema.sql +++ b/sql/postgresql/cluster_dfs_schema.sql @@ -25,3 +25,4 @@ CREATE TABLE ezdfsfile_cache ( CREATE INDEX ezdfsfile_cache_name ON ezdfsfile_cache ( name ); CREATE INDEX ezdfsfile_cache_mtime ON ezdfsfile_cache ( mtime ); + From fe979fe1841c20b07484f884441dafdd22767a69 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 16:59:47 +0200 Subject: [PATCH 02/21] remove old md5 comments --- clustering/dfs/ezpostsgresqlbackend.php | 11 - .../ezpostsgresqlbackend.php.MD5_incapsulato | 1921 ---------------- ...gresqlbackend.php.MD5_incapsulato_corretto | 1921 ---------------- .../dfs/ezpostsgresqlbackend.php.originale | 1910 ---------------- ...esqlbackend.php_ante_modifiche_luca_22_mag | 1924 ----------------- 5 files changed, 7687 deletions(-) delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.originale delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 6be86e5..fff1355 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -784,7 +784,6 @@ public function _rename( $srcFilePath, $dstFilePath ) $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -800,10 +799,8 @@ public function _rename( $srcFilePath, $dstFilePath ) // Create a new meta-data entry for the new file to make foreign keys happy. $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; "WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -819,7 +816,6 @@ public function _rename( $srcFilePath, $dstFilePath ) } // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -1414,7 +1410,6 @@ protected function _quote( $value ) **/ protected function _md5( $value ) { - //return "MD5(" . $this->_quote( $value ) . ")"; return $this->_quote( md5( $value ) ); } @@ -1566,7 +1561,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) // no rename: the .generating entry is just deleted if ( $rename === false ) { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); $this->dfsbackend->delete( $generatingFilePath ); return true; @@ -1578,7 +1572,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_begin( $fname ); // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); @@ -1587,7 +1580,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); if ( $stmt->rowCount() == 0 ) { @@ -1609,7 +1601,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_rollback( $fname ); throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } // the original file exists: we move the generating data to this file @@ -1624,13 +1615,11 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $mtime = $generatingMetaData['mtime']; $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato deleted file mode 100644 index 577a717..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato +++ /dev/null @@ -1,1921 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this->_quote( md5( $value ) ); - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto deleted file mode 100644 index 73927bd..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto +++ /dev/null @@ -1,1921 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePathStr ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePathStr ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this-> md5( $value ) ; - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.originale b/clustering/dfs/ezpostsgresqlbackend.php.originale deleted file mode 100644 index e89d353..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.originale +++ /dev/null @@ -1,1910 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - "WHERE name_hash=MD5($srcFilePathStr)"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - return "MD5(" . $this->_quote( $value ) . ")"; - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} \ No newline at end of file diff --git a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag deleted file mode 100644 index feaf076..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag +++ /dev/null @@ -1,1924 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this->_quote( md5( $value ) ); - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch - //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 || $scopes == false ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} From f0b506938a86929bb19514f63fdedec2b51cd9bf Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 22:18:30 +0200 Subject: [PATCH 03/21] fix phpdoc and some typos --- clustering/dfs/ezpostsgresqlbackend.php | 554 +++++++++++++----------- 1 file changed, 290 insertions(+), 264 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index fff1355..e1abc77 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -9,10 +9,10 @@ */ /** - * This class allows DFS based clustering using PostgreSQL + * This class allows DFS based clustering using PostgresSQL * @package Cluster */ -class eZDFSFileHandlerPostgresqlBackend +class eZDFSFileHandlerPostgresqlBackend implements eZDFSFileHandlerDBBackendInterface { /** @@ -26,7 +26,7 @@ class eZDFSFileHandlerPostgresqlBackend * @var int */ protected $maxCopyTries; - + public function __construct() { $this->eventHandler = ezpEvent::getInstance(); @@ -154,13 +154,17 @@ public function _disconnect() /** * Creates a copy of a file in DB+DFS + * + * @see eZDFSFileHandler::fileCopy + * @see _copyInner + * * @param string $srcFilePath Source file * @param string $dstFilePath Destination file - * @param string $fname + * @param bool|string $fname Optional caller name for debugging + * * @return bool * - * @see _copyInner - **/ + */ public function _copy( $srcFilePath, $dstFilePath, $fname = false ) { if ( $fname ) @@ -192,7 +196,7 @@ public function _copy( $srcFilePath, $dstFilePath, $fname = false ) * * @see _copy */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + protected function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) { $this->_delete( $dstFilePath, true, $fname ); @@ -216,13 +220,13 @@ private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), $fname ) === false ) { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); } // Copy file data. if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); } return true; } @@ -256,7 +260,7 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname } if ( !$stmt = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Purging file metadata for $filePath failed" ); + $this->_fail( "Purging file metadata for $filePath failed" ); } if ( $stmt->rowCount() == 1 ) { @@ -268,6 +272,10 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname /** * Purges meta-data and file-data for files matching a pattern using a SQL * LIKE syntax. + * This method should also remove the files from disk + * + * @see eZDFSFileHandler::purge + * @see _purge * * @param string $like * SQL LIKE string applied to ezdfsfile.name to look for files to @@ -275,13 +283,12 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname * @param bool $onlyExpired * Only purge expired files (ezdfsfile.expired = 1) * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry + * @param integer|bool $expiry * Timestamp used to limit deleted files: only files older than this * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge + * @param bool|string $fname Optional caller name for debugging + * * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk */ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) { @@ -311,9 +318,10 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry if ( !$stmt = $this->_query( $selectSQL, $fname ) ) { $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); + $this->_fail( "Selecting file metadata by like statement $like failed" ); } + $files = array(); // if there are no results, we can just return 0 and stop right here if ( $stmt->rowCount() == 0 ) { @@ -330,12 +338,12 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry } // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " WHERE name_hash IN " . "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) { $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); + $this->_fail( "Purging file metadata by like statement $like failed" ); } $deletedDBFiles = $stmt->rowCount(); $this->dfsbackend->delete( $files ); @@ -347,17 +355,20 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry /** * Deletes a file from DB - * * The file won't be removed from disk, _purge has to be used for this. * Only single files will be deleted, to delete multiple files, * _deleteByLike has to be used. * + * @see eZDFSFileHandler::fileDelete + * @see eZDFSFileHandler::delete + * @see _deleteInner + * @see _deleteByLike + * * @param string $filePath Path of the file to delete * @param bool $insideOfTransaction * Wether or not a transaction is already started * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike + * * @return bool */ public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) @@ -366,16 +377,13 @@ public function _delete( $filePath, $insideOfTransaction = false, $fname = false $fname .= "::_delete($filePath)"; else $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of + // @todo Check if this is required: _protect will already take care of // checking if a transaction is running. But leave it like this // for now. if ( $insideOfTransaction ) { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } + return $this->_deleteInner( $filePath, $fname ); + } else { @@ -394,23 +402,25 @@ public function _delete( $filePath, $insideOfTransaction = false, $fname = false protected function _deleteInner( $filePath, $fname ) { if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); + $this->_fail( "Deleting file $filePath failed" ); return true; } /** * Deletes multiple files using a SQL LIKE statement - * * Use _delete if you need to delete single files * + * @see eZDFSFileHandler::fileDelete + * @see _deleteByLikeInner + * @see _delete + * * @param string $like * SQL LIKE condition applied to ezdfsfile.name to look for files * to delete. Will use name_trunk if the LIKE string matches a * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging + * @param bool|string $fname Optional caller name for debugging + * * @return bool - * @see _deleteByLikeInner - * @see _delete */ public function _deleteByLike( $like, $fname = false ) { @@ -423,18 +433,23 @@ public function _deleteByLike( $like, $fname = false ) } /** - * Callback used by _deleteByLike to perform the deletion + * @see _deleteByLike * * @param string $like - * @param mixed $fname - * @return + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param bool|string $fname Optional caller name for debugging + * + * @return bool|void + * @throws Exception */ - private function _deleteByLikeInner( $like, $fname ) + protected function _deleteByLikeInner( $like, $fname ) { $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by like: '$like'" ); + $this->_fail( "Failed to delete files by like: '$like'" ); } return true; } @@ -458,19 +473,19 @@ public function _deleteByRegex( $regex, $fname = false ) } /** - * Callback used by _deleteByRegex to perform the deletion + * Deletes DB files by using a SQL regular expression applied to file names * - * @param mixed $regex + * @param string $regex * @param mixed $fname - * @return - * @deprecated Has severe performances issues + * @return bool + * @deprecated Has severe performance issues */ - public function _deleteByRegexInner( $regex, $fname ) + protected function _deleteByRegexInner( $regex, $fname ) { $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); + $this->_fail( "Failed to delete files by regex: '$regex'" ); } return true; } @@ -517,7 +532,7 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); } return true; } @@ -560,7 +575,7 @@ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, else $fname = "_exists($filePath)"; $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); + $fname, "Failed to check file '$filePath' existence: ", true ); if ( $row === false ) return false; @@ -608,14 +623,17 @@ protected function __mkdir_p( $dir ) } /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ + * Fetches the file $filePath from the database to its own name + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @see eZDFSFileHandler::fileFetch + * @see eZDFSFileHandler::fetchUnique + * + * @param string $filePath + * @param bool|string $uniqueName Alternative name to save the file to + * + * @return string|bool the file physical path, or false if fetch failed + */ public function _fetch( $filePath, $uniqueName = false ) { $metaData = $this->_fetchMetadata( $filePath ); @@ -703,16 +721,19 @@ public function _fetchContents( $filePath, $fname = false ) // @todo Catch an exception if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + eZDebug::writeError("An error occurred while reading contents of DFS://$filePath", __METHOD__ ); return false; } return $contents; } /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. + * Fetches and returns metadata for $filePath + * + * @see eZDFSFileHandler::loadMetaData + * @param string $filePath + * @param bool|string $fname Optional caller name for debugging + * @return array|false file metadata, or false if the file does not exist in database. */ function _fetchMetadata( $filePath, $fname = false ) { @@ -736,11 +757,16 @@ public function _linkCopy( $srcPath, $dstPath, $fname = false ) } /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) + * Passes $filePath content through + * + * @param string $filePath + * @param int $startOffset Byte offset to start download from + * @param int|bool $length Byte length to be sent + * @param bool|string $fname Optional caller name for debugging + * + * @return bool + */ + public function _passThrough( $filePath, $startOffset = 0, $length = false, $fname = false ) { if ( $fname ) $fname .= "::_passThrough($filePath)"; @@ -753,7 +779,7 @@ public function _passThrough( $filePath, $fname = false ) return false; // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); + $this->dfsbackend->passthrough( $filePath, $startOffset, $length ); return true; } @@ -768,7 +794,7 @@ public function _passThrough( $filePath, $fname = false ) public function _rename( $srcFilePath, $dstFilePath ) { if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; + return false; // fetch source file metadata $metaData = $this->_fetchMetadata( $srcFilePath ); @@ -779,7 +805,6 @@ public function _rename( $srcFilePath, $dstFilePath ) $this->_begin( __METHOD__ ); - $srcFilePathStr = $this->_quote( $srcFilePath ); $dstFilePathStr = $this->_quote( $dstFilePath ); $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); @@ -812,7 +837,7 @@ public function _rename( $srcFilePath, $dstFilePath ) if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); } // Remove old entry @@ -837,27 +862,29 @@ public function _rename( $srcFilePath, $dstFilePath ) /** * Stores $filePath to cluster * + * @see eZDFSFileHandler::fileStore + * * @param string $filePath * @param string $datatype * @param string $scope - * @param string $fname - * @return void + * @param bool|string $fname Optional caller name for debugging + * + * @return bool */ function _store( $filePath, $datatype, $scope, $fname = false ) { if ( !is_readable( $filePath ) ) { eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; + return false; } if ( $fname ) $fname .= "::_store($filePath, $datatype, $scope)"; else $fname = "_store($filePath, $datatype, $scope)"; - $return = $this->_protect( array( $this, '_storeInner' ), $fname, + return $this->_protect( array( $this, '_storeInner' ), $fname, $filePath, $datatype, $scope, $fname ); - return $return; } /** @@ -890,13 +917,13 @@ function _storeInner( $filePath, $datatype, $scope, $fname ) array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), $fname ) === false ) { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); } // copy given $filePath to DFS if ( !$this->dfsbackend->copyToDFS( $filePath ) ) { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); } return true; @@ -905,13 +932,17 @@ function _storeInner( $filePath, $datatype, $scope, $fname ) /** * Stores $contents as the contents of $filePath to the cluster * + * @see eZDFSFileHandler::fileStore + * @see eZDFSFileHandler::storeContents + * * @param string $filePath * @param string $contents * @param string $scope * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void + * @param bool|int $mtime + * @param bool|string $fname Optional caller name for debugging + * + * @return bool */ function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) { @@ -932,7 +963,6 @@ function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $ $nameTrunk = self::nameTrunk( $filePath, $scope ); if ( $mtime === false ) $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; // Copy file metadata. $result = $this->_insertUpdate( @@ -950,18 +980,35 @@ function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $ ); if ( $result === false ) { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); } if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); + $this->_fail( "Failed to open DFS://$filePath for writing" ); } return true; } - public function _getFileList( $scopes = false, $excludeScopes = false ) + /** + * Gets the list of cluster files, filtered by the optional params + * + * @see eZDFSFileHandler::getFileList + * + * @param array|bool $scopes filter by array of scopes to include in the list + * @param bool $excludeScopes if true, $scopes param acts as an exclude filter + * @param array|bool $limit limits the search to offset limit[0], limit limit[1] + * @param string|bool $path filter to include entries only including $path + * + * @return array|false the db list of entries of false if none found + */ + public function _getFileList( + $scopes = false, + $excludeScopes = false, + $limit = false, + $path = false + ) { $filePathList = array(); $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); @@ -1016,13 +1063,16 @@ protected function _die( $msg, $sql = null ) } /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ + * Performs an insert of the given items in $array. + * + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function + * + * @return bool + */ function _insert( $table, $array, $fname ) { $keys = array_keys( $array ); @@ -1030,28 +1080,30 @@ function _insert( $table, $array, $fname ) $res = $this->_query( $query, $fname ); if ( !$res ) { - // @todo Throw an exception return false; } + return true; } /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param array $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function + * @param bool $reportError + * + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + */ protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) { if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + $this->_fail( "Insert array must contain both name and name_hash" ); } if ( $row = $this->_fetchMetadata( $insert['name'] ) ) @@ -1078,12 +1130,15 @@ protected function _insertUpdate( $table, $insert, $update, $fname, $reportError "VALUES( " . implode( ', ', $quotedValues ) . ")"; } - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { + try + { + $this->_query( $sql, $fname, $reportError ); + } + catch ( PDOException $e ) + { $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; } + return true; } /** @@ -1107,18 +1162,18 @@ protected function _sqlList( $array ) } /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param bool|string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) { return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); @@ -1133,7 +1188,7 @@ protected function _selectOneRow( $query, $fname, $error = false, $debug = false * @param string $query * @param string $fname The function name that started the query, should * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors + * @param bool|string $error Sent to _error() in case of errors * @param bool $debug If true it will display the fetched row in addition * to the SQL. * @return array|false @@ -1144,19 +1199,21 @@ protected function _selectOneAssoc( $query, $fname, $error = false, $debug = fal } /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param bool|string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param int $fetchCall The callback to fetch the row. + * + * @return mixed + **/ protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgresSQL Cluster', 'DB queries' ); $time = microtime( true ); $stmt = $this->db->query( $query ); @@ -1197,12 +1254,8 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ * Starts a new transaction by executing a BEGIN call. * If a transaction is already started nothing is executed. **/ - protected function _begin( $fname = false ) + protected function _begin() { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; $this->transactionCount++; if ( $this->transactionCount == 1 ) $this->db->beginTransaction(); @@ -1212,12 +1265,8 @@ protected function _begin( $fname = false ) * Stops a current transaction and commits the changes by executing a COMMIT call. * If the current transaction is a sub-transaction nothing is executed. **/ - protected function _commit( $fname = false ) + protected function _commit() { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; $this->transactionCount--; if ( $this->transactionCount == 0 ) $this->db->commit(); @@ -1228,12 +1277,8 @@ protected function _commit( $fname = false ) * ROLLBACK call. * If the current transaction is a sub-transaction nothing is executed. **/ - protected function _rollback( $fname = false ) + protected function _rollback() { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; $this->transactionCount--; if ( $this->transactionCount == 0 ) $this->db->rollBack(); @@ -1253,6 +1298,7 @@ protected function _rollback( $fname = false ) **/ protected function _protect() { + $result = false; $args = func_get_args(); $callback = array_shift( $args ); $fname = array_shift( $args ); @@ -1268,34 +1314,14 @@ protected function _protect() } catch( PDOException $e ) { - print_r( compact( 'callback', 'args' ) ); eZDebug::writeError( $e ); return false; } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) + catch( Exception $e ) { - $this->_rollback( $fname ); + eZDebug::writeError( $e ); return false; } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - break; // All is good, so break out of loop } @@ -1303,39 +1329,13 @@ protected function _protect() return $result; } - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ + * Creates an error object which can be read by some backend functions. + * + * @param mixed $message The value which is sent to the debug system. + * @param PDOStatement|bool $result The failed SQL result + * @throws Exception + **/ protected function _fail( $message, $result = false) { // @todo Investigate the right function @@ -1352,14 +1352,16 @@ protected function _fail( $message, $result = false) } /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * + * @param string $query + * @param bool|string $fname Optional caller name for debugging + * @param bool $reportError + * + * @return PDOStatement The resulting PDOStatement object, or false if an error occurred + **/ protected function _query( $query, $fname = false, $reportError = true ) { eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); @@ -1384,7 +1386,7 @@ protected function _query( $query, $fname = false, $reportError = true ) } /** - * Make sure that $value is escaped and qouted according to type and returned + * Make sure that $value is escaped and quoted according to type and returned * as a string. * * @param string $value a SQL parameter to escape @@ -1407,7 +1409,10 @@ protected function _quote( $value ) /** * Provides the SQL calls to convert $value to MD5 * The returned value can directly be put into SQLs. - **/ + * @param $value + * + * @return string + */ protected function _md5( $value ) { return $this->_quote( md5( $value ) ); @@ -1417,7 +1422,7 @@ protected function _md5( $value ) * Prints error message $error to debug system. * @param string $query The query that was attempted, will be printed if * $error is \c false - * @param resource $res The result resource the error occured on + * @param PDOStatement|resource $res The result resource the error occurred on * @param string $fname The function name that started the query, should * contain relevant arguments in the text. * @param string $error The error message, if this is an array the first @@ -1438,16 +1443,18 @@ protected function _error( $query, $res, $fname, $error = "Failed to execute SQL } // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ) . ' ' .$query, $fname ); } /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ + * Report SQL $query to debug system. + * + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int|bool $numRows Number of affected rows. + **/ function _report( $query, $fname, $timeTaken, $numRows = false ) { if ( !self::$dbparams['sql_output'] ) @@ -1464,19 +1471,23 @@ function _report( $query, $fname, $timeTaken, $numRows = false ) } /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * + * @see eZDFSFileHandler::startCacheGeneration + * + * @param string $filePath + * @param string $generatingFilePath + * + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + */ public function _startCacheGeneration( $filePath, $generatingFilePath ) { $fname = "_startCacheGeneration( {$filePath} )"; @@ -1494,12 +1505,15 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . "VALUES(" . implode( ', ', $insertData ) . ")"; - //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch @todo //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; - try { + try + { $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { + } + catch( PDOException $e ) + { $errno = $e->getCode(); if ( $errno != self::ERROR_UNIQUE_VIOLATION ) { @@ -1508,7 +1522,7 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) else { - // generation timout check + // generation timeout check $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; $row = $this->_selectOneRow( $query, $fname, false, false ); @@ -1521,7 +1535,7 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) { $previousMTime = $row[0]; - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timed out, taking over", __METHOD__ ); $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; // we run the query manually since the default _query won't @@ -1533,8 +1547,8 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) } else { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); + throw new RuntimeException( "An error occurred taking over timed out generating cache file $generatingFilePath" ); + //return array( 'result' => 'error' ); } } else @@ -1548,12 +1562,19 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) } /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * + * @see eZDFSFileHandler::endCacheGeneration + * + * @param string $filePath + * @param string $generatingFilePath + * @param bool $rename if false the .generating entry is just deleted + * + * @return bool true + * + * @throw RuntimeException + */ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) { $fname = "_endCacheGeneration( $filePath )"; @@ -1575,7 +1596,7 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + throw new RuntimeException( "An error occurred getting a lock on $generatingFilePath" ); } $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); @@ -1592,14 +1613,14 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$this->_query( $insertSQL, $fname, true ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + throw new RuntimeException( "An error occurred creating the metadata entry for $filePath" ); } // here we rename the actual FILE. The .generating file has been // created on DFS, and should be renamed if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) { $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + throw new RuntimeException("An error occurred renaming DFS://$generatingFilePath to DFS://$filePath" ); } $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } @@ -1610,7 +1631,7 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + throw new RuntimeException( "An error occurred renaming DFS://$generatingFilePath to DFS://$filePath" ); } $mtime = $generatingMetaData['mtime']; @@ -1673,11 +1694,10 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; $stmt = $this->db->query( $query ); $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + if ( isset( $row[0] ) && $row[0] == $generatingFileMtime ) { return true; } - // @todo Check if an exception makes sense here return false; } @@ -1694,11 +1714,14 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi } /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ + * Aborts the cache generation process by removing the .generating file + * + * @see eZDFSFileHandler::abortCacheGeneration + * + * @param string $generatingFilePath .generating cache file path + * + * @return void + */ public function _abortCacheGeneration( $generatingFilePath ) { $fname = "_abortCacheGeneration( $generatingFilePath )"; @@ -1752,13 +1775,13 @@ static protected function nameTrunk( $filePath, $scope ) } /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param array $row + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ protected function remainingCacheGenerationTime( $row ) { if( !isset( $row[0] ) ) @@ -1770,14 +1793,17 @@ protected function remainingCacheGenerationTime( $row ) /** * Returns the list of expired files * + * @see eZDFSFileHandler::fetchExpiredItems + * * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. + * @param array|bool $limit Max number of items. Set to false for unlimited. + * @param int|bool $expiry Number of seconds, only items older than this will be returned. * * @return array(filepath) * * @since 4.3 */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = false ) { $tables = array( $this->metaDataTable, $this->metaDataTableCache ); @@ -1804,7 +1830,7 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) return $filePathList; } - + public function applyServerUri( $filePath ) { return $this->dfsbackend->applyServerUri( $filePath ); @@ -1830,7 +1856,7 @@ public function deleteCacheFiles( $limit ) $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) + if ( !$stmt = $this->_query( $query ) ) { throw new RuntimeException( "Error in $query" ); } @@ -1842,7 +1868,7 @@ public function deleteCacheFiles( $limit ) /** * DB connexion handle - * @var PDO + * @var PDO|resource */ public $db = null; @@ -1861,7 +1887,7 @@ public function deleteCacheFiles( $limit ) /** * Current transaction level. * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction + * or COMMIT (if we're committing the last running transaction * @var int */ protected $transactionCount = 0; From fb32d5941a33c89fe118e0319acc2f22cc644896 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 23:09:19 +0200 Subject: [PATCH 04/21] fix phpdoc --- clustering/dfs/ezpostsgresqlbackend.php | 76 +++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index e1abc77..1ee141d 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -12,7 +12,7 @@ * This class allows DFS based clustering using PostgresSQL * @package Cluster */ -class eZDFSFileHandlerPostgresqlBackend implements eZDFSFileHandlerDBBackendInterface +class eZDFSFileHandlerPostgresqlBackend { /** @@ -143,6 +143,9 @@ public function _connect() /** * Disconnects the handler from the database + * + * @see eZDFSFileHandler::disconnect + * @return void */ public function _disconnect() { @@ -233,16 +236,18 @@ protected function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) /** * Purges meta-data and file-data for a file entry - * * Will only expire a single file. Use _purgeByLike to purge multiple files * + * @see eZDFSFileHandler::purge + * @see _purgeByLike + * * @param string $filePath Path of the file to purge * @param bool $onlyExpired Only purges expired files * @param bool|int $expiry - * @param bool $fname + * @param bool|string $fname Optional caller name for debugging * - * @see _purgeByLike - **/ + * @return bool + */ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) { if ( $fname ) @@ -537,6 +542,19 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) return true; } + /** + * Deletes a list of files based on directory / filename components + * + * @see eZDFSFileHandler::fileDeleteByDirList + * + * @param array $dirList Array of directory that will be prefixed with + * $commonPath when looking for files + * @param string $commonPath Starting path common to every delete request + * @param string $commonSuffix Suffix appended to every delete request + * @param bool|string $fname Optional caller name for debugging + * + * @return bool + */ public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) { if ( $fname ) @@ -568,6 +586,19 @@ protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, return true; } + /** + * Check if given file/dir exists. + * + * @see eZDFSFileHandler::fileExists + * @see eZDFSFileHandler::exists + * + * @param $filePath + * @param bool|string $fname Optional caller name for debugging + * @param bool $ignoreExpiredFiles ignore ezdfsfile.mtime + * @param bool $checkOnDFS Checks if a file exists on the DFS + * + * @return bool + */ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) { if ( $fname ) @@ -704,6 +735,17 @@ public function _fetch( $filePath, $uniqueName = false ) return false; } + /** + * Returns file contents. + * + * @see eZDFSFileHandler::fileFetchContents + * @see eZDFSFileHandler::fetchContents + * + * @param string $filePath + * @param bool|string $fname Optional caller name for debugging + * + * @return string|bool contents string, or false in case of an error. + */ public function _fetchContents( $filePath, $fname = false ) { if ( $fname ) @@ -747,6 +789,17 @@ function _fetchMetadata( $filePath, $fname = false ) true ); } + /** + * Create symbolic or hard link to file. Alias of copy + * + * @see eZDFSFileHandler::fileLinkCopy + * + * @param string $srcPath Source file + * @param string $dstPath Destination file + * @param bool|string $fname Optional caller name for debugging + * + * @return mixed + */ public function _linkCopy( $srcPath, $dstPath, $fname = false ) { if ( $fname ) @@ -787,8 +840,12 @@ public function _passThrough( $filePath, $startOffset = 0, $length = false, $fna /** * Renames $srcFilePath to $dstFilePath * + * @see eZDFSFileHandler::fileMove + * @see eZDFSFileHandler::move + * * @param string $srcFilePath * @param string $dstFilePath + * * @return bool */ public function _rename( $srcFilePath, $dstFilePath ) @@ -1831,6 +1888,15 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = f return $filePathList; } + /** + * Transforms $filePath so that it contains a valid href to the file, wherever it is stored. + * + * @see eZDFSFileHandler::applyServerUri + * + * @param string $filePath + * + * @return string + */ public function applyServerUri( $filePath ) { return $this->dfsbackend->applyServerUri( $filePath ); From 554fa0bc298afbf65e8d5cbd299ddbfe2ddc3093 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 23:30:50 +0200 Subject: [PATCH 05/21] minor bugfixes --- clustering/dfs/ezpostsgresqlbackend.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 1ee141d..392dda8 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -558,9 +558,9 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) { if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + $fname .= "::_deleteByDirList(" . implode( ' ',$dirList ) . ", $commonPath, $commonSuffix)"; else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + $fname = "_deleteByDirList(" . implode( ' ',$dirList ) . ", $commonPath, $commonSuffix)"; return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, $dirList, $commonPath, $commonSuffix, $fname ); } @@ -1270,7 +1270,7 @@ protected function _selectOneAssoc( $query, $fname, $error = false, $debug = fal **/ protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgresSQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); $time = microtime( true ); $stmt = $this->db->query( $query ); From 0b59d15176b80e419c45e386e60add80295e9a91 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Sun, 5 Jul 2015 17:36:23 +0200 Subject: [PATCH 06/21] Fix md5 calls --- clustering/dfs/ezpostsgresqlbackend.php | 46 +- .../ezpostsgresqlbackend.php.MD5_incapsulato | 1921 ++++++++++++++++ ...gresqlbackend.php.MD5_incapsulato_corretto | 1921 ++++++++++++++++ .../dfs/ezpostsgresqlbackend.php.originale | 1910 ++++++++++++++++ ...esqlbackend.php_ante_modifiche_luca_22_mag | 1924 +++++++++++++++++ sql/postgresql/cluster_dfs_schema.sql | 1 + 6 files changed, 7707 insertions(+), 16 deletions(-) create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto create mode 100644 clustering/dfs/ezpostsgresqlbackend.php.originale create mode 100644 clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index e89d353..6be86e5 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -784,7 +784,8 @@ public function _rename( $srcFilePath, $dstFilePath ) $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); // Mark entry for update to lock it - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { // @todo Throw an exception @@ -799,9 +800,11 @@ public function _rename( $srcFilePath, $dstFilePath ) // Create a new meta-data entry for the new file to make foreign keys happy. $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - "WHERE name_hash=MD5($srcFilePathStr)"; + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); @@ -816,7 +819,8 @@ public function _rename( $srcFilePath, $dstFilePath ) } // Remove old entry - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); @@ -1162,7 +1166,7 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ $stmt = $this->db->query( $query ); if ( !$stmt ) { - $this->_error( $query, $fname, $error ); + $this->_error( $query, $stmt, $fname, $error ); eZDebug::accumulatorStop( 'postgresql_cluster_query' ); // @todo Throw an exception return false; @@ -1410,7 +1414,8 @@ protected function _quote( $value ) **/ protected function _md5( $value ) { - return "MD5(" . $this->_quote( $value ) . ")"; + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); } /** @@ -1494,6 +1499,9 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . "VALUES(" . implode( ', ', $insertData ) . ")"; + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; + try { $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); } catch( PDOException $e ) { @@ -1558,7 +1566,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) // no rename: the .generating entry is just deleted if ( $rename === false ) { - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); $this->dfsbackend->delete( $generatingFilePath ); return true; } @@ -1569,7 +1578,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_begin( $fname ); // both files are locked for update - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); @@ -1577,7 +1587,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); // the original file does not exist: we move the generating file - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); if ( $stmt->rowCount() == 0 ) { $metaData = $generatingMetaData; @@ -1598,7 +1609,8 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_rollback( $fname ); throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } // the original file exists: we move the generating data to this file // and update it @@ -1612,12 +1624,14 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $mtime = $generatingMetaData['mtime']; $filesize = $generatingMetaData['size']; - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } $this->_commit( $fname ); @@ -1652,7 +1666,7 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi if ( !$stmt ) { // @todo Throw an exception - $this->_error( $query, $fname ); + $this->_error( $query, $stmt, $fname ); return false; } $numRows = $stmt->rowCount(); @@ -1778,7 +1792,7 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) { $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - if ( count( $scopes ) == 0 ) + if ( count( $scopes ) == 0 || $scopes == false ) throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); $scopeString = $this->_sqlList( $scopes ); @@ -1839,7 +1853,7 @@ public function deleteCacheFiles( $limit ) /** * DB connexion handle - * @var handle + * @var PDO */ public $db = null; @@ -1907,4 +1921,4 @@ public function deleteCacheFiles( $limit ) * @var int */ const ERROR_UNIQUE_VIOLATION = 23505; -} \ No newline at end of file +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato new file mode 100644 index 0000000..577a717 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato @@ -0,0 +1,1921 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto new file mode 100644 index 0000000..73927bd --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto @@ -0,0 +1,1921 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePathStr ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePathStr ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this-> md5( $value ) ; + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.originale b/clustering/dfs/ezpostsgresqlbackend.php.originale new file mode 100644 index 0000000..e89d353 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php.originale @@ -0,0 +1,1910 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + "WHERE name_hash=MD5($srcFilePathStr)"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + return "MD5(" . $this->_quote( $value ) . ")"; + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} \ No newline at end of file diff --git a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag new file mode 100644 index 0000000..feaf076 --- /dev/null +++ b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag @@ -0,0 +1,1924 @@ +eventHandler = ezpEvent::getInstance(); + $fileINI = eZINI::instance( 'file.ini' ); + $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); + + if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) + { + $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; + } + else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) + { + $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); + } + + $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); + $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); + } + + /** + * Returns the database table name to use for the specified file. + * + * For files detected as cache files the cache table is returned, if not + * the generic table is returned. + * + * @param string $filePath + * @return string The database table name + */ + protected function dbTable( $filePath ) + { + if ( $this->metaDataTableCache == $this->metaDataTable ) + return $this->metaDataTable; + + if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + { + return $this->metaDataTableCache; + } + + return $this->metaDataTable; + } + + /** + * Connects to the database. + * + * @return void + * @throw eZClusterHandlerDBNoConnectionException + * @throw eZClusterHandlerDBNoDatabaseException + **/ + public function _connect() + { + $siteINI = eZINI::instance( 'site.ini' ); + // DB Connection setup + // This part is not actually required since _connect will only be called + // once, but it is useful to run the unit tests. So be it. + // @todo refactor this using eZINI::setVariable in unit tests + if ( self::$dbparams === null ) + { + $fileINI = eZINI::instance( 'file.ini' ); + + self::$dbparams = array(); + self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); + self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); + self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); + self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); + self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); + self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); + + self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); + self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); + + self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; + + self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); + } + + + $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', + self::$dbparams['host'], + self::$dbparams['port'], + self::$dbparams['dbname'], + self::$dbparams['user'], + self::$dbparams['pass'] + ); + $tries = 0; + while ( $tries < self::$dbparams['max_connect_tries'] ) + { + try { + $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } catch ( PDOException $e ) { + eZDebug::writeError( $e->getMessage() ); + ++$tries; + continue; + } + break; + } + if ( !( $this->db instanceof PDO ) ) + { + $this->_die( "Unable to connect to storage server" ); + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + } + + + if ( !$this->db ) + throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); + + $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $this->db->exec( "SET NAMES 'utf8'" ); + + + // DFS setup + if ( $this->dfsbackend === null ) + $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } + + /** + * Disconnects the handler from the database + */ + public function _disconnect() + { + if ( $this->db !== null ) + { + $this->db = null; + } + } + + /** + * Creates a copy of a file in DB+DFS + * @param string $srcFilePath Source file + * @param string $dstFilePath Destination file + * @param string $fname + * @return bool + * + * @see _copyInner + **/ + public function _copy( $srcFilePath, $dstFilePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_copy($srcFilePath, $dstFilePath)"; + else + $fname = "_copy($srcFilePath, $dstFilePath)"; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); + // if source file does not exist then do nothing. + // @todo Throw an exception here. + // Info: $srcFilePath + if ( !$metaData ) + { + return false; + } + return $this->_protect( array( $this, "_copyInner" ), $fname, + $srcFilePath, $dstFilePath, $fname, $metaData ); + } + + /** + * Inner function used by _copy to perform the operation in a transaction + * + * @param string $srcFilePath + * @param string $dstFilePath + * @param bool $fname + * @param array $metaData Source file's metadata + * @return bool + * + * @see _copy + */ + private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + { + $this->_delete( $dstFilePath, true, $fname ); + + $datatype = $metaData['datatype']; + $filePathHash = md5( $dstFilePath ); + $scope = $metaData['scope']; + $contentLength = $metaData['size']; + $fileMTime = $metaData['mtime']; + $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); + + // Copy file metadata. + if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + } + + // Copy file data. + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + return true; + } + + /** + * Purges meta-data and file-data for a file entry + * + * Will only expire a single file. Use _purgeByLike to purge multiple files + * + * @param string $filePath Path of the file to purge + * @param bool $onlyExpired Only purges expired files + * @param bool|int $expiry + * @param bool $fname + * + * @see _purgeByLike + **/ + public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purge($filePath)"; + else + $fname = "_purge($filePath)"; + $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + if ( $expiry !== false ) + { + $sql .= " AND mtime<" . (int)$expiry; + } + elseif ( $onlyExpired ) + { + $sql .= " AND expired=1"; + } + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Purging file metadata for $filePath failed" ); + } + if ( $stmt->rowCount() == 1 ) + { + $this->dfsbackend->delete( $filePath ); + } + return true; + } + + /** + * Purges meta-data and file-data for files matching a pattern using a SQL + * LIKE syntax. + * + * @param string $like + * SQL LIKE string applied to ezdfsfile.name to look for files to + * purge + * @param bool $onlyExpired + * Only purge expired files (ezdfsfile.expired = 1) + * @param integer $limit Maximum number of items to purge in one call + * @param integer $expiry + * Timestamp used to limit deleted files: only files older than this + * date will be deleted + * @param mixed $fname Optional caller name for debugging + * @see _purge + * @return bool|int false if it fails, number of affected rows otherwise + * @todo This method should also remove the files from disk + */ + public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_purgeByLike($like, $onlyExpired)"; + else + $fname = "_purgeByLike($like, $onlyExpired)"; + + // common query part used for both DELETE and SELECT + $where = " WHERE name LIKE " . $this->_quote( $like ); + + if ( $expiry !== false ) + $where .= " AND mtime < " . (int)$expiry; + elseif ( $onlyExpired ) + $where .= " AND expired = 1"; + + if ( $limit ) + $sqlLimit = " LIMIT $limit"; + else + $sqlLimit = ""; + + $this->_begin( $fname ); + + // select query, in FOR UPDATE mode + $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . + "{$where} {$sqlLimit} FOR UPDATE"; + if ( !$stmt = $this->_query( $selectSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Selecting file metadata by like statement $like failed" ); + } + + // if there are no results, we can just return 0 and stop right here + if ( $stmt->rowCount() == 0 ) + { + $this->_rollback( $fname ); + return 0; + } + // the candidate for purge are indexed in an array + else + { + while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) + { + $files[] = $row['name']; + } + } + + // delete query + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; + if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) + { + $this->_rollback( $fname ); + return $this->_fail( "Purging file metadata by like statement $like failed" ); + } + $deletedDBFiles = $stmt->rowCount(); + $this->dfsbackend->delete( $files ); + + $this->_commit( $fname ); + + return $deletedDBFiles; + } + + /** + * Deletes a file from DB + * + * The file won't be removed from disk, _purge has to be used for this. + * Only single files will be deleted, to delete multiple files, + * _deleteByLike has to be used. + * + * @param string $filePath Path of the file to delete + * @param bool $insideOfTransaction + * Wether or not a transaction is already started + * @param bool|string $fname Optional caller name for debugging + * @see _deleteInner + * @see _deleteByLike + * @return bool + */ + public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_delete($filePath)"; + else + $fname = "_delete($filePath)"; + // @todo Check if this is requried: _protec will already take care of + // checking if a transaction is running. But leave it like this + // for now. + if ( $insideOfTransaction ) + { + $res = $this->_deleteInner( $filePath, $fname ); + if ( !$res || $res instanceof eZMySQLBackendError ) + { + $this->_handleErrorType( $res ); + } + } + else + { + return $this->_protect( array( $this, '_deleteInner' ), $fname, + $filePath, $insideOfTransaction, $fname ); + } + } + + /** + * Callback method used by by _delete to delete a single file + * + * @param string $filePath Path of the file to delete + * @param string $fname Optional caller name for debugging + * @return bool + **/ + protected function _deleteInner( $filePath, $fname ) + { + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) + return $this->_fail( "Deleting file $filePath failed" ); + return true; + } + + /** + * Deletes multiple files using a SQL LIKE statement + * + * Use _delete if you need to delete single files + * + * @param string $like + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param string $fname Optional caller name for debugging + * @return bool + * @see _deleteByLikeInner + * @see _delete + */ + public function _deleteByLike( $like, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByLike($like)"; + else + $fname = "_deleteByLike($like)"; + return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, + $like, $fname ); + } + + /** + * Callback used by _deleteByLike to perform the deletion + * + * @param string $like + * @param mixed $fname + * @return + */ + private function _deleteByLikeInner( $like, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by like: '$like'" ); + } + return true; + } + + /** + * Deletes DB files by using a SQL regular expression applied to file names + * + * @param string $regex + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByRegex( $regex, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByRegex($regex)"; + else + $fname = "_deleteByRegex($regex)"; + return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, + $regex, $fname ); + } + + /** + * Callback used by _deleteByRegex to perform the deletion + * + * @param mixed $regex + * @param mixed $fname + * @return + * @deprecated Has severe performances issues + */ + public function _deleteByRegexInner( $regex, $fname ) + { + $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by regex: '$regex'" ); + } + return true; + } + + /** + * Deletes multiple DB files by wildcard + * + * @param string $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + public function _deleteByWildcard( $wildcard, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByWildcard($wildcard)"; + else + $fname = "_deleteByWildcard($wildcard)"; + return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, + $wildcard, $fname ); + } + + /** + * Callback used by _deleteByWildcard to perform the deletion + * + * @param mixed $wildcard + * @param mixed $fname + * @return bool + * @deprecated Has severe performance issues + */ + protected function _deleteByWildcardInner( $wildcard, $fname ) + { + // Convert wildcard to regexp. + $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; + + $regex = str_replace( array( '.' ), + array( '\.' ), + $regex ); + + $regex = str_replace( array( '?', '*', '{', '}', ',' ), + array( '.', '.*', '(', ')', '|' ), + $regex ); + + $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; + if ( !$res = $this->_query( $sql, $fname ) ) + { + return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + } + return true; + } + + public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) + { + if ( $fname ) + $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + else + $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, + $dirList, $commonPath, $commonSuffix, $fname ); + } + + protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) + { + foreach ( $dirList as $dirItem ) + { + if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) + { + $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; + } + else + { + $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; + } + $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; + if ( !$stmt = $this->_query( $sql, $fname ) ) + { + eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + } + } + return true; + } + + public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) + { + if ( $fname ) + $fname .= "::_exists($filePath)"; + else + $fname = "_exists($filePath)"; + $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), + $fname, "Failed to check file '$filePath' existance: ", true ); + if ( $row === false ) + return false; + + if ( $ignoreExpiredFiles ) + $rc = $row[1] >= 0; + else + $rc = true; + + if ( $checkOnDFS && $rc ) + { + $rc = $this->dfsbackend->existsOnDFS( $filePath ); + } + + return $rc; + } + + protected function __mkdir_p( $dir ) + { + // create parent directories + $dirElements = explode( '/', $dir ); + if ( count( $dirElements ) == 0 ) + return true; + + $result = true; + $currentDir = $dirElements[0]; + + if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + for ( $i = 1; $i < count( $dirElements ); ++$i ) + { + $dirElement = $dirElements[$i]; + if ( strlen( $dirElement ) == 0 ) + continue; + + $currentDir .= '/' . $dirElement; + + if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) + return false; + + $result = true; + } + + return $result; + } + + /** + * Fetches the file $filePath from the database to its own name + * + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @param string $filePath + * @param string $uniqueName Alternative name to save the file to + * @return string|bool the file physical path, or false if fetch failed + **/ + public function _fetch( $filePath, $uniqueName = false ) + { + $metaData = $this->_fetchMetadata( $filePath ); + if ( !$metaData ) + { + // @todo Throw an exception + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + return false; + } + + $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + $loopCount = 0; + $localFileSize = 0; + + do + { + // create temporary file + $tmpid = getmypid() . '-' . mt_rand() .'tmp'; + if ( strrpos( $filePath, '.' ) > 0 ) + $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); + else + $tmpFilePath = $filePath . '.' . $tmpid; + $this->__mkdir_p( dirname( $tmpFilePath ) ); + eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); + + // copy DFS file to temporary FS path + // @todo Throw an exception + if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) + { + eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + + if ( $uniqueName !== true ) + { + if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) + { + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + continue; + } + } + $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; + + // If all data has been written correctly, return the filepath. + // Otherwise let the loop continue + clearstatcache( true, $filePath ); + $localFileSize = filesize( $filePath ); + if ( $dfsFileSize == $localFileSize ) + { + return $filePath; + } + // Sizes might have been corrupted by FS problems. Enforcing temp file removal. + else if ( file_exists( $tmpFilePath ) ) + { + unlink( $tmpFilePath ); + } + + usleep( self::TIME_UNTIL_RETRY ); + ++$loopCount; + } + while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); + + // Copy from DFS has failed :-( + eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + return false; + } + + public function _fetchContents( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchContents($filePath)"; + else + $fname = "_fetchContents($filePath)"; + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + { + eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + return false; + } + + // @todo Catch an exception + if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) + { + eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + return false; + } + return $contents; + } + + /** + * Fetches and returns metadata for $filePath + * @return array|false file metadata, or false if the file does not exist in + * database. + */ + function _fetchMetadata( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_fetchMetadata($filePath)"; + else + $fname = "_fetchMetadata($filePath)"; + $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); + return $this->_selectOneAssoc( $sql, $fname, + "Failed to retrieve file metadata: $filePath", + true ); + } + + public function _linkCopy( $srcPath, $dstPath, $fname = false ) + { + if ( $fname ) + $fname .= "::_linkCopy($srcPath,$dstPath)"; + else + $fname = "_linkCopy($srcPath,$dstPath)"; + return $this->_copy( $srcPath, $dstPath, $fname ); + } + + /** + * Passes $filePath content through + * @param string $filePath + * @deprecated should not be used since it cannot handle reading errors + **/ + public function _passThrough( $filePath, $fname = false ) + { + if ( $fname ) + $fname .= "::_passThrough($filePath)"; + else + $fname = "_passThrough($filePath)"; + + $metaData = $this->_fetchMetadata( $filePath, $fname ); + // @todo Throw an exception + if ( !$metaData ) + return false; + + // @todo Catch an exception + $this->dfsbackend->passthrough( $filePath ); + + return true; + } + + /** + * Renames $srcFilePath to $dstFilePath + * + * @param string $srcFilePath + * @param string $dstFilePath + * @return bool + */ + public function _rename( $srcFilePath, $dstFilePath ) + { + if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) + return; + + // fetch source file metadata + $metaData = $this->_fetchMetadata( $srcFilePath ); + // if source file does not exist then do nothing. + // @todo Throw an exception + if ( !$metaData ) + return false; + + $this->_begin( __METHOD__ ); + + $srcFilePathStr = $this->_quote( $srcFilePath ); + $dstFilePathStr = $this->_quote( $dstFilePath ); + $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); + + // Mark entry for update to lock it + //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; + $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + // @todo Throw an exception + eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + return false; + } + + if ( $this->_exists( $dstFilePath, false, false ) ) + $this->_purge( $dstFilePath, false ); + + // Create a new meta-data entry for the new file to make foreign keys happy. + $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". + "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . + //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . + "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . + "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . + //"WHERE name_hash=MD5($srcFilePathStr)"; + "WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) + { + return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + } + + // Remove old entry + //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; + $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); + if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) + { + eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + $this->_rollback( __METHOD__ ); + // @todo Throw an exception + return false; + } + + // delete original DFS file + // @todo Catch an exception + $this->dfsbackend->delete( $srcFilePath ); + + $this->_commit( __METHOD__ ); + + return true; + } + + /** + * Stores $filePath to cluster + * + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @return void + */ + function _store( $filePath, $datatype, $scope, $fname = false ) + { + if ( !is_readable( $filePath ) ) + { + eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + return; + } + if ( $fname ) + $fname .= "::_store($filePath, $datatype, $scope)"; + else + $fname = "_store($filePath, $datatype, $scope)"; + + $return = $this->_protect( array( $this, '_storeInner' ), $fname, + $filePath, $datatype, $scope, $fname ); + return $return; + } + + /** + * Callback function used to perform the actual file store operation + * @param string $filePath + * @param string $datatype + * @param string $scope + * @param string $fname + * @see eZDFSFileHandlerMySQLBackend::_store() + * @return bool + **/ + function _storeInner( $filePath, $datatype, $scope, $fname ) + { + // Insert file metadata. + clearstatcache(); + $fileMTime = filemtime( $filePath ); + $contentLength = filesize( $filePath ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + + if ( $this->_insertUpdate( $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) + { + return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + } + + // copy given $filePath to DFS + if ( !$this->dfsbackend->copyToDFS( $filePath ) ) + { + return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + } + + return true; + } + + /** + * Stores $contents as the contents of $filePath to the cluster + * + * @param string $filePath + * @param string $contents + * @param string $scope + * @param string $datatype + * @param int $mtime + * @param string $fname + * @return void + */ + function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) + { + if ( $fname ) + $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; + else + $fname = "_storeContents($filePath, ..., $scope, $datatype)"; + + return $this->_protect( array( $this, '_storeContentsInner' ), $fname, + $filePath, $contents, $scope, $datatype, $mtime, $fname ); + } + + function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) + { + // File metadata. + $contentLength = strlen( $contents ); + $filePathHash = md5( $filePath ); + $nameTrunk = self::nameTrunk( $filePath, $scope ); + if ( $mtime === false ) + $mtime = time(); + $expired = ( $mtime < 0 ) ? '1' : '0'; + + // Copy file metadata. + $result = $this->_insertUpdate( + $this->dbTable( $filePath ), + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $mtime, + 'expired' => ( $mtime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname + ); + if ( $result === false ) + { + return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + } + + if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) + { + return $this->_fail( "Failed to open DFS://$filePath for writing" ); + } + + return true; + } + + public function _getFileList( $scopes = false, $excludeScopes = false ) + { + $filePathList = array(); + $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); + + foreach ( $tables as $table ) + { + $query = 'SELECT name FROM ' . $table; + + if ( is_array( $scopes ) && count( $scopes ) > 0 ) + { + $query .= ' WHERE scope '; + if ( $excludeScopes ) + $query .= 'NOT '; + $query .= "IN ('" . implode( "', '", $scopes ) . "')"; + } + + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); + if ( !$stmt ) + { + eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); + // @todo Throw an exception + return false; + } + + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + + unset( $stmt ); + } + return $filePathList; + } + + /** + * Handles a DB error, displaying it as an eZDebug error + * @see eZDebug::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ + protected function _die( $msg, $sql = null ) + { + if ( $this->db ) + { + $error = $this->db->errorInfo(); + eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + } + else + { + eZDebug::writeError( $sql, $msg ); + } + } + + /** + * Performs an insert of the given items in $array. + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function (for logging purpuse) + **/ + function _insert( $table, $array, $fname ) + { + $keys = array_keys( $array ); + $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; + $res = $this->_query( $query, $fname ); + if ( !$res ) + { + // @todo Throw an exception + return false; + } + } + + /** + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param string $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function (for logging purpuse) + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + **/ + protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) + { + if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) + { + throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + } + + if ( $row = $this->_fetchMetadata( $insert['name'] ) ) + { + $sql = "UPDATE $table SET "; + $setEntries = array(); + foreach( $update as $field ) + { + $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); + } + $sql .= implode( ', ', $setEntries ) . + " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); + } + else + { + // create file in db + $quotedValues = array(); + foreach( $insert as $value ) + { + $quotedValues[] = $this->_quote( $value ); + } + $sql = "INSERT INTO $table " . + "(" . implode( ', ', array_keys( $insert ) ) . ") " . + "VALUES( " . implode( ', ', $quotedValues ) . ")"; + } + + try { + $stmt = $this->_query( $sql, $fname, $reportError ); + } catch( PDOException $e ) { + $this->_fail( "Failed insert/updating: " . $e->getMessage() ); + return false; + } + } + + /** + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ + protected function _sqlList( $array ) + { + $text = ""; + $sep = ""; + foreach ( $array as $e ) + { + $text .= $sep; + $text .= $this->_quote( $e ); + $sep = ", "; + } + return $text; + } + + /** + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); + } + + /** + * Runs a select query and returns one associative row from the result. + * + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ + protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) + { + return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); + } + + /** + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param callback $fetchCall The callback to fetch the row. + * @return mixed + **/ + protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + $this->_error( $query, $fname, $error ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo Throw an exception + return false; + } + + $numRows = $stmt->rowCount(); + if ( $numRows > 1 ) + { + eZDebug::writeError( 'Duplicate entries found', $fname ); + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + // @todo throw an exception instead. Should NOT happen. + } + elseif ( $numRows === 0 ) + { + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + return false; + } + + $row = $stmt->fetch( $fetchCall ); + unset( $stmt ); + if ( $debug ) + $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time ); + return $row; + } + + /** + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ + protected function _begin( $fname = false ) + { + if ( $fname ) + $fname .= "::_begin"; + else + $fname = "_begin"; + $this->transactionCount++; + if ( $this->transactionCount == 1 ) + $this->db->beginTransaction(); + } + + /** + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _commit( $fname = false ) + { + if ( $fname ) + $fname .= "::_commit"; + else + $fname = "_commit"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->commit(); + } + + /** + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ + protected function _rollback( $fname = false ) + { + if ( $fname ) + $fname .= "::_rollback"; + else + $fname = "_rollback"; + $this->transactionCount--; + if ( $this->transactionCount == 0 ) + $this->db->rollBack(); + } + + /** + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ + protected function _protect() + { + $args = func_get_args(); + $callback = array_shift( $args ); + $fname = array_shift( $args ); + + $maxTries = self::$dbparams['max_execute_tries']; + $tries = 0; + while ( $tries < $maxTries ) + { + $this->_begin( $fname ); + + try { + $result = call_user_func_array( $callback, $args ); + } + catch( PDOException $e ) + { + print_r( compact( 'callback', 'args' ) ); + eZDebug::writeError( $e ); + return false; + } + + /*// @todo Investigate the right function + $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); + if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) + $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) + { + $tries++; + $this->_rollback( $fname ); + continue; + } + + // @todo replace with an exception + if ( $result === false ) + { + $this->_rollback( $fname ); + return false; + } + elseif ( $result instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $result->errorValue, $result->errorText ); + $this->_rollback( $fname ); + return false; + }*/ + + break; // All is good, so break out of loop + } + + $this->_commit( $fname ); + return $result; + } + + protected function _handleErrorType( $res ) + { + if ( $res === false ) + { + eZDebug::writeError( "SQL failed" ); + } + elseif ( $res instanceof eZMySQLBackendError ) + { + eZDebug::writeError( $res->errorValue, $res->errorText ); + } + } + + /** + * Checks if $result is a failure type and returns true if so, false + * otherwise. + * + * A failure is either the value false or an error object of type + * eZMySQLBackendError. + **/ + protected function _isFailure( $result ) + { + if ( $result === false || ($result instanceof eZMySQLBackendError ) ) + { + return true; + } + return false; + } + + /** + * Creates an error object which can be read by some backend functions. + * @param mixed $value The value which is sent to the debug system. + * @param PDOStatement $result The failed SQL result + **/ + protected function _fail( $message, $result = false) + { + // @todo Investigate the right function + if ( $result !== false ) + { + $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); + } + else + { + $errorInfo = $this->db->errorInfo(); + $message .= "\n$errorInfo[2]"; + } + throw new Exception( $message ); + } + + /** + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @return PDOStatement The resulting PDOStatement object, or false if an error occured + **/ + protected function _query( $query, $fname = false, $reportError = true ) + { + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $stmt = $this->db->query( $query ); + if ( $stmt == false ) + { + if ( $reportError ) + $this->_error( $query, $stmt, $fname ); + return $stmt; + } + + $numRows = $stmt->rowCount(); + + $time = microtime( true ) - $time; + eZDebug::accumulatorStop( 'postgresql_cluster_query' ); + + $this->_report( $query, $fname, $time, $numRows ); + + return $stmt; + } + + /** + * Make sure that $value is escaped and qouted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ + protected function _quote( $value ) + { + if ( $value === null ) + return 'NULL'; + elseif ( is_integer( $value ) ) + { + return $this->db->quote( $value, PDO::PARAM_INT ); + } + else + { + return $this->db->quote( $value, PDO::PARAM_STR ); + } + } + + /** + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. + **/ + protected function _md5( $value ) + { + //return "MD5(" . $this->_quote( $value ) . ")"; + return $this->_quote( md5( $value ) ); + } + + /** + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param resource $res The result resource the error occured on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. + */ + protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) + { + if ( $error === false ) + { + $error = "Failed to execute SQL for function:"; + } + else if ( is_array( $error ) ) + { + $fname = $error[1]; + $error = $error[0]; + } + + // @todo Investigate error methods + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + } + + /** + * Report SQL $query to debug system. + * + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int $numRows Number of affected rows. + **/ + function _report( $query, $fname, $timeTaken, $numRows = false ) + { + if ( !self::$dbparams['sql_output'] ) + return; + + $rowText = ''; + if ( $numRows !== false ) + $rowText = "$numRows rows, "; + static $numQueries = 0; + if ( strlen( $fname ) == 0 ) + $fname = "_query"; + $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); + eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); + } + + /** + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * @param string $filePath + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + * @throws RuntimeException + **/ + public function _startCacheGeneration( $filePath, $generatingFilePath ) + { + $fname = "_startCacheGeneration( {$filePath} )"; + + $nameHash = $this->_md5( $generatingFilePath ); + $mtime = time(); + + $insertData = array( 'name' => $this->_quote( $generatingFilePath ), + 'name_trunk' => $this->_quote( $generatingFilePath ), + 'name_hash' => $nameHash, + 'scope' => "''", + 'datatype' => "''", + 'mtime' => $this->_quote( $mtime ), + 'expired' => 0 ); + $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . + "VALUES(" . implode( ', ', $insertData ) . ")"; + + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; + + try { + $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); + } catch( PDOException $e ) { + $errno = $e->getCode(); + if ( $errno != self::ERROR_UNIQUE_VIOLATION ) + { + throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); + } + // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) + else + { + // generation timout check + $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; + $row = $this->_selectOneRow( $query, $fname, false, false ); + + // file has been renamed, i.e it is no longer a .generating file + if( $row and !isset( $row[0] ) ) + return array( 'result' => 'ok', 'mtime' => $mtime ); + + $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); + if ( $remainingGenerationTime < 0 ) + { + $previousMTime = $row[0]; + + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; + + // we run the query manually since the default _query won't + // report affected rows + $stmt = $this->db->query( $updateQuery ); + if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) + { + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + else + { + throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); + return array( 'result' => 'error' ); + } + } + else + { + return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); + } + } + } + + return array( 'result' => 'ok', 'mtime' => $mtime ); + } + + /** + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * @param string $filePath + * @return bool + * @throws RuntimeException + **/ + public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) + { + $fname = "_endCacheGeneration( $filePath )"; + + // no rename: the .generating entry is just deleted + if ( $rename === false ) + { + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + $this->dfsbackend->delete( $generatingFilePath ); + return true; + } + // rename mode: the generating file and its contents are renamed to the + // final name + else + { + $this->_begin( $fname ); + + // both files are locked for update + //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) + if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + } + $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); + + // the original file does not exist: we move the generating file + //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); + $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); + if ( $stmt->rowCount() == 0 ) + { + $metaData = $generatingMetaData; + $metaData['name'] = $filePath; + $metaData['name_hash'] = md5( $filePath ); + $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); + $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . + "VALUES( " . $this->_sqlList( $metaData ) . ")"; + if ( !$this->_query( $insertSQL, $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + } + // here we rename the actual FILE. The .generating file has been + // created on DFS, and should be renamed + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + // the original file exists: we move the generating data to this file + // and update it + else + { + if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + } + + $mtime = $generatingMetaData['mtime']; + $filesize = $generatingMetaData['size']; + //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) + if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) + { + $this->_rollback( $fname ); + throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); + } + //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); + $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); + } + + $this->_commit( $fname ); + } + + return true; + } + + /** + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ + public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) + { + $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; + + // reporting + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + $time = microtime( true ); + + $nameHash = $this->_md5( $generatingFilePath ); + $newMtime = time(); + + // The update query will only succeed if the mtime wasn't changed in between + $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; + $stmt = $this->db->query( $query ); + if ( !$stmt ) + { + // @todo Throw an exception + $this->_error( $query, $fname ); + return false; + } + $numRows = $stmt->rowCount(); + + // reporting. Manual here since we don't use _query + $time = microtime( true ) - $time; + $this->_report( $query, $fname, $time, $numRows ); + + // no rows affected or row updated with the same value + // f.e a cache-block which takes less than 1 sec to get generated + // if a line has been updated by the same values, mysqli_affected_rows + // returns 0, and updates nothing, we need to extra check this, + if( $numRows == 0 ) + { + $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; + $stmt = $this->db->query( $query ); + $row = $stmt->fetch( PDO::FETCH_NUM ); + if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + { + return true; + } + + // @todo Check if an exception makes sense here + return false; + } + // rows affected: mtime has changed, or row has been removed + if ( $numRows == 1 ) + { + return true; + } + else + { + eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); + return false; + } + } + + /** + * Aborts the cache generation process by removing the .generating file + * @param string $filePath Real cache file path + * @param string $generatingFilePath .generating cache file path + * @return void + **/ + public function _abortCacheGeneration( $generatingFilePath ) + { + $fname = "_abortCacheGeneration( $generatingFilePath )"; + + $this->_begin( $fname ); + + $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); + $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); + $this->dfsbackend->delete( $generatingFilePath ); + + $this->_commit( $fname ); + } + + /** + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ + static protected function nameTrunk( $filePath, $scope ) + { + switch ( $scope ) + { + case 'viewcache': + { + $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); + } break; + + case 'template-block': + { + $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); + $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); + if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) + { + // 6 = strlen( 'cache/' ); + $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; + $nameTrunk = substr( $filePath, 0, $len ); + } + else + { + $nameTrunk = $filePath; + } + } break; + + default: + { + $nameTrunk = $filePath; + } + } + return $nameTrunk; + } + + /** + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param resource $fileRow + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ + protected function remainingCacheGenerationTime( $row ) + { + if( !isset( $row[0] ) ) + return -1; + + return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); + } + + /** + * Returns the list of expired files + * + * @param array $scopes Array of scopes to consider. At least one. + * @param int $limit Max number of items. Set to false for unlimited. + * + * @return array(filepath) + * + * @since 4.3 + */ + public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + { + $tables = array( $this->metaDataTable, $this->metaDataTableCache ); + + if ( count( $scopes ) == 0 || $scopes == false ) + throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); + + $scopeString = $this->_sqlList( $scopes ); + + $filePathList = array(); + + foreach ( $tables as $table) + { + $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; + if ( $limit !== false ) + { + $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; + } + $stmt = $this->_query( $query, __METHOD__ ); + $filePathList = array(); + while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) + $filePathList[] = $row[0]; + unset( $stmt ); + } + + return $filePathList; + } + + public function applyServerUri( $filePath ) + { + return $this->dfsbackend->applyServerUri( $filePath ); + } + + /** + * Deletes a batch of cache files from the storage table. + * + * @param int $limit + * + * @return int The number of moved rows + * + * @throws RuntimeException if a MySQL query occurs + * @throws InvalidArgumentException if the split table feature is disabled + */ + public function deleteCacheFiles( $limit ) + { + if ( $this->metaDataTable === $this->metaDataTableCache ) + { + throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); + } + + $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; + + $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; + if ( !$stmt = $this->_query( $sql ) ) + { + throw new RuntimeException( "Error in $query" ); + } + + return $stmt->rowCount(); + } + + + + /** + * DB connexion handle + * @var handle + */ + public $db = null; + + /** + * DB connexion parameters + * @var array + */ + protected static $dbparams = null; + + /** + * Amount of executed queries, for debugging purpose + * @var int + */ + protected $numQueries = 0; + + /** + * Current transaction level. + * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) + * or COMMIT (if we're commiting the last running transaction + * @var int + */ + protected $transactionCount = 0; + + /** + * Distributed filesystem backend + * @var eZDFSFileHandlerDFSBackendInterface + */ + protected $dfsbackend = null; + + /** + * Event handler + * @var ezpEvent + */ + protected $eventHandler; + + /** + * custom dfs table name support + * @var string + */ + protected $metaDataTable = 'ezdfsfile'; + + /** + * Custom DFS table for cache storage. + * Defaults to the "normal" storage table, meaning that only one table is used. + * @var string + */ + protected $metaDataTableCache = 'ezdfsfile_cache'; + + /** + * Cache files directory, including leading & trailing slashes. + * Will be filled in using FileSettings.CacheDir from site.ini + * @var string + */ + protected $cacheDir; + + /** + * Storage directory, including leading & trailing slashes. + * Will be filled in using FileSettings.StorageDir from site.ini + * @var string + */ + protected $storageDir; + + /** + * Unique constraint violation error, used for stale cache management + * @var int + */ + const ERROR_UNIQUE_VIOLATION = 23505; +} diff --git a/sql/postgresql/cluster_dfs_schema.sql b/sql/postgresql/cluster_dfs_schema.sql index aabdca4..46cc92e 100644 --- a/sql/postgresql/cluster_dfs_schema.sql +++ b/sql/postgresql/cluster_dfs_schema.sql @@ -25,3 +25,4 @@ CREATE TABLE ezdfsfile_cache ( CREATE INDEX ezdfsfile_cache_name ON ezdfsfile_cache ( name ); CREATE INDEX ezdfsfile_cache_mtime ON ezdfsfile_cache ( mtime ); + From e080046ff7f29b74bf942f3f252949847dc7cafa Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 16:59:47 +0200 Subject: [PATCH 07/21] remove old md5 comments --- clustering/dfs/ezpostsgresqlbackend.php | 11 - .../ezpostsgresqlbackend.php.MD5_incapsulato | 1921 ---------------- ...gresqlbackend.php.MD5_incapsulato_corretto | 1921 ---------------- .../dfs/ezpostsgresqlbackend.php.originale | 1910 ---------------- ...esqlbackend.php_ante_modifiche_luca_22_mag | 1924 ----------------- 5 files changed, 7687 deletions(-) delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php.originale delete mode 100644 clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 6be86e5..fff1355 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -784,7 +784,6 @@ public function _rename( $srcFilePath, $dstFilePath ) $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -800,10 +799,8 @@ public function _rename( $srcFilePath, $dstFilePath ) // Create a new meta-data entry for the new file to make foreign keys happy. $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; "WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -819,7 +816,6 @@ public function _rename( $srcFilePath, $dstFilePath ) } // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { @@ -1414,7 +1410,6 @@ protected function _quote( $value ) **/ protected function _md5( $value ) { - //return "MD5(" . $this->_quote( $value ) . ")"; return $this->_quote( md5( $value ) ); } @@ -1566,7 +1561,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) // no rename: the .generating entry is just deleted if ( $rename === false ) { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); $this->dfsbackend->delete( $generatingFilePath ); return true; @@ -1578,7 +1572,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_begin( $fname ); // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); @@ -1587,7 +1580,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); if ( $stmt->rowCount() == 0 ) { @@ -1609,7 +1601,6 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $this->_rollback( $fname ); throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } // the original file exists: we move the generating data to this file @@ -1624,13 +1615,11 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) $mtime = $generatingMetaData['mtime']; $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) { $this->_rollback( $fname ); throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato deleted file mode 100644 index 577a717..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato +++ /dev/null @@ -1,1921 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this->_quote( md5( $value ) ); - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto b/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto deleted file mode 100644 index 73927bd..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.MD5_incapsulato_corretto +++ /dev/null @@ -1,1921 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePathStr ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePathStr ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePathStr ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this-> md5( $value ) ; - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} diff --git a/clustering/dfs/ezpostsgresqlbackend.php.originale b/clustering/dfs/ezpostsgresqlbackend.php.originale deleted file mode 100644 index e89d353..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php.originale +++ /dev/null @@ -1,1910 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - "WHERE name_hash=MD5($srcFilePathStr)"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - return "MD5(" . $this->_quote( $value ) . ")"; - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} \ No newline at end of file diff --git a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag b/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag deleted file mode 100644 index feaf076..0000000 --- a/clustering/dfs/ezpostsgresqlbackend.php_ante_modifiche_luca_22_mag +++ /dev/null @@ -1,1924 +0,0 @@ -eventHandler = ezpEvent::getInstance(); - $fileINI = eZINI::instance( 'file.ini' ); - $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); - - if ( defined( 'CLUSTER_METADATA_TABLE_CACHE' ) ) - { - $this->metaDataTableCache = CLUSTER_METADATA_TABLE_CACHE; - } - else if ( $fileINI->hasVariable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ) ) - { - $this->metaDataTableCache = $fileINI->variable( 'eZDFSClusteringSettings', 'MetaDataTableNameCache' ); - } - - $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); - $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); - } - - /** - * Returns the database table name to use for the specified file. - * - * For files detected as cache files the cache table is returned, if not - * the generic table is returned. - * - * @param string $filePath - * @return string The database table name - */ - protected function dbTable( $filePath ) - { - if ( $this->metaDataTableCache == $this->metaDataTable ) - return $this->metaDataTable; - - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) - { - return $this->metaDataTableCache; - } - - return $this->metaDataTable; - } - - /** - * Connects to the database. - * - * @return void - * @throw eZClusterHandlerDBNoConnectionException - * @throw eZClusterHandlerDBNoDatabaseException - **/ - public function _connect() - { - $siteINI = eZINI::instance( 'site.ini' ); - // DB Connection setup - // This part is not actually required since _connect will only be called - // once, but it is useful to run the unit tests. So be it. - // @todo refactor this using eZINI::setVariable in unit tests - if ( self::$dbparams === null ) - { - $fileINI = eZINI::instance( 'file.ini' ); - - self::$dbparams = array(); - self::$dbparams['host'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBHost' ); - self::$dbparams['port'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPort' ); - self::$dbparams['socket'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBSocket' ); - self::$dbparams['dbname'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBName' ); - self::$dbparams['user'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBUser' ); - self::$dbparams['pass'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBPassword' ); - - self::$dbparams['max_connect_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBConnectRetries' ); - self::$dbparams['max_execute_tries'] = $fileINI->variable( 'eZDFSClusteringSettings', 'DBExecuteRetries' ); - - self::$dbparams['sql_output'] = $siteINI->variable( 'DatabaseSettings', 'SQLOutput' ) == 'enabled'; - - self::$dbparams['cache_generation_timeout'] = $siteINI->variable( 'ContentSettings', 'CacheGenerationTimeout' ); - } - - - $connectString = sprintf( 'pgsql:host=%s;port=%s;dbname=%s;user=%s;password=%s', - self::$dbparams['host'], - self::$dbparams['port'], - self::$dbparams['dbname'], - self::$dbparams['user'], - self::$dbparams['pass'] - ); - $tries = 0; - while ( $tries < self::$dbparams['max_connect_tries'] ) - { - try { - $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); - ++$tries; - continue; - } - break; - } - if ( !( $this->db instanceof PDO ) ) - { - $this->_die( "Unable to connect to storage server" ); - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - } - - - if ( !$this->db ) - throw new eZClusterHandlerDBNoConnectionException( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); - - $this->db->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - $this->db->exec( "SET NAMES 'utf8'" ); - - - // DFS setup - if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); - } - - /** - * Disconnects the handler from the database - */ - public function _disconnect() - { - if ( $this->db !== null ) - { - $this->db = null; - } - } - - /** - * Creates a copy of a file in DB+DFS - * @param string $srcFilePath Source file - * @param string $dstFilePath Destination file - * @param string $fname - * @return bool - * - * @see _copyInner - **/ - public function _copy( $srcFilePath, $dstFilePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_copy($srcFilePath, $dstFilePath)"; - else - $fname = "_copy($srcFilePath, $dstFilePath)"; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath, $fname ); - // if source file does not exist then do nothing. - // @todo Throw an exception here. - // Info: $srcFilePath - if ( !$metaData ) - { - return false; - } - return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); - } - - /** - * Inner function used by _copy to perform the operation in a transaction - * - * @param string $srcFilePath - * @param string $dstFilePath - * @param bool $fname - * @param array $metaData Source file's metadata - * @return bool - * - * @see _copy - */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) - { - $this->_delete( $dstFilePath, true, $fname ); - - $datatype = $metaData['datatype']; - $filePathHash = md5( $dstFilePath ); - $scope = $metaData['scope']; - $contentLength = $metaData['size']; - $fileMTime = $metaData['mtime']; - $nameTrunk = self::nameTrunk( $dstFilePath, $scope ); - - // Copy file metadata. - if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); - } - - // Copy file data. - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - return true; - } - - /** - * Purges meta-data and file-data for a file entry - * - * Will only expire a single file. Use _purgeByLike to purge multiple files - * - * @param string $filePath Path of the file to purge - * @param bool $onlyExpired Only purges expired files - * @param bool|int $expiry - * @param bool $fname - * - * @see _purgeByLike - **/ - public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purge($filePath)"; - else - $fname = "_purge($filePath)"; - $sql = "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - if ( $expiry !== false ) - { - $sql .= " AND mtime<" . (int)$expiry; - } - elseif ( $onlyExpired ) - { - $sql .= " AND expired=1"; - } - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Purging file metadata for $filePath failed" ); - } - if ( $stmt->rowCount() == 1 ) - { - $this->dfsbackend->delete( $filePath ); - } - return true; - } - - /** - * Purges meta-data and file-data for files matching a pattern using a SQL - * LIKE syntax. - * - * @param string $like - * SQL LIKE string applied to ezdfsfile.name to look for files to - * purge - * @param bool $onlyExpired - * Only purge expired files (ezdfsfile.expired = 1) - * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry - * Timestamp used to limit deleted files: only files older than this - * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge - * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk - */ - public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_purgeByLike($like, $onlyExpired)"; - else - $fname = "_purgeByLike($like, $onlyExpired)"; - - // common query part used for both DELETE and SELECT - $where = " WHERE name LIKE " . $this->_quote( $like ); - - if ( $expiry !== false ) - $where .= " AND mtime < " . (int)$expiry; - elseif ( $onlyExpired ) - $where .= " AND expired = 1"; - - if ( $limit ) - $sqlLimit = " LIMIT $limit"; - else - $sqlLimit = ""; - - $this->_begin( $fname ); - - // select query, in FOR UPDATE mode - $selectSQL = "SELECT name FROM " . $this->dbTable( $like ) . - "{$where} {$sqlLimit} FOR UPDATE"; - if ( !$stmt = $this->_query( $selectSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); - } - - // if there are no results, we can just return 0 and stop right here - if ( $stmt->rowCount() == 0 ) - { - $this->_rollback( $fname ); - return 0; - } - // the candidate for purge are indexed in an array - else - { - while( $row = $stmt->fetch( PDO::FETCH_ASSOC ) ) - { - $files[] = $row['name']; - } - } - - // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . - "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; - if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) - { - $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); - } - $deletedDBFiles = $stmt->rowCount(); - $this->dfsbackend->delete( $files ); - - $this->_commit( $fname ); - - return $deletedDBFiles; - } - - /** - * Deletes a file from DB - * - * The file won't be removed from disk, _purge has to be used for this. - * Only single files will be deleted, to delete multiple files, - * _deleteByLike has to be used. - * - * @param string $filePath Path of the file to delete - * @param bool $insideOfTransaction - * Wether or not a transaction is already started - * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike - * @return bool - */ - public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_delete($filePath)"; - else - $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of - // checking if a transaction is running. But leave it like this - // for now. - if ( $insideOfTransaction ) - { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } - } - else - { - return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); - } - } - - /** - * Callback method used by by _delete to delete a single file - * - * @param string $filePath Path of the file to delete - * @param string $fname Optional caller name for debugging - * @return bool - **/ - protected function _deleteInner( $filePath, $fname ) - { - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); - return true; - } - - /** - * Deletes multiple files using a SQL LIKE statement - * - * Use _delete if you need to delete single files - * - * @param string $like - * SQL LIKE condition applied to ezdfsfile.name to look for files - * to delete. Will use name_trunk if the LIKE string matches a - * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging - * @return bool - * @see _deleteByLikeInner - * @see _delete - */ - public function _deleteByLike( $like, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByLike($like)"; - else - $fname = "_deleteByLike($like)"; - return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); - } - - /** - * Callback used by _deleteByLike to perform the deletion - * - * @param string $like - * @param mixed $fname - * @return - */ - private function _deleteByLikeInner( $like, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by like: '$like'" ); - } - return true; - } - - /** - * Deletes DB files by using a SQL regular expression applied to file names - * - * @param string $regex - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByRegex( $regex, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByRegex($regex)"; - else - $fname = "_deleteByRegex($regex)"; - return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); - } - - /** - * Callback used by _deleteByRegex to perform the deletion - * - * @param mixed $regex - * @param mixed $fname - * @return - * @deprecated Has severe performances issues - */ - public function _deleteByRegexInner( $regex, $fname ) - { - $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); - } - return true; - } - - /** - * Deletes multiple DB files by wildcard - * - * @param string $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - public function _deleteByWildcard( $wildcard, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByWildcard($wildcard)"; - else - $fname = "_deleteByWildcard($wildcard)"; - return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); - } - - /** - * Callback used by _deleteByWildcard to perform the deletion - * - * @param mixed $wildcard - * @param mixed $fname - * @return bool - * @deprecated Has severe performance issues - */ - protected function _deleteByWildcardInner( $wildcard, $fname ) - { - // Convert wildcard to regexp. - $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; - - $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); - - $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); - - $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; - if ( !$res = $this->_query( $sql, $fname ) ) - { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); - } - return true; - } - - public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) - { - if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; - return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); - } - - protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) - { - foreach ( $dirList as $dirItem ) - { - if ( strstr( $commonPath, '/cache/content' ) !== false or strstr( $commonPath, '/cache/template-block' ) !== false ) - { - $where = "WHERE name_trunk = '$commonPath/$dirItem/$commonSuffix'"; - } - else - { - $where = "WHERE name LIKE '$commonPath/$dirItem/$commonSuffix%'"; - } - $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; - if ( !$stmt = $this->_query( $sql, $fname ) ) - { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); - } - } - return true; - } - - public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) - { - if ( $fname ) - $fname .= "::_exists($filePath)"; - else - $fname = "_exists($filePath)"; - $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); - if ( $row === false ) - return false; - - if ( $ignoreExpiredFiles ) - $rc = $row[1] >= 0; - else - $rc = true; - - if ( $checkOnDFS && $rc ) - { - $rc = $this->dfsbackend->existsOnDFS( $filePath ); - } - - return $rc; - } - - protected function __mkdir_p( $dir ) - { - // create parent directories - $dirElements = explode( '/', $dir ); - if ( count( $dirElements ) == 0 ) - return true; - - $result = true; - $currentDir = $dirElements[0]; - - if ( $currentDir != '' && !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - for ( $i = 1; $i < count( $dirElements ); ++$i ) - { - $dirElement = $dirElements[$i]; - if ( strlen( $dirElement ) == 0 ) - continue; - - $currentDir .= '/' . $dirElement; - - if ( !file_exists( $currentDir ) && !eZDir::mkdir( $currentDir, false ) ) - return false; - - $result = true; - } - - return $result; - } - - /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ - public function _fetch( $filePath, $uniqueName = false ) - { - $metaData = $this->_fetchMetadata( $filePath ); - if ( !$metaData ) - { - // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); - return false; - } - - $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); - $loopCount = 0; - $localFileSize = 0; - - do - { - // create temporary file - $tmpid = getmypid() . '-' . mt_rand() .'tmp'; - if ( strrpos( $filePath, '.' ) > 0 ) - $tmpFilePath = substr_replace( $filePath, $tmpid, strrpos( $filePath, '.' ), 0 ); - else - $tmpFilePath = $filePath . '.' . $tmpid; - $this->__mkdir_p( dirname( $tmpFilePath ) ); - eZDebugSetting::writeDebug( 'kernel-clustering', "copying DFS://$filePath to FS://$tmpFilePath on try: $loopCount " ); - - // copy DFS file to temporary FS path - // @todo Throw an exception - if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) - { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - - if ( $uniqueName !== true ) - { - if( !eZFile::rename( $tmpFilePath, $filePath, false, eZFile::CLEAN_ON_FAILURE | eZFile::APPEND_DEBUG_ON_FAILURE ) ) - { - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - continue; - } - } - $filePath = ($uniqueName) ? $tmpFilePath : $filePath ; - - // If all data has been written correctly, return the filepath. - // Otherwise let the loop continue - clearstatcache( true, $filePath ); - $localFileSize = filesize( $filePath ); - if ( $dfsFileSize == $localFileSize ) - { - return $filePath; - } - // Sizes might have been corrupted by FS problems. Enforcing temp file removal. - else if ( file_exists( $tmpFilePath ) ) - { - unlink( $tmpFilePath ); - } - - usleep( self::TIME_UNTIL_RETRY ); - ++$loopCount; - } - while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); - - // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); - return false; - } - - public function _fetchContents( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchContents($filePath)"; - else - $fname = "_fetchContents($filePath)"; - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); - return false; - } - - // @todo Catch an exception - if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) - { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); - return false; - } - return $contents; - } - - /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. - */ - function _fetchMetadata( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_fetchMetadata($filePath)"; - else - $fname = "_fetchMetadata($filePath)"; - $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); - return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); - } - - public function _linkCopy( $srcPath, $dstPath, $fname = false ) - { - if ( $fname ) - $fname .= "::_linkCopy($srcPath,$dstPath)"; - else - $fname = "_linkCopy($srcPath,$dstPath)"; - return $this->_copy( $srcPath, $dstPath, $fname ); - } - - /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) - { - if ( $fname ) - $fname .= "::_passThrough($filePath)"; - else - $fname = "_passThrough($filePath)"; - - $metaData = $this->_fetchMetadata( $filePath, $fname ); - // @todo Throw an exception - if ( !$metaData ) - return false; - - // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); - - return true; - } - - /** - * Renames $srcFilePath to $dstFilePath - * - * @param string $srcFilePath - * @param string $dstFilePath - * @return bool - */ - public function _rename( $srcFilePath, $dstFilePath ) - { - if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; - - // fetch source file metadata - $metaData = $this->_fetchMetadata( $srcFilePath ); - // if source file does not exist then do nothing. - // @todo Throw an exception - if ( !$metaData ) - return false; - - $this->_begin( __METHOD__ ); - - $srcFilePathStr = $this->_quote( $srcFilePath ); - $dstFilePathStr = $this->_quote( $dstFilePath ); - $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); - - // Mark entry for update to lock it - //$sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr) FOR UPDATE"; - $sql = "SELECT * FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ) . " FOR UPDATE"; - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - return false; - } - - if ( $this->_exists( $dstFilePath, false, false ) ) - $this->_purge( $dstFilePath, false ); - - // Create a new meta-data entry for the new file to make foreign keys happy. - $sql = "INSERT INTO " . $this->dbTable( $srcFilePath ) . " ". - "(name, name_trunk, name_hash, datatype, scope, size, mtime, expired) " . - //"SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, MD5( $dstFilePathStr ) AS name_hash, " . - "SELECT $dstFilePathStr AS name, $dstNameTrunkStr as name_trunk, " . $this->_md5( $dstFilePath ) . " AS name_hash, " . - "datatype, scope, size, mtime, expired FROM " . $this->dbTable( $srcFilePath ) . " " . - //"WHERE name_hash=MD5($srcFilePathStr)"; - "WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) - { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); - } - - // Remove old entry - //$sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=MD5($srcFilePathStr)"; - $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); - if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) - { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); - $this->_rollback( __METHOD__ ); - // @todo Throw an exception - return false; - } - - // delete original DFS file - // @todo Catch an exception - $this->dfsbackend->delete( $srcFilePath ); - - $this->_commit( __METHOD__ ); - - return true; - } - - /** - * Stores $filePath to cluster - * - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @return void - */ - function _store( $filePath, $datatype, $scope, $fname = false ) - { - if ( !is_readable( $filePath ) ) - { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; - } - if ( $fname ) - $fname .= "::_store($filePath, $datatype, $scope)"; - else - $fname = "_store($filePath, $datatype, $scope)"; - - $return = $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); - return $return; - } - - /** - * Callback function used to perform the actual file store operation - * @param string $filePath - * @param string $datatype - * @param string $scope - * @param string $fname - * @see eZDFSFileHandlerMySQLBackend::_store() - * @return bool - **/ - function _storeInner( $filePath, $datatype, $scope, $fname ) - { - // Insert file metadata. - clearstatcache(); - $fileMTime = filemtime( $filePath ); - $contentLength = filesize( $filePath ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - - if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) - { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); - } - - // copy given $filePath to DFS - if ( !$this->dfsbackend->copyToDFS( $filePath ) ) - { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); - } - - return true; - } - - /** - * Stores $contents as the contents of $filePath to the cluster - * - * @param string $filePath - * @param string $contents - * @param string $scope - * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void - */ - function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) - { - if ( $fname ) - $fname .= "::_storeContents($filePath, ..., $scope, $datatype)"; - else - $fname = "_storeContents($filePath, ..., $scope, $datatype)"; - - return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); - } - - function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) - { - // File metadata. - $contentLength = strlen( $contents ); - $filePathHash = md5( $filePath ); - $nameTrunk = self::nameTrunk( $filePath, $scope ); - if ( $mtime === false ) - $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; - - // Copy file metadata. - $result = $this->_insertUpdate( - $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $mtime, - 'expired' => ( $mtime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname - ); - if ( $result === false ) - { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); - } - - if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) - { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); - } - - return true; - } - - public function _getFileList( $scopes = false, $excludeScopes = false ) - { - $filePathList = array(); - $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - - foreach ( $tables as $table ) - { - $query = 'SELECT name FROM ' . $table; - - if ( is_array( $scopes ) && count( $scopes ) > 0 ) - { - $query .= ' WHERE scope '; - if ( $excludeScopes ) - $query .= 'NOT '; - $query .= "IN ('" . implode( "', '", $scopes ) . "')"; - } - - $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); - if ( !$stmt ) - { - eZDebug::writeDebug( 'Unable to get file list', __METHOD__ ); - // @todo Throw an exception - return false; - } - - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - - unset( $stmt ); - } - return $filePathList; - } - - /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ - protected function _die( $msg, $sql = null ) - { - if ( $this->db ) - { - $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); - } - else - { - eZDebug::writeError( $sql, $msg ); - } - } - - /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ - function _insert( $table, $array, $fname ) - { - $keys = array_keys( $array ); - $query = "INSERT INTO $table (" . join( ", ", $keys ) . ") VALUES (" . $this->_sqlList( $array ) . ")"; - $res = $this->_query( $query, $fname ); - if ( !$res ) - { - // @todo Throw an exception - return false; - } - } - - /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ - protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) - { - if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) - { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); - } - - if ( $row = $this->_fetchMetadata( $insert['name'] ) ) - { - $sql = "UPDATE $table SET "; - $setEntries = array(); - foreach( $update as $field ) - { - $setEntries[] = "$field=" . $this->_quote( $insert[$field] ); - } - $sql .= implode( ', ', $setEntries ) . - " WHERE name_hash=" . $this->_quote( $insert['name_hash'] ); - } - else - { - // create file in db - $quotedValues = array(); - foreach( $insert as $value ) - { - $quotedValues[] = $this->_quote( $value ); - } - $sql = "INSERT INTO $table " . - "(" . implode( ', ', array_keys( $insert ) ) . ") " . - "VALUES( " . implode( ', ', $quotedValues ) . ")"; - } - - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { - $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; - } - } - - /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ - protected function _sqlList( $array ) - { - $text = ""; - $sep = ""; - foreach ( $array as $e ) - { - $text .= $sep; - $text .= $this->_quote( $e ); - $sep = ", "; - } - return $text; - } - - /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); - } - - /** - * Runs a select query and returns one associative row from the result. - * - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ - protected function _selectOneAssoc( $query, $fname, $error = false, $debug = false ) - { - return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_ASSOC ); - } - - /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ - protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - $this->_error( $query, $fname, $error ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo Throw an exception - return false; - } - - $numRows = $stmt->rowCount(); - if ( $numRows > 1 ) - { - eZDebug::writeError( 'Duplicate entries found', $fname ); - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - // @todo throw an exception instead. Should NOT happen. - } - elseif ( $numRows === 0 ) - { - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - return false; - } - - $row = $stmt->fetch( $fetchCall ); - unset( $stmt ); - if ( $debug ) - $query = "SQL for _selectOneAssoc:\n" . $query . "\n\nRESULT:\n" . var_export( $row, true ); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time ); - return $row; - } - - /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ - protected function _begin( $fname = false ) - { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; - $this->transactionCount++; - if ( $this->transactionCount == 1 ) - $this->db->beginTransaction(); - } - - /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _commit( $fname = false ) - { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->commit(); - } - - /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ - protected function _rollback( $fname = false ) - { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; - $this->transactionCount--; - if ( $this->transactionCount == 0 ) - $this->db->rollBack(); - } - - /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ - protected function _protect() - { - $args = func_get_args(); - $callback = array_shift( $args ); - $fname = array_shift( $args ); - - $maxTries = self::$dbparams['max_execute_tries']; - $tries = 0; - while ( $tries < $maxTries ) - { - $this->_begin( $fname ); - - try { - $result = call_user_func_array( $callback, $args ); - } - catch( PDOException $e ) - { - print_r( compact( 'callback', 'args' ) ); - eZDebug::writeError( $e ); - return false; - } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) - { - $this->_rollback( $fname ); - return false; - } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - - break; // All is good, so break out of loop - } - - $this->_commit( $fname ); - return $result; - } - - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ - protected function _fail( $message, $result = false) - { - // @todo Investigate the right function - if ( $result !== false ) - { - $message .= "\n" . pg_result_error( $result, PGSQL_DIAG_SQLSTATE ) . ": " . pg_result_error( $result, PGSQL_DIAG_MESSAGE_PRIMARY ); - } - else - { - $errorInfo = $this->db->errorInfo(); - $message .= "\n$errorInfo[2]"; - } - throw new Exception( $message ); - } - - /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ - protected function _query( $query, $fname = false, $reportError = true ) - { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $stmt = $this->db->query( $query ); - if ( $stmt == false ) - { - if ( $reportError ) - $this->_error( $query, $stmt, $fname ); - return $stmt; - } - - $numRows = $stmt->rowCount(); - - $time = microtime( true ) - $time; - eZDebug::accumulatorStop( 'postgresql_cluster_query' ); - - $this->_report( $query, $fname, $time, $numRows ); - - return $stmt; - } - - /** - * Make sure that $value is escaped and qouted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ - protected function _quote( $value ) - { - if ( $value === null ) - return 'NULL'; - elseif ( is_integer( $value ) ) - { - return $this->db->quote( $value, PDO::PARAM_INT ); - } - else - { - return $this->db->quote( $value, PDO::PARAM_STR ); - } - } - - /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. - **/ - protected function _md5( $value ) - { - //return "MD5(" . $this->_quote( $value ) . ")"; - return $this->_quote( md5( $value ) ); - } - - /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param resource $res The result resource the error occured on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. - */ - protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) - { - if ( $error === false ) - { - $error = "Failed to execute SQL for function:"; - } - else if ( is_array( $error ) ) - { - $fname = $error[1]; - $error = $error[0]; - } - - // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); - } - - /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ - function _report( $query, $fname, $timeTaken, $numRows = false ) - { - if ( !self::$dbparams['sql_output'] ) - return; - - $rowText = ''; - if ( $numRows !== false ) - $rowText = "$numRows rows, "; - static $numQueries = 0; - if ( strlen( $fname ) == 0 ) - $fname = "_query"; - $backgroundClass = ($this->transactionCount > 0 ? "debugtransaction transactionlevel-$this->transactionCount" : ""); - eZDebug::writeNotice( "$query", "cluster::posgresql::{$fname}[{$rowText}" . number_format( $timeTaken, 3 ) . " ms] query number per page:" . $numQueries++, $backgroundClass ); - } - - /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ - public function _startCacheGeneration( $filePath, $generatingFilePath ) - { - $fname = "_startCacheGeneration( {$filePath} )"; - - $nameHash = $this->_md5( $generatingFilePath ); - $mtime = time(); - - $insertData = array( 'name' => $this->_quote( $generatingFilePath ), - 'name_trunk' => $this->_quote( $generatingFilePath ), - 'name_hash' => $nameHash, - 'scope' => "''", - 'datatype' => "''", - 'mtime' => $this->_quote( $mtime ), - 'expired' => 0 ); - $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . - "VALUES(" . implode( ', ', $insertData ) . ")"; - - //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch - //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; - - try { - $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { - $errno = $e->getCode(); - if ( $errno != self::ERROR_UNIQUE_VIOLATION ) - { - throw new RuntimeException( "Unexpected error #$errno when trying to start cache generation on $filePath (" . $e->getMessage() . ')' ); - } - // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) - else - { - // generation timout check - $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; - $row = $this->_selectOneRow( $query, $fname, false, false ); - - // file has been renamed, i.e it is no longer a .generating file - if( $row and !isset( $row[0] ) ) - return array( 'result' => 'ok', 'mtime' => $mtime ); - - $remainingGenerationTime = $this->remainingCacheGenerationTime( $row ); - if ( $remainingGenerationTime < 0 ) - { - $previousMTime = $row[0]; - - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); - $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; - - // we run the query manually since the default _query won't - // report affected rows - $stmt = $this->db->query( $updateQuery ); - if ( ( $stmt !== false ) && $stmt->rowCount() == 1 ) - { - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - else - { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); - } - } - else - { - return array( 'result' => 'ko', 'remaining' => $remainingGenerationTime ); - } - } - } - - return array( 'result' => 'ok', 'mtime' => $mtime ); - } - - /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ - public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) - { - $fname = "_endCacheGeneration( $filePath )"; - - // no rename: the .generating entry is just deleted - if ( $rename === false ) - { - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - $this->dfsbackend->delete( $generatingFilePath ); - return true; - } - // rename mode: the generating file and its contents are renamed to the - // final name - else - { - $this->_begin( $fname ); - - // both files are locked for update - //if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath') FOR UPDATE", $fname, true ) ) - if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); - } - $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); - - // the original file does not exist: we move the generating file - //$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$filePath') FOR UPDATE", $fname, false ); - $stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ) . " FOR UPDATE", $fname, false ); - if ( $stmt->rowCount() == 0 ) - { - $metaData = $generatingMetaData; - $metaData['name'] = $filePath; - $metaData['name_hash'] = md5( $filePath ); - $metaData['name_trunk'] = $this->nameTrunk( $filePath, $metaData['scope'] ); - $insertSQL = "INSERT INTO " . $this->dbTable( $filePath ) . " ( " . implode( ', ', array_keys( $metaData ) ) . " ) " . - "VALUES( " . $this->_sqlList( $metaData ) . ")"; - if ( !$this->_query( $insertSQL, $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); - } - // here we rename the actual FILE. The .generating file has been - // created on DFS, and should be renamed - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - // the original file exists: we move the generating data to this file - // and update it - else - { - if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); - } - - $mtime = $generatingMetaData['mtime']; - $filesize = $generatingMetaData['size']; - //if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=MD5('$filePath')", $fname, true ) ) - if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = '{$mtime}', expired = 0, size = '{$filesize}' WHERE name_hash=" . $this->_md5( $filePath ), $fname, true ) ) - { - $this->_rollback( $fname ); - throw new RuntimeException( "An error marking '$filePath' as not expired in the database" ); - } - //$this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=MD5('$generatingFilePath')", $fname, true ); - $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); - } - - $this->_commit( $fname ); - } - - return true; - } - - /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ - public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) - { - $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; - - // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); - $time = microtime( true ); - - $nameHash = $this->_md5( $generatingFilePath ); - $newMtime = time(); - - // The update query will only succeed if the mtime wasn't changed in between - $query = "UPDATE " . $this->dbTable( $generatingFilePath ) . " SET mtime = $newMtime WHERE name_hash = {$nameHash} AND mtime = $generatingFileMtime"; - $stmt = $this->db->query( $query ); - if ( !$stmt ) - { - // @todo Throw an exception - $this->_error( $query, $fname ); - return false; - } - $numRows = $stmt->rowCount(); - - // reporting. Manual here since we don't use _query - $time = microtime( true ) - $time; - $this->_report( $query, $fname, $time, $numRows ); - - // no rows affected or row updated with the same value - // f.e a cache-block which takes less than 1 sec to get generated - // if a line has been updated by the same values, mysqli_affected_rows - // returns 0, and updates nothing, we need to extra check this, - if( $numRows == 0 ) - { - $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; - $stmt = $this->db->query( $query ); - $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); - { - return true; - } - - // @todo Check if an exception makes sense here - return false; - } - // rows affected: mtime has changed, or row has been removed - if ( $numRows == 1 ) - { - return true; - } - else - { - eZDebugSetting::writeDebug( 'kernel-clustering', "No rows affected by query '$query', record has been modified", __METHOD__ ); - return false; - } - } - - /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ - public function _abortCacheGeneration( $generatingFilePath ) - { - $fname = "_abortCacheGeneration( $generatingFilePath )"; - - $this->_begin( $fname ); - - $sql = "DELETE FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = " . $this->_md5( $generatingFilePath ); - $this->_query( $sql, "_abortCacheGeneration( '$generatingFilePath' )" ); - $this->dfsbackend->delete( $generatingFilePath ); - - $this->_commit( $fname ); - } - - /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ - static protected function nameTrunk( $filePath, $scope ) - { - switch ( $scope ) - { - case 'viewcache': - { - $nameTrunk = substr( $filePath, 0, strrpos( $filePath, '-' ) + 1 ); - } break; - - case 'template-block': - { - $templateBlockCacheDir = eZTemplateCacheBlock::templateBlockCacheDir(); - $templateBlockPath = str_replace( $templateBlockCacheDir, '', $filePath ); - if ( strstr( $templateBlockPath, 'subtree/' ) !== false ) - { - // 6 = strlen( 'cache/' ); - $len = strlen( $templateBlockCacheDir ) + strpos( $templateBlockPath, 'cache/' ) + 6; - $nameTrunk = substr( $filePath, 0, $len ); - } - else - { - $nameTrunk = $filePath; - } - } break; - - default: - { - $nameTrunk = $filePath; - } - } - return $nameTrunk; - } - - /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ - protected function remainingCacheGenerationTime( $row ) - { - if( !isset( $row[0] ) ) - return -1; - - return ( $row[0] + self::$dbparams['cache_generation_timeout'] ) - time(); - } - - /** - * Returns the list of expired files - * - * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. - * - * @return array(filepath) - * - * @since 4.3 - */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) - { - $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - - if ( count( $scopes ) == 0 || $scopes == false ) - throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); - - $scopeString = $this->_sqlList( $scopes ); - - $filePathList = array(); - - foreach ( $tables as $table) - { - $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; - if ( $limit !== false ) - { - $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; - } - $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; - unset( $stmt ); - } - - return $filePathList; - } - - public function applyServerUri( $filePath ) - { - return $this->dfsbackend->applyServerUri( $filePath ); - } - - /** - * Deletes a batch of cache files from the storage table. - * - * @param int $limit - * - * @return int The number of moved rows - * - * @throws RuntimeException if a MySQL query occurs - * @throws InvalidArgumentException if the split table feature is disabled - */ - public function deleteCacheFiles( $limit ) - { - if ( $this->metaDataTable === $this->metaDataTableCache ) - { - throw new InvalidArgumentException( "The split table features is disabled: cache and storage table are identical" ); - } - - $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; - - $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) - { - throw new RuntimeException( "Error in $query" ); - } - - return $stmt->rowCount(); - } - - - - /** - * DB connexion handle - * @var handle - */ - public $db = null; - - /** - * DB connexion parameters - * @var array - */ - protected static $dbparams = null; - - /** - * Amount of executed queries, for debugging purpose - * @var int - */ - protected $numQueries = 0; - - /** - * Current transaction level. - * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction - * @var int - */ - protected $transactionCount = 0; - - /** - * Distributed filesystem backend - * @var eZDFSFileHandlerDFSBackendInterface - */ - protected $dfsbackend = null; - - /** - * Event handler - * @var ezpEvent - */ - protected $eventHandler; - - /** - * custom dfs table name support - * @var string - */ - protected $metaDataTable = 'ezdfsfile'; - - /** - * Custom DFS table for cache storage. - * Defaults to the "normal" storage table, meaning that only one table is used. - * @var string - */ - protected $metaDataTableCache = 'ezdfsfile_cache'; - - /** - * Cache files directory, including leading & trailing slashes. - * Will be filled in using FileSettings.CacheDir from site.ini - * @var string - */ - protected $cacheDir; - - /** - * Storage directory, including leading & trailing slashes. - * Will be filled in using FileSettings.StorageDir from site.ini - * @var string - */ - protected $storageDir; - - /** - * Unique constraint violation error, used for stale cache management - * @var int - */ - const ERROR_UNIQUE_VIOLATION = 23505; -} From fd7460a18420b4e6e6a88a5bc40f3a76bcbccdfe Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 22:18:30 +0200 Subject: [PATCH 08/21] fix phpdoc and some typos --- clustering/dfs/ezpostsgresqlbackend.php | 554 +++++++++++++----------- 1 file changed, 290 insertions(+), 264 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index fff1355..e1abc77 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -9,10 +9,10 @@ */ /** - * This class allows DFS based clustering using PostgreSQL + * This class allows DFS based clustering using PostgresSQL * @package Cluster */ -class eZDFSFileHandlerPostgresqlBackend +class eZDFSFileHandlerPostgresqlBackend implements eZDFSFileHandlerDBBackendInterface { /** @@ -26,7 +26,7 @@ class eZDFSFileHandlerPostgresqlBackend * @var int */ protected $maxCopyTries; - + public function __construct() { $this->eventHandler = ezpEvent::getInstance(); @@ -154,13 +154,17 @@ public function _disconnect() /** * Creates a copy of a file in DB+DFS + * + * @see eZDFSFileHandler::fileCopy + * @see _copyInner + * * @param string $srcFilePath Source file * @param string $dstFilePath Destination file - * @param string $fname + * @param bool|string $fname Optional caller name for debugging + * * @return bool * - * @see _copyInner - **/ + */ public function _copy( $srcFilePath, $dstFilePath, $fname = false ) { if ( $fname ) @@ -192,7 +196,7 @@ public function _copy( $srcFilePath, $dstFilePath, $fname = false ) * * @see _copy */ - private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) + protected function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) { $this->_delete( $dstFilePath, true, $fname ); @@ -216,13 +220,13 @@ private function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), $fname ) === false ) { - return $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); + $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); } // Copy file data. if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) { - return $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + $this->_fail( $srcFilePath, "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); } return true; } @@ -256,7 +260,7 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname } if ( !$stmt = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Purging file metadata for $filePath failed" ); + $this->_fail( "Purging file metadata for $filePath failed" ); } if ( $stmt->rowCount() == 1 ) { @@ -268,6 +272,10 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname /** * Purges meta-data and file-data for files matching a pattern using a SQL * LIKE syntax. + * This method should also remove the files from disk + * + * @see eZDFSFileHandler::purge + * @see _purge * * @param string $like * SQL LIKE string applied to ezdfsfile.name to look for files to @@ -275,13 +283,12 @@ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname * @param bool $onlyExpired * Only purge expired files (ezdfsfile.expired = 1) * @param integer $limit Maximum number of items to purge in one call - * @param integer $expiry + * @param integer|bool $expiry * Timestamp used to limit deleted files: only files older than this * date will be deleted - * @param mixed $fname Optional caller name for debugging - * @see _purge + * @param bool|string $fname Optional caller name for debugging + * * @return bool|int false if it fails, number of affected rows otherwise - * @todo This method should also remove the files from disk */ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry = false, $fname = false ) { @@ -311,9 +318,10 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry if ( !$stmt = $this->_query( $selectSQL, $fname ) ) { $this->_rollback( $fname ); - return $this->_fail( "Selecting file metadata by like statement $like failed" ); + $this->_fail( "Selecting file metadata by like statement $like failed" ); } + $files = array(); // if there are no results, we can just return 0 and stop right here if ( $stmt->rowCount() == 0 ) { @@ -330,12 +338,12 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry } // delete query - $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " " . "WHERE name_hash IN " . + $deleteSQL = "DELETE FROM " . $this->dbTable( $like ) . " WHERE name_hash IN " . "(SELECT name_hash FROM ". $this->dbTable( $like ) . " $where $sqlLimit)"; if ( !$stmt = $this->_query( $deleteSQL, $fname ) ) { $this->_rollback( $fname ); - return $this->_fail( "Purging file metadata by like statement $like failed" ); + $this->_fail( "Purging file metadata by like statement $like failed" ); } $deletedDBFiles = $stmt->rowCount(); $this->dfsbackend->delete( $files ); @@ -347,17 +355,20 @@ public function _purgeByLike( $like, $onlyExpired = false, $limit = 50, $expiry /** * Deletes a file from DB - * * The file won't be removed from disk, _purge has to be used for this. * Only single files will be deleted, to delete multiple files, * _deleteByLike has to be used. * + * @see eZDFSFileHandler::fileDelete + * @see eZDFSFileHandler::delete + * @see _deleteInner + * @see _deleteByLike + * * @param string $filePath Path of the file to delete * @param bool $insideOfTransaction * Wether or not a transaction is already started * @param bool|string $fname Optional caller name for debugging - * @see _deleteInner - * @see _deleteByLike + * * @return bool */ public function _delete( $filePath, $insideOfTransaction = false, $fname = false ) @@ -366,16 +377,13 @@ public function _delete( $filePath, $insideOfTransaction = false, $fname = false $fname .= "::_delete($filePath)"; else $fname = "_delete($filePath)"; - // @todo Check if this is requried: _protec will already take care of + // @todo Check if this is required: _protect will already take care of // checking if a transaction is running. But leave it like this // for now. if ( $insideOfTransaction ) { - $res = $this->_deleteInner( $filePath, $fname ); - if ( !$res || $res instanceof eZMySQLBackendError ) - { - $this->_handleErrorType( $res ); - } + return $this->_deleteInner( $filePath, $fname ); + } else { @@ -394,23 +402,25 @@ public function _delete( $filePath, $insideOfTransaction = false, $fname = false protected function _deleteInner( $filePath, $fname ) { if ( !$this->_query( "UPDATE " . $this->dbTable( $filePath ) . " SET mtime=-ABS(mtime), expired=1 WHERE name_hash=" . $this->_md5( $filePath ), $fname ) ) - return $this->_fail( "Deleting file $filePath failed" ); + $this->_fail( "Deleting file $filePath failed" ); return true; } /** * Deletes multiple files using a SQL LIKE statement - * * Use _delete if you need to delete single files * + * @see eZDFSFileHandler::fileDelete + * @see _deleteByLikeInner + * @see _delete + * * @param string $like * SQL LIKE condition applied to ezdfsfile.name to look for files * to delete. Will use name_trunk if the LIKE string matches a * filetype that supports name_trunk. - * @param string $fname Optional caller name for debugging + * @param bool|string $fname Optional caller name for debugging + * * @return bool - * @see _deleteByLikeInner - * @see _delete */ public function _deleteByLike( $like, $fname = false ) { @@ -423,18 +433,23 @@ public function _deleteByLike( $like, $fname = false ) } /** - * Callback used by _deleteByLike to perform the deletion + * @see _deleteByLike * * @param string $like - * @param mixed $fname - * @return + * SQL LIKE condition applied to ezdfsfile.name to look for files + * to delete. Will use name_trunk if the LIKE string matches a + * filetype that supports name_trunk. + * @param bool|string $fname Optional caller name for debugging + * + * @return bool|void + * @throws Exception */ - private function _deleteByLikeInner( $like, $fname ) + protected function _deleteByLikeInner( $like, $fname ) { $sql = "UPDATE " . $this->dbTable( $like ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name like ". $this->_quote( $like ); if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by like: '$like'" ); + $this->_fail( "Failed to delete files by like: '$like'" ); } return true; } @@ -458,19 +473,19 @@ public function _deleteByRegex( $regex, $fname = false ) } /** - * Callback used by _deleteByRegex to perform the deletion + * Deletes DB files by using a SQL regular expression applied to file names * - * @param mixed $regex + * @param string $regex * @param mixed $fname - * @return - * @deprecated Has severe performances issues + * @return bool + * @deprecated Has severe performance issues */ - public function _deleteByRegexInner( $regex, $fname ) + protected function _deleteByRegexInner( $regex, $fname ) { $sql = "UPDATE " . $this->dbTable( $regex ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP " . $this->_quote( $regex ); if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by regex: '$regex'" ); + $this->_fail( "Failed to delete files by regex: '$regex'" ); } return true; } @@ -517,7 +532,7 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; if ( !$res = $this->_query( $sql, $fname ) ) { - return $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); + $this->_fail( "Failed to delete files by wildcard: '$wildcard'" ); } return true; } @@ -560,7 +575,7 @@ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, else $fname = "_exists($filePath)"; $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existance: ", true ); + $fname, "Failed to check file '$filePath' existence: ", true ); if ( $row === false ) return false; @@ -608,14 +623,17 @@ protected function __mkdir_p( $dir ) } /** - * Fetches the file $filePath from the database to its own name - * - * Saving $filePath locally with its original name, or $uniqueName if given - * - * @param string $filePath - * @param string $uniqueName Alternative name to save the file to - * @return string|bool the file physical path, or false if fetch failed - **/ + * Fetches the file $filePath from the database to its own name + * Saving $filePath locally with its original name, or $uniqueName if given + * + * @see eZDFSFileHandler::fileFetch + * @see eZDFSFileHandler::fetchUnique + * + * @param string $filePath + * @param bool|string $uniqueName Alternative name to save the file to + * + * @return string|bool the file physical path, or false if fetch failed + */ public function _fetch( $filePath, $uniqueName = false ) { $metaData = $this->_fetchMetadata( $filePath ); @@ -703,16 +721,19 @@ public function _fetchContents( $filePath, $fname = false ) // @todo Catch an exception if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) { - eZDebug::writeError("An error occured while reading contents of DFS://$filePath", __METHOD__ ); + eZDebug::writeError("An error occurred while reading contents of DFS://$filePath", __METHOD__ ); return false; } return $contents; } /** - * Fetches and returns metadata for $filePath - * @return array|false file metadata, or false if the file does not exist in - * database. + * Fetches and returns metadata for $filePath + * + * @see eZDFSFileHandler::loadMetaData + * @param string $filePath + * @param bool|string $fname Optional caller name for debugging + * @return array|false file metadata, or false if the file does not exist in database. */ function _fetchMetadata( $filePath, $fname = false ) { @@ -736,11 +757,16 @@ public function _linkCopy( $srcPath, $dstPath, $fname = false ) } /** - * Passes $filePath content through - * @param string $filePath - * @deprecated should not be used since it cannot handle reading errors - **/ - public function _passThrough( $filePath, $fname = false ) + * Passes $filePath content through + * + * @param string $filePath + * @param int $startOffset Byte offset to start download from + * @param int|bool $length Byte length to be sent + * @param bool|string $fname Optional caller name for debugging + * + * @return bool + */ + public function _passThrough( $filePath, $startOffset = 0, $length = false, $fname = false ) { if ( $fname ) $fname .= "::_passThrough($filePath)"; @@ -753,7 +779,7 @@ public function _passThrough( $filePath, $fname = false ) return false; // @todo Catch an exception - $this->dfsbackend->passthrough( $filePath ); + $this->dfsbackend->passthrough( $filePath, $startOffset, $length ); return true; } @@ -768,7 +794,7 @@ public function _passThrough( $filePath, $fname = false ) public function _rename( $srcFilePath, $dstFilePath ) { if ( strcmp( $srcFilePath, $dstFilePath ) == 0 ) - return; + return false; // fetch source file metadata $metaData = $this->_fetchMetadata( $srcFilePath ); @@ -779,7 +805,6 @@ public function _rename( $srcFilePath, $dstFilePath ) $this->_begin( __METHOD__ ); - $srcFilePathStr = $this->_quote( $srcFilePath ); $dstFilePathStr = $this->_quote( $dstFilePath ); $dstNameTrunkStr = $this->_quote( self::nameTrunk( $dstFilePath, $metaData['scope'] ) ); @@ -812,7 +837,7 @@ public function _rename( $srcFilePath, $dstFilePath ) if ( !$this->dfsbackend->copyFromDFSToDFS( $srcFilePath, $dstFilePath ) ) { - return $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); + $this->_fail( "Failed to copy DFS://$srcFilePath to DFS://$dstFilePath" ); } // Remove old entry @@ -837,27 +862,29 @@ public function _rename( $srcFilePath, $dstFilePath ) /** * Stores $filePath to cluster * + * @see eZDFSFileHandler::fileStore + * * @param string $filePath * @param string $datatype * @param string $scope - * @param string $fname - * @return void + * @param bool|string $fname Optional caller name for debugging + * + * @return bool */ function _store( $filePath, $datatype, $scope, $fname = false ) { if ( !is_readable( $filePath ) ) { eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); - return; + return false; } if ( $fname ) $fname .= "::_store($filePath, $datatype, $scope)"; else $fname = "_store($filePath, $datatype, $scope)"; - $return = $this->_protect( array( $this, '_storeInner' ), $fname, + return $this->_protect( array( $this, '_storeInner' ), $fname, $filePath, $datatype, $scope, $fname ); - return $return; } /** @@ -890,13 +917,13 @@ function _storeInner( $filePath, $datatype, $scope, $fname ) array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), $fname ) === false ) { - return $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); + $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); } // copy given $filePath to DFS if ( !$this->dfsbackend->copyToDFS( $filePath ) ) { - return $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); + $this->_fail( "Failed to copy FS://$filePath to DFS://$filePath" ); } return true; @@ -905,13 +932,17 @@ function _storeInner( $filePath, $datatype, $scope, $fname ) /** * Stores $contents as the contents of $filePath to the cluster * + * @see eZDFSFileHandler::fileStore + * @see eZDFSFileHandler::storeContents + * * @param string $filePath * @param string $contents * @param string $scope * @param string $datatype - * @param int $mtime - * @param string $fname - * @return void + * @param bool|int $mtime + * @param bool|string $fname Optional caller name for debugging + * + * @return bool */ function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false, $fname = false ) { @@ -932,7 +963,6 @@ function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $ $nameTrunk = self::nameTrunk( $filePath, $scope ); if ( $mtime === false ) $mtime = time(); - $expired = ( $mtime < 0 ) ? '1' : '0'; // Copy file metadata. $result = $this->_insertUpdate( @@ -950,18 +980,35 @@ function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $ ); if ( $result === false ) { - return $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); + $this->_fail( "Failed to insert file metadata while storing contents. Possible race condition", $result ); } if ( !$this->dfsbackend->createFileOnDFS( $filePath, $contents ) ) { - return $this->_fail( "Failed to open DFS://$filePath for writing" ); + $this->_fail( "Failed to open DFS://$filePath for writing" ); } return true; } - public function _getFileList( $scopes = false, $excludeScopes = false ) + /** + * Gets the list of cluster files, filtered by the optional params + * + * @see eZDFSFileHandler::getFileList + * + * @param array|bool $scopes filter by array of scopes to include in the list + * @param bool $excludeScopes if true, $scopes param acts as an exclude filter + * @param array|bool $limit limits the search to offset limit[0], limit limit[1] + * @param string|bool $path filter to include entries only including $path + * + * @return array|false the db list of entries of false if none found + */ + public function _getFileList( + $scopes = false, + $excludeScopes = false, + $limit = false, + $path = false + ) { $filePathList = array(); $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); @@ -1016,13 +1063,16 @@ protected function _die( $msg, $sql = null ) } /** - * Performs an insert of the given items in $array. - * @param string $table Name of table to execute insert on. - * @param array $array Associative array with data to insert, the keys are - * the field names and the values will be quoted - * according to type. - * @param string $fname Name of caller function (for logging purpuse) - **/ + * Performs an insert of the given items in $array. + * + * @param string $table Name of table to execute insert on. + * @param array $array Associative array with data to insert, the keys are + * the field names and the values will be quoted + * according to type. + * @param string $fname Name of caller function + * + * @return bool + */ function _insert( $table, $array, $fname ) { $keys = array_keys( $array ); @@ -1030,28 +1080,30 @@ function _insert( $table, $array, $fname ) $res = $this->_query( $query, $fname ); if ( !$res ) { - // @todo Throw an exception return false; } + return true; } /** - * Performs an insert of the given items in $insert. - * - * If entry specified already exists, fields in $update are updated with the values from $insert - * - * @param string $table Name of table to execute insert on. - * @param array $insert Associative array with data to insert, the keys - * are the field names and the values are the quoted field values - * @param string $update Array of fields that must be updated if an entry exists - * @param string $fname Name of caller function (for logging purpuse) - * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert - **/ + * Performs an insert of the given items in $insert. + * + * If entry specified already exists, fields in $update are updated with the values from $insert + * + * @param string $table Name of table to execute insert on. + * @param array $insert Associative array with data to insert, the keys + * are the field names and the values are the quoted field values + * @param array $update Array of fields that must be updated if an entry exists + * @param string $fname Name of caller function + * @param bool $reportError + * + * @throws InvalidArgumentException when either name or name_hash aren't provided in $insert + */ protected function _insertUpdate( $table, $insert, $update, $fname, $reportError = true ) { if ( !isset( $insert['name'] ) || !isset( $insert['name_hash'] ) ) { - throw new InvalidArgumentException( "Insert array must contain both name and name_hash" ); + $this->_fail( "Insert array must contain both name and name_hash" ); } if ( $row = $this->_fetchMetadata( $insert['name'] ) ) @@ -1078,12 +1130,15 @@ protected function _insertUpdate( $table, $insert, $update, $fname, $reportError "VALUES( " . implode( ', ', $quotedValues ) . ")"; } - try { - $stmt = $this->_query( $sql, $fname, $reportError ); - } catch( PDOException $e ) { + try + { + $this->_query( $sql, $fname, $reportError ); + } + catch ( PDOException $e ) + { $this->_fail( "Failed insert/updating: " . $e->getMessage() ); - return false; } + return true; } /** @@ -1107,18 +1162,18 @@ protected function _sqlList( $array ) } /** - * Runs a select query and returns one numeric indexed row from the result - * If there are more than one row it will fail and exit, if 0 it returns - * false. - * - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition - * to the SQL. - * @return array|false - **/ + * Runs a select query and returns one numeric indexed row from the result + * If there are more than one row it will fail and exit, if 0 it returns + * false. + * + * @param string $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param bool|string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition + * to the SQL. + * @return array|false + **/ protected function _selectOneRow( $query, $fname, $error = false, $debug = false ) { return $this->_selectOne( $query, $fname, $error, $debug, PDO::FETCH_NUM ); @@ -1133,7 +1188,7 @@ protected function _selectOneRow( $query, $fname, $error = false, $debug = false * @param string $query * @param string $fname The function name that started the query, should * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors + * @param bool|string $error Sent to _error() in case of errors * @param bool $debug If true it will display the fetched row in addition * to the SQL. * @return array|false @@ -1144,19 +1199,21 @@ protected function _selectOneAssoc( $query, $fname, $error = false, $debug = fal } /** - * Runs a select query, applying the $fetchCall callback to one result - * If there are more than one row it will fail and exit, if 0 it returns false. - * - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error Sent to _error() in case of errors - * @param bool $debug If true it will display the fetched row in addition to the SQL. - * @param callback $fetchCall The callback to fetch the row. - * @return mixed - **/ + * Runs a select query, applying the $fetchCall callback to one result + * If there are more than one row it will fail and exit, if 0 it returns false. + * + * @param $query + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param bool|string $error Sent to _error() in case of errors + * @param bool $debug If true it will display the fetched row in addition to the SQL. + * @param int $fetchCall The callback to fetch the row. + * + * @return mixed + **/ protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgresSQL Cluster', 'DB queries' ); $time = microtime( true ); $stmt = $this->db->query( $query ); @@ -1197,12 +1254,8 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ * Starts a new transaction by executing a BEGIN call. * If a transaction is already started nothing is executed. **/ - protected function _begin( $fname = false ) + protected function _begin() { - if ( $fname ) - $fname .= "::_begin"; - else - $fname = "_begin"; $this->transactionCount++; if ( $this->transactionCount == 1 ) $this->db->beginTransaction(); @@ -1212,12 +1265,8 @@ protected function _begin( $fname = false ) * Stops a current transaction and commits the changes by executing a COMMIT call. * If the current transaction is a sub-transaction nothing is executed. **/ - protected function _commit( $fname = false ) + protected function _commit() { - if ( $fname ) - $fname .= "::_commit"; - else - $fname = "_commit"; $this->transactionCount--; if ( $this->transactionCount == 0 ) $this->db->commit(); @@ -1228,12 +1277,8 @@ protected function _commit( $fname = false ) * ROLLBACK call. * If the current transaction is a sub-transaction nothing is executed. **/ - protected function _rollback( $fname = false ) + protected function _rollback() { - if ( $fname ) - $fname .= "::_rollback"; - else - $fname = "_rollback"; $this->transactionCount--; if ( $this->transactionCount == 0 ) $this->db->rollBack(); @@ -1253,6 +1298,7 @@ protected function _rollback( $fname = false ) **/ protected function _protect() { + $result = false; $args = func_get_args(); $callback = array_shift( $args ); $fname = array_shift( $args ); @@ -1268,34 +1314,14 @@ protected function _protect() } catch( PDOException $e ) { - print_r( compact( 'callback', 'args' ) ); eZDebug::writeError( $e ); return false; } - - /*// @todo Investigate the right function - $errno = pg_result_error( $result, PGSQL_DIAG_SQLSTATE ); - if ( $errno == 1205 || // Error: 1205 SQLSTATE: HY000 (ER_LOCK_WAIT_TIMEOUT) - $errno == 1213 ) // Error: 1213 SQLSTATE: 40001 (ER_LOCK_DEADLOCK) - { - $tries++; - $this->_rollback( $fname ); - continue; - } - - // @todo replace with an exception - if ( $result === false ) + catch( Exception $e ) { - $this->_rollback( $fname ); + eZDebug::writeError( $e ); return false; } - elseif ( $result instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $result->errorValue, $result->errorText ); - $this->_rollback( $fname ); - return false; - }*/ - break; // All is good, so break out of loop } @@ -1303,39 +1329,13 @@ protected function _protect() return $result; } - protected function _handleErrorType( $res ) - { - if ( $res === false ) - { - eZDebug::writeError( "SQL failed" ); - } - elseif ( $res instanceof eZMySQLBackendError ) - { - eZDebug::writeError( $res->errorValue, $res->errorText ); - } - } - - /** - * Checks if $result is a failure type and returns true if so, false - * otherwise. - * - * A failure is either the value false or an error object of type - * eZMySQLBackendError. - **/ - protected function _isFailure( $result ) - { - if ( $result === false || ($result instanceof eZMySQLBackendError ) ) - { - return true; - } - return false; - } - /** - * Creates an error object which can be read by some backend functions. - * @param mixed $value The value which is sent to the debug system. - * @param PDOStatement $result The failed SQL result - **/ + * Creates an error object which can be read by some backend functions. + * + * @param mixed $message The value which is sent to the debug system. + * @param PDOStatement|bool $result The failed SQL result + * @throws Exception + **/ protected function _fail( $message, $result = false) { // @todo Investigate the right function @@ -1352,14 +1352,16 @@ protected function _fail( $message, $result = false) } /** - * Performs mysql query and returns mysql result. - * Times the sql execution, adds accumulator timings and reports SQL to - * debug. - * @param string $query - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @return PDOStatement The resulting PDOStatement object, or false if an error occured - **/ + * Performs mysql query and returns mysql result. + * Times the sql execution, adds accumulator timings and reports SQL to + * debug. + * + * @param string $query + * @param bool|string $fname Optional caller name for debugging + * @param bool $reportError + * + * @return PDOStatement The resulting PDOStatement object, or false if an error occurred + **/ protected function _query( $query, $fname = false, $reportError = true ) { eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); @@ -1384,7 +1386,7 @@ protected function _query( $query, $fname = false, $reportError = true ) } /** - * Make sure that $value is escaped and qouted according to type and returned + * Make sure that $value is escaped and quoted according to type and returned * as a string. * * @param string $value a SQL parameter to escape @@ -1407,7 +1409,10 @@ protected function _quote( $value ) /** * Provides the SQL calls to convert $value to MD5 * The returned value can directly be put into SQLs. - **/ + * @param $value + * + * @return string + */ protected function _md5( $value ) { return $this->_quote( md5( $value ) ); @@ -1417,7 +1422,7 @@ protected function _md5( $value ) * Prints error message $error to debug system. * @param string $query The query that was attempted, will be printed if * $error is \c false - * @param resource $res The result resource the error occured on + * @param PDOStatement|resource $res The result resource the error occurred on * @param string $fname The function name that started the query, should * contain relevant arguments in the text. * @param string $error The error message, if this is an array the first @@ -1438,16 +1443,18 @@ protected function _error( $query, $res, $fname, $error = "Failed to execute SQL } // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ), $fname ); + eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ) . ' ' .$query, $fname ); } /** - * Report SQL $query to debug system. - * - * @param string $fname The function name that started the query, should contain relevant arguments in the text. - * @param int $timeTaken Number of seconds the query + related operations took (as float). - * @param int $numRows Number of affected rows. - **/ + * Report SQL $query to debug system. + * + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param string $fname The function name that started the query, should contain relevant arguments in the text. + * @param int $timeTaken Number of seconds the query + related operations took (as float). + * @param int|bool $numRows Number of affected rows. + **/ function _report( $query, $fname, $timeTaken, $numRows = false ) { if ( !self::$dbparams['sql_output'] ) @@ -1464,19 +1471,23 @@ function _report( $query, $fname, $timeTaken, $numRows = false ) } /** - * Attempts to begin cache generation by creating a new file named as the - * given filepath, suffixed with .generating. If the file already exists, - * insertion is not performed and false is returned (means that the file - * is already being generated) - * @param string $filePath - * @return array array with 2 indexes: 'result', containing either ok or ko, - * and another index that depends on the result: - * - if result == 'ok', the 'mtime' index contains the generating - * file's mtime - * - if result == 'ko', the 'remaining' index contains the remaining - * generation time (time until timeout) in seconds - * @throws RuntimeException - **/ + * Attempts to begin cache generation by creating a new file named as the + * given filepath, suffixed with .generating. If the file already exists, + * insertion is not performed and false is returned (means that the file + * is already being generated) + * + * @see eZDFSFileHandler::startCacheGeneration + * + * @param string $filePath + * @param string $generatingFilePath + * + * @return array array with 2 indexes: 'result', containing either ok or ko, + * and another index that depends on the result: + * - if result == 'ok', the 'mtime' index contains the generating + * file's mtime + * - if result == 'ko', the 'remaining' index contains the remaining + * generation time (time until timeout) in seconds + */ public function _startCacheGeneration( $filePath, $generatingFilePath ) { $fname = "_startCacheGeneration( {$filePath} )"; @@ -1494,12 +1505,15 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) $query = 'INSERT INTO ' . $this->dbTable( $filePath ) . ' ( '. implode(', ', array_keys( $insertData ) ) . ' ) ' . "VALUES(" . implode( ', ', $insertData ) . ")"; - //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch + //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch @todo //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; - try { + try + { $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); - } catch( PDOException $e ) { + } + catch( PDOException $e ) + { $errno = $e->getCode(); if ( $errno != self::ERROR_UNIQUE_VIOLATION ) { @@ -1508,7 +1522,7 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) // error self::ERROR_UNIQUE_VIOLATION is expected, since it means duplicate key (file is being generated) else { - // generation timout check + // generation timeout check $query = "SELECT mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash = {$nameHash}"; $row = $this->_selectOneRow( $query, $fname, false, false ); @@ -1521,7 +1535,7 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) { $previousMTime = $row[0]; - eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timedout, taking over", __METHOD__ ); + eZDebugSetting::writeDebug( 'kernel-clustering', "$filePath generation has timed out, taking over", __METHOD__ ); $updateQuery = "UPDATE " . $this->dbTable( $filePath ) . " SET mtime = {$mtime} WHERE name_hash = {$nameHash} AND mtime = {$previousMTime}"; // we run the query manually since the default _query won't @@ -1533,8 +1547,8 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) } else { - throw new RuntimeException( "An error occured taking over timedout generating cache file $generatingFilePath" ); - return array( 'result' => 'error' ); + throw new RuntimeException( "An error occurred taking over timed out generating cache file $generatingFilePath" ); + //return array( 'result' => 'error' ); } } else @@ -1548,12 +1562,19 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) } /** - * Ends the cache generation for the current file: moves the (meta)data for - * the .generating file to the actual file, and removed the .generating - * @param string $filePath - * @return bool - * @throws RuntimeException - **/ + * Ends the cache generation for the current file: moves the (meta)data for + * the .generating file to the actual file, and removed the .generating + * + * @see eZDFSFileHandler::endCacheGeneration + * + * @param string $filePath + * @param string $generatingFilePath + * @param bool $rename if false the .generating entry is just deleted + * + * @return bool true + * + * @throw RuntimeException + */ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) { $fname = "_endCacheGeneration( $filePath )"; @@ -1575,7 +1596,7 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$stmt = $this->_query( "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ) . " FOR UPDATE", $fname, true ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occcured getting a lock on $generatingFilePath" ); + throw new RuntimeException( "An error occurred getting a lock on $generatingFilePath" ); } $generatingMetaData = $stmt->fetch( PDO::FETCH_ASSOC ); @@ -1592,14 +1613,14 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$this->_query( $insertSQL, $fname, true ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occured creating the metadata entry for $filePath" ); + throw new RuntimeException( "An error occurred creating the metadata entry for $filePath" ); } // here we rename the actual FILE. The .generating file has been // created on DFS, and should be renamed if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) { $this->_rollback( $fname ); - throw new RuntimeException("An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + throw new RuntimeException("An error occurred renaming DFS://$generatingFilePath to DFS://$filePath" ); } $this->_query( "DELETE FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $generatingFilePath ), $fname, true ); } @@ -1610,7 +1631,7 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) if ( !$this->dfsbackend->renameOnDFS( $generatingFilePath, $filePath ) ) { $this->_rollback( $fname ); - throw new RuntimeException( "An error occured renaming DFS://$generatingFilePath to DFS://$filePath" ); + throw new RuntimeException( "An error occurred renaming DFS://$generatingFilePath to DFS://$filePath" ); } $mtime = $generatingMetaData['mtime']; @@ -1673,11 +1694,10 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi $query = "SELECT mtime FROM " . $this->dbTable( $generatingFilePath ) . " WHERE name_hash = {$nameHash}"; $stmt = $this->db->query( $query ); $row = $stmt->fetch( PDO::FETCH_NUM ); - if ( isset( $row[0] ) and $row[0] == $generatingFileMtime ); + if ( isset( $row[0] ) && $row[0] == $generatingFileMtime ) { return true; } - // @todo Check if an exception makes sense here return false; } @@ -1694,11 +1714,14 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi } /** - * Aborts the cache generation process by removing the .generating file - * @param string $filePath Real cache file path - * @param string $generatingFilePath .generating cache file path - * @return void - **/ + * Aborts the cache generation process by removing the .generating file + * + * @see eZDFSFileHandler::abortCacheGeneration + * + * @param string $generatingFilePath .generating cache file path + * + * @return void + */ public function _abortCacheGeneration( $generatingFilePath ) { $fname = "_abortCacheGeneration( $generatingFilePath )"; @@ -1752,13 +1775,13 @@ static protected function nameTrunk( $filePath, $scope ) } /** - * Returns the remaining time, in seconds, before the generating file times - * out - * - * @param resource $fileRow - * - * @return int Remaining generation seconds. A negative value indicates a timeout. - **/ + * Returns the remaining time, in seconds, before the generating file times + * out + * + * @param array $row + * + * @return int Remaining generation seconds. A negative value indicates a timeout. + **/ protected function remainingCacheGenerationTime( $row ) { if( !isset( $row[0] ) ) @@ -1770,14 +1793,17 @@ protected function remainingCacheGenerationTime( $row ) /** * Returns the list of expired files * + * @see eZDFSFileHandler::fetchExpiredItems + * * @param array $scopes Array of scopes to consider. At least one. - * @param int $limit Max number of items. Set to false for unlimited. + * @param array|bool $limit Max number of items. Set to false for unlimited. + * @param int|bool $expiry Number of seconds, only items older than this will be returned. * * @return array(filepath) * * @since 4.3 */ - public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) + public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = false ) { $tables = array( $this->metaDataTable, $this->metaDataTableCache ); @@ -1804,7 +1830,7 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ) ) return $filePathList; } - + public function applyServerUri( $filePath ) { return $this->dfsbackend->applyServerUri( $filePath ); @@ -1830,7 +1856,7 @@ public function deleteCacheFiles( $limit ) $like = addcslashes( eZSys::cacheDirectory(), '_' ) . DIRECTORY_SEPARATOR . '%'; $query = "DELETE FROM {$this->metaDataTable} WHERE name LIKE '$like' LIMIT $limit"; - if ( !$stmt = $this->_query( $sql ) ) + if ( !$stmt = $this->_query( $query ) ) { throw new RuntimeException( "Error in $query" ); } @@ -1842,7 +1868,7 @@ public function deleteCacheFiles( $limit ) /** * DB connexion handle - * @var PDO + * @var PDO|resource */ public $db = null; @@ -1861,7 +1887,7 @@ public function deleteCacheFiles( $limit ) /** * Current transaction level. * Will be used to decide wether we can BEGIN (if it's the first BEGIN call) - * or COMMIT (if we're commiting the last running transaction + * or COMMIT (if we're committing the last running transaction * @var int */ protected $transactionCount = 0; From 5cd7c1f917c0b4f5bd2e93f3c81c19f06312febc Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 23:09:19 +0200 Subject: [PATCH 09/21] fix phpdoc --- clustering/dfs/ezpostsgresqlbackend.php | 76 +++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 5 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index e1abc77..1ee141d 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -12,7 +12,7 @@ * This class allows DFS based clustering using PostgresSQL * @package Cluster */ -class eZDFSFileHandlerPostgresqlBackend implements eZDFSFileHandlerDBBackendInterface +class eZDFSFileHandlerPostgresqlBackend { /** @@ -143,6 +143,9 @@ public function _connect() /** * Disconnects the handler from the database + * + * @see eZDFSFileHandler::disconnect + * @return void */ public function _disconnect() { @@ -233,16 +236,18 @@ protected function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) /** * Purges meta-data and file-data for a file entry - * * Will only expire a single file. Use _purgeByLike to purge multiple files * + * @see eZDFSFileHandler::purge + * @see _purgeByLike + * * @param string $filePath Path of the file to purge * @param bool $onlyExpired Only purges expired files * @param bool|int $expiry - * @param bool $fname + * @param bool|string $fname Optional caller name for debugging * - * @see _purgeByLike - **/ + * @return bool + */ public function _purge( $filePath, $onlyExpired = false, $expiry = false, $fname = false ) { if ( $fname ) @@ -537,6 +542,19 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) return true; } + /** + * Deletes a list of files based on directory / filename components + * + * @see eZDFSFileHandler::fileDeleteByDirList + * + * @param array $dirList Array of directory that will be prefixed with + * $commonPath when looking for files + * @param string $commonPath Starting path common to every delete request + * @param string $commonSuffix Suffix appended to every delete request + * @param bool|string $fname Optional caller name for debugging + * + * @return bool + */ public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) { if ( $fname ) @@ -568,6 +586,19 @@ protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, return true; } + /** + * Check if given file/dir exists. + * + * @see eZDFSFileHandler::fileExists + * @see eZDFSFileHandler::exists + * + * @param $filePath + * @param bool|string $fname Optional caller name for debugging + * @param bool $ignoreExpiredFiles ignore ezdfsfile.mtime + * @param bool $checkOnDFS Checks if a file exists on the DFS + * + * @return bool + */ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, $checkOnDFS = false ) { if ( $fname ) @@ -704,6 +735,17 @@ public function _fetch( $filePath, $uniqueName = false ) return false; } + /** + * Returns file contents. + * + * @see eZDFSFileHandler::fileFetchContents + * @see eZDFSFileHandler::fetchContents + * + * @param string $filePath + * @param bool|string $fname Optional caller name for debugging + * + * @return string|bool contents string, or false in case of an error. + */ public function _fetchContents( $filePath, $fname = false ) { if ( $fname ) @@ -747,6 +789,17 @@ function _fetchMetadata( $filePath, $fname = false ) true ); } + /** + * Create symbolic or hard link to file. Alias of copy + * + * @see eZDFSFileHandler::fileLinkCopy + * + * @param string $srcPath Source file + * @param string $dstPath Destination file + * @param bool|string $fname Optional caller name for debugging + * + * @return mixed + */ public function _linkCopy( $srcPath, $dstPath, $fname = false ) { if ( $fname ) @@ -787,8 +840,12 @@ public function _passThrough( $filePath, $startOffset = 0, $length = false, $fna /** * Renames $srcFilePath to $dstFilePath * + * @see eZDFSFileHandler::fileMove + * @see eZDFSFileHandler::move + * * @param string $srcFilePath * @param string $dstFilePath + * * @return bool */ public function _rename( $srcFilePath, $dstFilePath ) @@ -1831,6 +1888,15 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = f return $filePathList; } + /** + * Transforms $filePath so that it contains a valid href to the file, wherever it is stored. + * + * @see eZDFSFileHandler::applyServerUri + * + * @param string $filePath + * + * @return string + */ public function applyServerUri( $filePath ) { return $this->dfsbackend->applyServerUri( $filePath ); From aacd7c3b78e562b94bbe1c4f9ecd6ddafca26f2d Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 7 Jul 2015 23:30:50 +0200 Subject: [PATCH 10/21] minor bugfixes --- clustering/dfs/ezpostsgresqlbackend.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 1ee141d..392dda8 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -558,9 +558,9 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = false ) { if ( $fname ) - $fname .= "::_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + $fname .= "::_deleteByDirList(" . implode( ' ',$dirList ) . ", $commonPath, $commonSuffix)"; else - $fname = "_deleteByDirList($dirList, $commonPath, $commonSuffix)"; + $fname = "_deleteByDirList(" . implode( ' ',$dirList ) . ", $commonPath, $commonSuffix)"; return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, $dirList, $commonPath, $commonSuffix, $fname ); } @@ -1270,7 +1270,7 @@ protected function _selectOneAssoc( $query, $fname, $error = false, $debug = fal **/ protected function _selectOne( $query, $fname, $error = false, $debug = false, $fetchCall ) { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgresSQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); $time = microtime( true ); $stmt = $this->db->query( $query ); From f53ae7cc830ce3494842c2c71fa91e2ef1c3d1ce Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Fri, 17 Feb 2017 15:55:06 +0100 Subject: [PATCH 11/21] fix scope varchar limit --- sql/postgresql/cluster_dfs_schema.sql | 5 ++--- sql/update/schema.sql | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 sql/update/schema.sql diff --git a/sql/postgresql/cluster_dfs_schema.sql b/sql/postgresql/cluster_dfs_schema.sql index 46cc92e..2d416c9 100644 --- a/sql/postgresql/cluster_dfs_schema.sql +++ b/sql/postgresql/cluster_dfs_schema.sql @@ -3,7 +3,7 @@ CREATE TABLE ezdfsfile ( name text NOT NULL, name_trunk text NOT NULL, name_hash character(32) DEFAULT '' NOT NULL PRIMARY KEY, - scope varchar(20) DEFAULT '' NOT NULL, + scope varchar(100) DEFAULT '' NOT NULL, size integer DEFAULT 0 NOT NULL, mtime integer DEFAULT 0 NOT NULL, expired integer DEFAULT 0 NOT NULL @@ -17,7 +17,7 @@ CREATE TABLE ezdfsfile_cache ( name text NOT NULL, name_trunk text NOT NULL, name_hash character(32) DEFAULT '' NOT NULL PRIMARY KEY, - scope varchar(20) DEFAULT '' NOT NULL, + scope varchar(100) DEFAULT '' NOT NULL, size integer DEFAULT 0 NOT NULL, mtime integer DEFAULT 0 NOT NULL, expired integer DEFAULT 0 NOT NULL @@ -25,4 +25,3 @@ CREATE TABLE ezdfsfile_cache ( CREATE INDEX ezdfsfile_cache_name ON ezdfsfile_cache ( name ); CREATE INDEX ezdfsfile_cache_mtime ON ezdfsfile_cache ( mtime ); - diff --git a/sql/update/schema.sql b/sql/update/schema.sql new file mode 100644 index 0000000..a707dbe --- /dev/null +++ b/sql/update/schema.sql @@ -0,0 +1,2 @@ +ALTER TABLE ezdfsfile ALTER COLUMN scope TYPE varchar(100); +ALTER TABLE ezdfsfile_cache ALTER COLUMN scope TYPE varchar(100); From 7aee74accafba57de7bb5b771f0839ef5183fd2e Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Fri, 17 Feb 2017 16:00:48 +0100 Subject: [PATCH 12/21] Revert "fix scope varchar limit" This reverts commit f53ae7cc830ce3494842c2c71fa91e2ef1c3d1ce. --- sql/postgresql/cluster_dfs_schema.sql | 5 +++-- sql/update/schema.sql | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) delete mode 100644 sql/update/schema.sql diff --git a/sql/postgresql/cluster_dfs_schema.sql b/sql/postgresql/cluster_dfs_schema.sql index 2d416c9..46cc92e 100644 --- a/sql/postgresql/cluster_dfs_schema.sql +++ b/sql/postgresql/cluster_dfs_schema.sql @@ -3,7 +3,7 @@ CREATE TABLE ezdfsfile ( name text NOT NULL, name_trunk text NOT NULL, name_hash character(32) DEFAULT '' NOT NULL PRIMARY KEY, - scope varchar(100) DEFAULT '' NOT NULL, + scope varchar(20) DEFAULT '' NOT NULL, size integer DEFAULT 0 NOT NULL, mtime integer DEFAULT 0 NOT NULL, expired integer DEFAULT 0 NOT NULL @@ -17,7 +17,7 @@ CREATE TABLE ezdfsfile_cache ( name text NOT NULL, name_trunk text NOT NULL, name_hash character(32) DEFAULT '' NOT NULL PRIMARY KEY, - scope varchar(100) DEFAULT '' NOT NULL, + scope varchar(20) DEFAULT '' NOT NULL, size integer DEFAULT 0 NOT NULL, mtime integer DEFAULT 0 NOT NULL, expired integer DEFAULT 0 NOT NULL @@ -25,3 +25,4 @@ CREATE TABLE ezdfsfile_cache ( CREATE INDEX ezdfsfile_cache_name ON ezdfsfile_cache ( name ); CREATE INDEX ezdfsfile_cache_mtime ON ezdfsfile_cache ( mtime ); + diff --git a/sql/update/schema.sql b/sql/update/schema.sql deleted file mode 100644 index a707dbe..0000000 --- a/sql/update/schema.sql +++ /dev/null @@ -1,2 +0,0 @@ -ALTER TABLE ezdfsfile ALTER COLUMN scope TYPE varchar(100); -ALTER TABLE ezdfsfile_cache ALTER COLUMN scope TYPE varchar(100); From 643449cc71601652128b06a41eb5b41f8b763efe Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Mon, 6 Nov 2017 14:25:54 +0100 Subject: [PATCH 13/21] add composer.json --- composer.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 composer.json diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..8decb2c --- /dev/null +++ b/composer.json @@ -0,0 +1,13 @@ +{ + "name": "opencontent/ezpostgresqlcluster-ls", + "description": "PostgreSQL Cluster", + "type": "ezpublish-legacy-extension", + "license": "GPL-2.0", + "minimum-stability": "dev", + "require": { + "ezsystems/ezpublish-legacy-installer": "*" + }, + "extra": { + "ezpublish-legacy-extension-name": "ezpostgresqlcluster" + } +} From e3abceebd570195a8f45ff470f09a4a885b4f2e4 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Wed, 8 Nov 2017 17:37:34 +0100 Subject: [PATCH 14/21] fix error logging --- clustering/dfs/ezpostsgresqlbackend.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 392dda8..cd7fc8e 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -1371,12 +1371,12 @@ protected function _protect() } catch( PDOException $e ) { - eZDebug::writeError( $e ); + eZDebug::writeError( $e->getMessage(), __METHOD__ ); return false; } catch( Exception $e ) { - eZDebug::writeError( $e ); + eZDebug::writeError( $e->getMessage(), __METHOD__ ); return false; } break; // All is good, so break out of loop From fe77a92f49a4443bceef1b9691dbb25ec9e257d8 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 19 Dec 2017 00:32:37 +0100 Subject: [PATCH 15/21] Write error log per siteaccess --- clustering/dfs/ezpostsgresqlbackend.php | 275 +++++++++++++----------- 1 file changed, 145 insertions(+), 130 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index cd7fc8e..167fdee 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -27,8 +27,23 @@ class eZDFSFileHandlerPostgresqlBackend */ protected $maxCopyTries; + protected static function writeError($string, $label = "", $backgroundClass = "") + { + $logName = 'cluster_error.log'; + if ( isset( $GLOBALS['eZCurrentAccess']['name'] ) ){ + $logName = $GLOBALS['eZCurrentAccess']['name'] . '_cluster_error.log'; + } + + if ($label){ + $message = "[$label] $string"; + }else{ + $message = $string; + } + eZLog::write($message, $logName); + } + public function __construct() - { + { $this->eventHandler = ezpEvent::getInstance(); $fileINI = eZINI::instance( 'file.ini' ); $this->maxCopyTries = (int)$fileINI->variable( 'eZDFSClusteringSettings', 'MaxCopyRetries' ); @@ -45,7 +60,7 @@ public function __construct() $this->cacheDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'CacheDir' ); $this->storageDir = eZINI::instance( 'site.ini' )->variable( 'FileSettings', 'StorageDir' ); } - + /** * Returns the database table name to use for the specified file. * @@ -67,7 +82,7 @@ protected function dbTable( $filePath ) return $this->metaDataTable; } - + /** * Connects to the database. * @@ -116,7 +131,7 @@ public function _connect() try { $this->db = new PDO( $connectString, self::$dbparams['user'], self::$dbparams['pass'] ); } catch ( PDOException $e ) { - eZDebug::writeError( $e->getMessage() ); + self::writeError( $e->getMessage() ); ++$tries; continue; } @@ -185,7 +200,7 @@ public function _copy( $srcFilePath, $dstFilePath, $fname = false ) return false; } return $this->_protect( array( $this, "_copyInner" ), $fname, - $srcFilePath, $dstFilePath, $fname, $metaData ); + $srcFilePath, $dstFilePath, $fname, $metaData ); } /** @@ -212,16 +227,16 @@ protected function _copyInner( $srcFilePath, $dstFilePath, $fname, $metaData ) // Copy file metadata. if ( $this->_insertUpdate( $this->dbTable( $dstFilePath ), - array( 'datatype'=> $datatype, - 'name' => $dstFilePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) + array( 'datatype'=> $datatype, + 'name' => $dstFilePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) { $this->_fail( $srcFilePath, "Failed to insert file metadata on copying." ); } @@ -393,7 +408,7 @@ public function _delete( $filePath, $insideOfTransaction = false, $fname = false else { return $this->_protect( array( $this, '_deleteInner' ), $fname, - $filePath, $insideOfTransaction, $fname ); + $filePath, $insideOfTransaction, $fname ); } } @@ -434,7 +449,7 @@ public function _deleteByLike( $like, $fname = false ) else $fname = "_deleteByLike($like)"; return $this->_protect( array( $this, '_deleteByLikeInner' ), $fname, - $like, $fname ); + $like, $fname ); } /** @@ -474,7 +489,7 @@ public function _deleteByRegex( $regex, $fname = false ) else $fname = "_deleteByRegex($regex)"; return $this->_protect( array( $this, '_deleteByRegexInner' ), $fname, - $regex, $fname ); + $regex, $fname ); } /** @@ -510,7 +525,7 @@ public function _deleteByWildcard( $wildcard, $fname = false ) else $fname = "_deleteByWildcard($wildcard)"; return $this->_protect( array( $this, '_deleteByWildcardInner' ), $fname, - $wildcard, $fname ); + $wildcard, $fname ); } /** @@ -527,12 +542,12 @@ protected function _deleteByWildcardInner( $wildcard, $fname ) $regex = '^' . pg_escape_string( $this->db, $wildcard ) . '$'; $regex = str_replace( array( '.' ), - array( '\.' ), - $regex ); + array( '\.' ), + $regex ); $regex = str_replace( array( '?', '*', '{', '}', ',' ), - array( '.', '.*', '(', ')', '|' ), - $regex ); + array( '.', '.*', '(', ')', '|' ), + $regex ); $sql = "UPDATE " . $this->dbTable( $wildcard ) . " SET mtime=-ABS(mtime), expired=1\nWHERE name REGEXP '$regex'"; if ( !$res = $this->_query( $sql, $fname ) ) @@ -562,7 +577,7 @@ public function _deleteByDirList( $dirList, $commonPath, $commonSuffix, $fname = else $fname = "_deleteByDirList(" . implode( ' ',$dirList ) . ", $commonPath, $commonSuffix)"; return $this->_protect( array( $this, '_deleteByDirListInner' ), $fname, - $dirList, $commonPath, $commonSuffix, $fname ); + $dirList, $commonPath, $commonSuffix, $fname ); } protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $fname ) @@ -580,7 +595,7 @@ protected function _deleteByDirListInner( $dirList, $commonPath, $commonSuffix, $sql = "UPDATE " . $this->dbTable( $commonPath ) . " SET mtime=-ABS(mtime), expired=1\n$where"; if ( !$stmt = $this->_query( $sql, $fname ) ) { - eZDebug::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); + self::writeError( "Failed to delete files in dir: '$commonPath/$dirItem/$commonSuffix%'", __METHOD__ ); } } return true; @@ -606,7 +621,7 @@ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, else $fname = "_exists($filePath)"; $row = $this->_selectOneRow( "SELECT name, mtime FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ), - $fname, "Failed to check file '$filePath' existence: ", true ); + $fname, "Failed to check file '$filePath' existence: ", true ); if ( $row === false ) return false; @@ -619,7 +634,7 @@ public function _exists( $filePath, $fname = false, $ignoreExpiredFiles = true, { $rc = $this->dfsbackend->existsOnDFS( $filePath ); } - + return $rc; } @@ -671,7 +686,7 @@ public function _fetch( $filePath, $uniqueName = false ) if ( !$metaData ) { // @todo Throw an exception - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); + self::writeError( "File '$filePath' does not exist while trying to fetch.", __METHOD__ ); return false; } @@ -694,7 +709,7 @@ public function _fetch( $filePath, $uniqueName = false ) // @todo Throw an exception if ( !$this->dfsbackend->copyFromDFS( $filePath, $tmpFilePath ) ) { - eZDebug::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); + self::writeError("Failed copying DFS://$filePath to FS://$tmpFilePath "); usleep( self::TIME_UNTIL_RETRY ); ++$loopCount; continue; @@ -731,7 +746,7 @@ public function _fetch( $filePath, $uniqueName = false ) while ( $dfsFileSize > $localFileSize && $loopCount < $this->maxCopyTries ); // Copy from DFS has failed :-( - eZDebug::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); + self::writeError( "Size ({$localFileSize}) of written data for file '{$filePath}' does not match expected size {$metaData['size']}", __METHOD__ ); return false; } @@ -756,14 +771,14 @@ public function _fetchContents( $filePath, $fname = false ) // @todo Throw an exception if ( !$metaData ) { - eZDebug::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); + self::writeError( "File '$filePath' does not exist while trying to fetch its contents.", __METHOD__ ); return false; } // @todo Catch an exception if ( !$contents = $this->dfsbackend->getContents( $filePath ) ) { - eZDebug::writeError("An error occurred while reading contents of DFS://$filePath", __METHOD__ ); + self::writeError("An error occurred while reading contents of DFS://$filePath", __METHOD__ ); return false; } return $contents; @@ -785,8 +800,8 @@ function _fetchMetadata( $filePath, $fname = false ) $fname = "_fetchMetadata($filePath)"; $sql = "SELECT * FROM " . $this->dbTable( $filePath ) . " WHERE name_hash=" . $this->_md5( $filePath ); return $this->_selectOneAssoc( $sql, $fname, - "Failed to retrieve file metadata: $filePath", - true ); + "Failed to retrieve file metadata: $filePath", + true ); } /** @@ -870,7 +885,7 @@ public function _rename( $srcFilePath, $dstFilePath ) if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { // @todo Throw an exception - eZDebug::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); + self::writeError( "Failed locking file '$srcFilePath'", __METHOD__ ); $this->_rollback( __METHOD__ ); return false; } @@ -886,7 +901,7 @@ public function _rename( $srcFilePath, $dstFilePath ) "WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { - eZDebug::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); + self::writeError( "Failed making new file entry '$dstFilePath'", __METHOD__ ); $this->_rollback( __METHOD__ ); // @todo Throw an exception return false; @@ -901,7 +916,7 @@ public function _rename( $srcFilePath, $dstFilePath ) $sql = "DELETE FROM " . $this->dbTable( $srcFilePath ) . " WHERE name_hash=" . $this->_md5( $srcFilePath ); if ( !$this->_query( $sql, "_rename($srcFilePath, $dstFilePath)" ) ) { - eZDebug::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); + self::writeError( "Failed removing old file '$srcFilePath'", __METHOD__ ); $this->_rollback( __METHOD__ ); // @todo Throw an exception return false; @@ -932,7 +947,7 @@ function _store( $filePath, $datatype, $scope, $fname = false ) { if ( !is_readable( $filePath ) ) { - eZDebug::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); + self::writeError( "Unable to store file '$filePath' since it is not readable.", __METHOD__ ); return false; } if ( $fname ) @@ -941,7 +956,7 @@ function _store( $filePath, $datatype, $scope, $fname = false ) $fname = "_store($filePath, $datatype, $scope)"; return $this->_protect( array( $this, '_storeInner' ), $fname, - $filePath, $datatype, $scope, $fname ); + $filePath, $datatype, $scope, $fname ); } /** @@ -963,16 +978,16 @@ function _storeInner( $filePath, $datatype, $scope, $fname ) $nameTrunk = self::nameTrunk( $filePath, $scope ); if ( $this->_insertUpdate( $this->dbTable( $filePath ), - array( 'datatype' => $datatype, - 'name' => $filePath, - 'name_trunk' => $nameTrunk, - 'name_hash' => $filePathHash, - 'scope' => $scope, - 'size' => $contentLength, - 'mtime' => $fileMTime, - 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), - array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), - $fname ) === false ) + array( 'datatype' => $datatype, + 'name' => $filePath, + 'name_trunk' => $nameTrunk, + 'name_hash' => $filePathHash, + 'scope' => $scope, + 'size' => $contentLength, + 'mtime' => $fileMTime, + 'expired' => ( $fileMTime < 0 ) ? 1 : 0 ), + array( 'datatype', 'scope', 'size', 'mtime', 'expired' ), + $fname ) === false ) { $this->_fail( "Failed to insert file metadata while storing. Possible race condition" ); } @@ -1009,7 +1024,7 @@ function _storeContents( $filePath, $contents, $scope, $datatype, $mtime = false $fname = "_storeContents($filePath, ..., $scope, $datatype)"; return $this->_protect( array( $this, '_storeContentsInner' ), $fname, - $filePath, $contents, $scope, $datatype, $mtime, $fname ); + $filePath, $contents, $scope, $datatype, $mtime, $fname ); } function _storeContentsInner( $filePath, $contents, $scope, $datatype, $mtime, $fname ) @@ -1069,11 +1084,11 @@ public function _getFileList( { $filePathList = array(); $tables = array_unique( array( $this->metaDataTable, $this->metaDataTableCache ) ); - + foreach ( $tables as $table ) { $query = 'SELECT name FROM ' . $table; - + if ( is_array( $scopes ) && count( $scopes ) > 0 ) { $query .= ' WHERE scope '; @@ -1081,7 +1096,7 @@ public function _getFileList( $query .= 'NOT '; $query .= "IN ('" . implode( "', '", $scopes ) . "')"; } - + $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); if ( !$stmt ) { @@ -1089,33 +1104,33 @@ public function _getFileList( // @todo Throw an exception return false; } - + $filePathList = array(); while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) $filePathList[] = $row[0]; - + unset( $stmt ); } return $filePathList; } /** - * Handles a DB error, displaying it as an eZDebug error - * @see eZDebug::writeError - * @param string $msg Message to display - * @param string $sql SQL query to display error for - * @return void - **/ + * Handles a DB error, displaying it as an eZDebug error + * @see self::writeError + * @param string $msg Message to display + * @param string $sql SQL query to display error for + * @return void + **/ protected function _die( $msg, $sql = null ) { if ( $this->db ) { $error = $this->db->errorInfo(); - eZDebug::writeError( $sql, "$msg: {$error[2]}" ); + self::writeError( $sql, "$msg: {$error[2]}" ); } else { - eZDebug::writeError( $sql, $msg ); + self::writeError( $sql, $msg ); } } @@ -1199,12 +1214,12 @@ protected function _insertUpdate( $table, $insert, $update, $fname, $reportError } /** - * Formats a list of entries as an SQL list which is separated by commas. - * Each entry in the list is quoted using _quote(). - * - * @param array $array - * @return array - **/ + * Formats a list of entries as an SQL list which is separated by commas. + * Each entry in the list is quoted using _quote(). + * + * @param array $array + * @return array + **/ protected function _sqlList( $array ) { $text = ""; @@ -1285,7 +1300,7 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ $numRows = $stmt->rowCount(); if ( $numRows > 1 ) { - eZDebug::writeError( 'Duplicate entries found', $fname ); + self::writeError( 'Duplicate entries found', $fname ); eZDebug::accumulatorStop( 'postgresql_cluster_query' ); // @todo throw an exception instead. Should NOT happen. } @@ -1308,9 +1323,9 @@ protected function _selectOne( $query, $fname, $error = false, $debug = false, $ } /** - * Starts a new transaction by executing a BEGIN call. - * If a transaction is already started nothing is executed. - **/ + * Starts a new transaction by executing a BEGIN call. + * If a transaction is already started nothing is executed. + **/ protected function _begin() { $this->transactionCount++; @@ -1319,9 +1334,9 @@ protected function _begin() } /** - * Stops a current transaction and commits the changes by executing a COMMIT call. - * If the current transaction is a sub-transaction nothing is executed. - **/ + * Stops a current transaction and commits the changes by executing a COMMIT call. + * If the current transaction is a sub-transaction nothing is executed. + **/ protected function _commit() { $this->transactionCount--; @@ -1330,10 +1345,10 @@ protected function _commit() } /** - * Stops a current transaction and discards all changes by executing a - * ROLLBACK call. - * If the current transaction is a sub-transaction nothing is executed. - **/ + * Stops a current transaction and discards all changes by executing a + * ROLLBACK call. + * If the current transaction is a sub-transaction nothing is executed. + **/ protected function _rollback() { $this->transactionCount--; @@ -1342,17 +1357,17 @@ protected function _rollback() } /** - * Protects a custom function with SQL queries in a database transaction. - * If the function reports an error the transaction is ROLLBACKed. - * - * The first argument to the _protect() is the callback and the second is the - * name of the function (for query reporting). The remainder of arguments are - * sent to the callback. - * - * A return value of false from the callback is considered a failure, any - * other value is returned from _protect(). For extended error handling call - * _fail() and return the value. - **/ + * Protects a custom function with SQL queries in a database transaction. + * If the function reports an error the transaction is ROLLBACKed. + * + * The first argument to the _protect() is the callback and the second is the + * name of the function (for query reporting). The remainder of arguments are + * sent to the callback. + * + * A return value of false from the callback is considered a failure, any + * other value is returned from _protect(). For extended error handling call + * _fail() and return the value. + **/ protected function _protect() { $result = false; @@ -1371,12 +1386,12 @@ protected function _protect() } catch( PDOException $e ) { - eZDebug::writeError( $e->getMessage(), __METHOD__ ); + self::writeError( $e->getMessage(), __METHOD__ ); return false; } catch( Exception $e ) { - eZDebug::writeError( $e->getMessage(), __METHOD__ ); + self::writeError( $e->getMessage(), __METHOD__ ); return false; } break; // All is good, so break out of loop @@ -1443,12 +1458,12 @@ protected function _query( $query, $fname = false, $reportError = true ) } /** - * Make sure that $value is escaped and quoted according to type and returned - * as a string. - * - * @param string $value a SQL parameter to escape - * @return string a string that can safely be used in SQL queries - **/ + * Make sure that $value is escaped and quoted according to type and returned + * as a string. + * + * @param string $value a SQL parameter to escape + * @return string a string that can safely be used in SQL queries + **/ protected function _quote( $value ) { if ( $value === null ) @@ -1464,8 +1479,8 @@ protected function _quote( $value ) } /** - * Provides the SQL calls to convert $value to MD5 - * The returned value can directly be put into SQLs. + * Provides the SQL calls to convert $value to MD5 + * The returned value can directly be put into SQLs. * @param $value * * @return string @@ -1476,16 +1491,16 @@ protected function _md5( $value ) } /** - * Prints error message $error to debug system. - * @param string $query The query that was attempted, will be printed if - * $error is \c false - * @param PDOStatement|resource $res The result resource the error occurred on - * @param string $fname The function name that started the query, should - * contain relevant arguments in the text. - * @param string $error The error message, if this is an array the first - * element is the value to dump and the second the error - * header (for eZDebug::writeNotice). If this is \c - * false a generic message is shown. + * Prints error message $error to debug system. + * @param string $query The query that was attempted, will be printed if + * $error is \c false + * @param PDOStatement|resource $res The result resource the error occurred on + * @param string $fname The function name that started the query, should + * contain relevant arguments in the text. + * @param string $error The error message, if this is an array the first + * element is the value to dump and the second the error + * header (for eZDebug::writeNotice). If this is \c + * false a generic message is shown. */ protected function _error( $query, $res, $fname, $error = "Failed to execute SQL for function:" ) { @@ -1500,7 +1515,7 @@ protected function _error( $query, $res, $fname, $error = "Failed to execute SQL } // @todo Investigate error methods - eZDebug::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ) . ' ' .$query, $fname ); + self::writeError( "$error\n" . pg_result_error_field( $res, PGSQL_DIAG_SQLSTATE ) . ': ' . pg_result_error_field( $res, PGSQL_DIAG_MESSAGE_PRIMARY ) . ' ' .$query, $fname ); } /** @@ -1564,7 +1579,7 @@ public function _startCacheGeneration( $filePath, $generatingFilePath ) //per testare scommenta la riga 1503 e sposta righe 1516-1548 fuori dal catch @todo //$query .= " WHERE NOT EXISTS ( SELECT name_hash FROM ' . $this->dbTable( $filePath ) . ' WHERE name_hash = {$nameHash} );"; - + try { $stmt = $this->_query( $query, "_startCacheGeneration( $filePath )", false ); @@ -1708,14 +1723,14 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) } /** - * Checks if generation has timed out by looking for the .generating file - * and comparing its timestamp to the one assigned when the file was created - * - * @param string $generatingFilePath - * @param int $generatingFileMtime - * - * @return bool true if the file didn't timeout, false otherwise - **/ + * Checks if generation has timed out by looking for the .generating file + * and comparing its timestamp to the one assigned when the file was created + * + * @param string $generatingFilePath + * @param int $generatingFileMtime + * + * @return bool true if the file didn't timeout, false otherwise + **/ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) { $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; @@ -1793,11 +1808,11 @@ public function _abortCacheGeneration( $generatingFilePath ) } /** - * Returns the name_trunk for a file path - * @param string $filePath - * @param string $scope - * @return string - **/ + * Returns the name_trunk for a file path + * @param string $filePath + * @param string $scope + * @return string + **/ static protected function nameTrunk( $filePath, $scope ) { switch ( $scope ) @@ -1863,14 +1878,14 @@ protected function remainingCacheGenerationTime( $row ) public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = false ) { $tables = array( $this->metaDataTable, $this->metaDataTableCache ); - + if ( count( $scopes ) == 0 || $scopes == false ) throw new ezcBaseValueException( 'scopes', $scopes, "array of scopes", "parameter" ); $scopeString = $this->_sqlList( $scopes ); - + $filePathList = array(); - + foreach ( $tables as $table) { $query = "SELECT name FROM " . $table . " WHERE expired = 1 AND scope IN( $scopeString )"; @@ -1901,7 +1916,7 @@ public function applyServerUri( $filePath ) { return $this->dfsbackend->applyServerUri( $filePath ); } - + /** * Deletes a batch of cache files from the storage table. * From 1195f0accabe74bb59900320d21735b85abe582f Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Wed, 10 Oct 2018 22:40:35 +0200 Subject: [PATCH 16/21] fix debug report --- clustering/dfs/ezpostsgresqlbackend.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 167fdee..a958ce6 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -1436,7 +1436,7 @@ protected function _fail( $message, $result = false) **/ protected function _query( $query, $fname = false, $reportError = true ) { - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); $time = microtime( true ); $stmt = $this->db->query( $query ); @@ -1736,7 +1736,7 @@ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFi $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; // reporting - eZDebug::accumulatorStart( 'postgresql_cluster_query', 'MySQL Cluster', 'DB queries' ); + eZDebug::accumulatorStart( 'postgresql_cluster_query', 'PostgreSQL Cluster', 'DB queries' ); $time = microtime( true ); $nameHash = $this->_md5( $generatingFilePath ); From 7282a55cb493767f782f90c903b920c2ea318180 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Thu, 11 Oct 2018 15:47:44 +0200 Subject: [PATCH 17/21] fix getfilelist method --- clustering/dfs/ezpostsgresqlbackend.php | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index a958ce6..b00ff71 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -1096,6 +1096,18 @@ public function _getFileList( $query .= 'NOT '; $query .= "IN ('" . implode( "', '", $scopes ) . "')"; } + if ( $path != false && $scopes == false) + { + $query .= " WHERE name LIKE '" . $path . "%'"; + } + else if ( $path != false) + { + $query .= " AND name LIKE '" . $path . "%'"; + } + if ( $limit && array_sum($limit) ) + { + $query .= " LIMIT {$limit[0]}, {$limit[1]}"; + } $stmt = $this->_query( $query, "_getFileList( array( " . implode( ', ', $scopes ) . " ), $excludeScopes )" ); if ( !$stmt ) @@ -1106,8 +1118,8 @@ public function _getFileList( } $filePathList = array(); - while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) - $filePathList[] = $row[0]; + foreach ($stmt->fetch( PDO::FETCH_NUM ) as $row) + $filePathList[] = $row; unset( $stmt ); } From 67f38ba8bd3f0566f6c67737eff0942ae47c6793 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Mon, 15 Oct 2018 11:38:18 +0200 Subject: [PATCH 18/21] fix corner case in select dbTable when the cache filepath contains string 'storage' --- clustering/dfs/ezpostsgresqlbackend.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index b00ff71..c31b628 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -75,7 +75,16 @@ protected function dbTable( $filePath ) if ( $this->metaDataTableCache == $this->metaDataTable ) return $this->metaDataTable; - if ( strpos( $filePath, $this->cacheDir ) !== false && strpos( $filePath, $this->storageDir ) === false ) + $isInCacheDir = strpos( $filePath, $this->cacheDir ); + $isInStorageDir = strpos( $filePath, $this->storageDir ); + + if ( $isInCacheDir !== false && $isInStorageDir === false ) + { + return $this->metaDataTableCache; + } + + // example var/site/cache/my_custom_cache/storage.cache + if ( $isInCacheDir !== false && $isInStorageDir !== false && ( $isInCacheDir < $isInStorageDir ) ) { return $this->metaDataTableCache; } From 8bba92ca3c89b1cf5cc28e20ff4ebc3e0364321a Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Tue, 16 Oct 2018 23:02:33 +0200 Subject: [PATCH 19/21] avoid sql error in _checkCacheGenerationTimeout --- clustering/dfs/ezpostsgresqlbackend.php | 1 + 1 file changed, 1 insertion(+) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index c31b628..02ade96 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -1754,6 +1754,7 @@ public function _endCacheGeneration( $filePath, $generatingFilePath, $rename ) **/ public function _checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime ) { + $generatingFileMtime = intval($generatingFileMtime); $fname = "_checkCacheGenerationTimeout( $generatingFilePath, $generatingFileMtime )"; // reporting From 0eccb8df7b8840e2bc22168047a0a58ecbb67dc9 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Wed, 27 Mar 2019 22:12:34 +0100 Subject: [PATCH 20/21] change log name logic --- clustering/dfs/ezpostsgresqlbackend.php | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 02ade96..0026d73 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -30,14 +30,17 @@ class eZDFSFileHandlerPostgresqlBackend protected static function writeError($string, $label = "", $backgroundClass = "") { $logName = 'cluster_error.log'; - if ( isset( $GLOBALS['eZCurrentAccess']['name'] ) ){ - $logName = $GLOBALS['eZCurrentAccess']['name'] . '_cluster_error.log'; - } + //if ( isset( $GLOBALS['eZCurrentAccess']['name'] ) ){ + // $logName = $GLOBALS['eZCurrentAccess']['name'] . '_cluster_error.log'; + //} + + $instanceName = OpenPABase::getCurrentSiteaccessIdentifier(); + $message = "[$instanceName] "; if ($label){ - $message = "[$label] $string"; + $message .= "[$label] $string"; }else{ - $message = $string; + $message .= $string; } eZLog::write($message, $logName); } @@ -162,7 +165,9 @@ public function _connect() // DFS setup if ( $this->dfsbackend === null ) - $this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + $this->dfsbackend = eZDFSFileHandlerBackendFactory::build(); + //$this->dfsbackend = new eZDFSFileHandlerDFSBackend(); + } /** @@ -700,6 +705,12 @@ public function _fetch( $filePath, $uniqueName = false ) } $dfsFileSize = $this->dfsbackend->getDfsFileSize( $filePath ); + if ( !$dfsFileSize ) + { + // @todo Throw an exception + self::writeError( "Error getting filesize of file '$filePath'.", __METHOD__ ); + return false; + } $loopCount = 0; $localFileSize = 0; From db456e4e60217f56ac994b99e84b813929f51a02 Mon Sep 17 00:00:00 2001 From: Luca Realdi Date: Mon, 1 Jul 2019 14:52:55 +0200 Subject: [PATCH 21/21] fix expiredFilesList var reset --- clustering/dfs/ezpostsgresqlbackend.php | 1 - 1 file changed, 1 deletion(-) diff --git a/clustering/dfs/ezpostsgresqlbackend.php b/clustering/dfs/ezpostsgresqlbackend.php index 0026d73..1d83449 100644 --- a/clustering/dfs/ezpostsgresqlbackend.php +++ b/clustering/dfs/ezpostsgresqlbackend.php @@ -1927,7 +1927,6 @@ public function expiredFilesList( $scopes, $limit = array( 0, 100 ), $expiry = f $query .= " LIMIT {$limit[1]} OFFSET {$limit[0]}"; } $stmt = $this->_query( $query, __METHOD__ ); - $filePathList = array(); while ( $row = $stmt->fetch( PDO::FETCH_NUM ) ) $filePathList[] = $row[0]; unset( $stmt );