From 429c915ca91fda6e2e865b30246f2b73e83433c1 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Sat, 26 Sep 2020 15:12:16 +0200 Subject: [PATCH 1/5] BUGFIX: Correctly initialize a Request --- .../ProxyAwareFileSystemSymlinkTarget.php | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php index 713e792..e1140a4 100644 --- a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -48,11 +48,12 @@ class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget */ protected $resourceManager; - public function initializeObject() { - // intialize uribuilder with request + public function initializeObject() + { + // initialize uriBuilder with request $requestHandler = $this->bootstrap->getActiveRequestHandler(); if ($requestHandler instanceof HttpRequestHandlerInterface) { - $request = new ActionRequest($requestHandler->getHttpRequest()); + $request = ActionRequest::fromHttpRequest($requestHandler->getComponentContext()->getHttpRequest()); $this->uriBuilder->setRequest($request); } parent::initializeObject(); From 53f3d275444d7e94356b9361b80e30731a6f4ae8 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Sat, 26 Sep 2020 15:14:32 +0200 Subject: [PATCH 2/5] FEATURE: Add interfaces for ProxyAwareTargets and ProxyAwareStorages --- .../ProxyAwareFileSystemSymlinkTarget.php | 18 ++++++++++------ .../ProxyAwareStorageInterface.php | 11 ++++++++++ .../ProxyAwareTargetInterface.php | 9 ++++++++ .../ProxyAwareWritableFileSystemStorage.php | 21 ++++++++++--------- 4 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 Classes/ResourceManagement/ProxyAwareStorageInterface.php create mode 100644 Classes/ResourceManagement/ProxyAwareTargetInterface.php diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php index e1140a4..2188f5b 100644 --- a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -3,6 +3,8 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\Core\Bootstrap; +use Neos\Flow\Http\Exception as HttpException; +use Neos\Flow\Mvc\Routing\Exception\MissingActionNameException; use Neos\Flow\ResourceManagement\CollectionInterface; use Neos\Flow\ResourceManagement\PersistentResource; use Neos\Flow\ResourceManagement\ResourceManager; @@ -16,7 +18,7 @@ use Sitegeist\MagicWand\ResourceManagement\ProxyAwareWritableFileSystemStorage; use Neos\Flow\ResourceManagement\Storage\StorageObject; -class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget +class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget implements ProxyAwareTargetInterface { /** * @Flow\Inject @@ -69,15 +71,17 @@ public function initializeObject() public function publishCollection(CollectionInterface $collection, callable $callback = null) { if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { - return parent::publishCollection($collection, $callback); + parent::publishCollection($collection, $callback); + return; } /** * @var ProxyAwareWritableFileSystemStorage $storage */ $storage = $collection->getStorage(); - if (!$storage instanceof ProxyAwareWritableFileSystemStorage) { - return parent::publishCollection($collection, $callback); + if (!$storage instanceof ProxyAwareStorageInterface) { + parent::publishCollection($collection, $callback); + return; } foreach ($collection->getObjects($callback) as $object) { @@ -97,6 +101,8 @@ public function publishCollection(CollectionInterface $collection, callable $cal * @param PersistentResource $resource * @return string * @throws Exception + * @throws HttpException + * @throws MissingActionNameException */ public function getPublicPersistentResourceUri(PersistentResource $resource) { @@ -107,7 +113,7 @@ public function getPublicPersistentResourceUri(PersistentResource $resource) $collection = $this->resourceManager->getCollection($resource->getCollectionName()); $storage = $collection->getStorage(); - if (!$storage instanceof ProxyAwareWritableFileSystemStorage) { + if (!$storage instanceof ProxyAwareStorageInterface) { return parent::getPublicPersistentResourceUri($resource); } @@ -115,7 +121,7 @@ public function getPublicPersistentResourceUri(PersistentResource $resource) return parent::getPublicPersistentResourceUri($resource); } - // build uri to resoucre controller that will fetch and publish + // build uri to resource controller that will fetch and publish // the resource asynchronously return $this->uriBuilder->uriFor( 'index', diff --git a/Classes/ResourceManagement/ProxyAwareStorageInterface.php b/Classes/ResourceManagement/ProxyAwareStorageInterface.php new file mode 100644 index 0000000..8810b44 --- /dev/null +++ b/Classes/ResourceManagement/ProxyAwareStorageInterface.php @@ -0,0 +1,11 @@ +getStoragePathAndFilenameByHash($resource->getSha1()); + public function resourceIsPresentInStorage(ResourceMetaDataInterface $resource): bool + { + $path = $this->getStoragePathAndFilenameByHash($resource->getSha1()); return file_exists($path); } @@ -57,7 +58,7 @@ public function getStreamByResource(PersistentResource $resource) $curlEngine = new CurlEngine(); $curlOptions = $resourceProxyConfiguration['curlOptions'] ?? []; - foreach($curlOptions as $key => $value) { + foreach ($curlOptions as $key => $value) { $curlEngine->setOption(constant($key), $value); } @@ -67,16 +68,16 @@ public function getStreamByResource(PersistentResource $resource) $subdivideHashPathSegment = $resourceProxyConfiguration['subdivideHashPathSegment'] ?? false; if ($subdivideHashPathSegment) { $sha1Hash = $resource->getSha1(); - $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . rawurlencode($resource->getFilename()); + $uri = $resourceProxyConfiguration['baseUri'] . '/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . rawurlencode($resource->getFilename()); } else { - $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $resource->getSha1() . '/' . rawurlencode($resource->getFilename()); + $uri = $resourceProxyConfiguration['baseUri'] . '/_Resources/Persistent/' . $resource->getSha1() . '/' . rawurlencode($resource->getFilename()); } $response = $browser->request($uri); - if ($response->getStatusCode() == 200 ) { + if ($response->getStatusCode() == 200) { $stream = $response->getBody()->detach(); - $targetPathAndFilename = $this->getStoragePathAndFilenameByHash($resource->getSha1()); + $targetPathAndFilename = $this->getStoragePathAndFilenameByHash($resource->getSha1()); if (!file_exists(dirname($targetPathAndFilename))) { Files::createDirectoryRecursively(dirname($targetPathAndFilename)); } From 44d008ed683881978626315c22487344adac22b2 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Sat, 26 Sep 2020 15:27:01 +0200 Subject: [PATCH 3/5] FEATURE: Add ProxyAwareFileSystemTarget --- .../ProxyAwareFileSystemSymlinkTarget.php | 107 +----------------- .../ProxyAwareFileSystemTarget.php | 40 +++++++ .../ProxyAwareTargetTrait.php | 99 ++++++++++++++++ 3 files changed, 145 insertions(+), 101 deletions(-) create mode 100644 Classes/ResourceManagement/ProxyAwareFileSystemTarget.php create mode 100644 Classes/ResourceManagement/ProxyAwareTargetTrait.php diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php index 2188f5b..7a73b28 100644 --- a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -3,131 +3,36 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\Core\Bootstrap; -use Neos\Flow\Http\Exception as HttpException; -use Neos\Flow\Mvc\Routing\Exception\MissingActionNameException; -use Neos\Flow\ResourceManagement\CollectionInterface; -use Neos\Flow\ResourceManagement\PersistentResource; +use Neos\Flow\Mvc\Routing\UriBuilder; use Neos\Flow\ResourceManagement\ResourceManager; -use Neos\Flow\ResourceManagement\ResourceRepository; -use Neos\Flow\ResourceManagement\Target\Exception; use Neos\Flow\ResourceManagement\Target\FileSystemSymlinkTarget; -use Neos\Flow\Mvc\Routing\UriBuilder; -use Neos\Flow\Http\HttpRequestHandlerInterface; -use Neos\Flow\Mvc\ActionRequest; use Sitegeist\MagicWand\Domain\Service\ConfigurationService; -use Sitegeist\MagicWand\ResourceManagement\ProxyAwareWritableFileSystemStorage; -use Neos\Flow\ResourceManagement\Storage\StorageObject; class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget implements ProxyAwareTargetInterface { + use ProxyAwareTargetTrait; + /** * @Flow\Inject * @var Bootstrap */ protected $bootstrap; - /** - * @var ConfigurationService - * @Flow\Inject - */ - protected $configurationService; - /** * @var UriBuilder * @Flow\Inject */ protected $uriBuilder; - /** - * @var ResourceRepository - * @Flow\Inject - */ - protected $resourceRepository; - /** * @var ResourceManager * @Flow\Inject */ protected $resourceManager; - public function initializeObject() - { - // initialize uriBuilder with request - $requestHandler = $this->bootstrap->getActiveRequestHandler(); - if ($requestHandler instanceof HttpRequestHandlerInterface) { - $request = ActionRequest::fromHttpRequest($requestHandler->getComponentContext()->getHttpRequest()); - $this->uriBuilder->setRequest($request); - } - parent::initializeObject(); - } - /** - * Publishes the whole collection to this target - * - * @param CollectionInterface $collection The collection to publish - * @param callable $callback Function called after each resource publishing - * @return void - */ - public function publishCollection(CollectionInterface $collection, callable $callback = null) - { - if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { - parent::publishCollection($collection, $callback); - return; - } - - /** - * @var ProxyAwareWritableFileSystemStorage $storage - */ - $storage = $collection->getStorage(); - if (!$storage instanceof ProxyAwareStorageInterface) { - parent::publishCollection($collection, $callback); - return; - } - - foreach ($collection->getObjects($callback) as $object) { - /** @var StorageObject $object */ - if ($storage->resourceIsPresentInStorage($object) === false) { - // this storage ignores resources that are not yet in the filesystem as they - // are optimistically created during read operations - continue; - } - $sourceStream = $object->getStream(); - $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($object)); - fclose($sourceStream); - } - } - - /** - * @param PersistentResource $resource - * @return string - * @throws Exception - * @throws HttpException - * @throws MissingActionNameException + * @var ConfigurationService + * @Flow\Inject */ - public function getPublicPersistentResourceUri(PersistentResource $resource) - { - if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { - return parent::getPublicPersistentResourceUri($resource); - } - - $collection = $this->resourceManager->getCollection($resource->getCollectionName()); - $storage = $collection->getStorage(); - - if (!$storage instanceof ProxyAwareStorageInterface) { - return parent::getPublicPersistentResourceUri($resource); - } - - if ($storage->resourceIsPresentInStorage($resource)) { - return parent::getPublicPersistentResourceUri($resource); - } - - // build uri to resource controller that will fetch and publish - // the resource asynchronously - return $this->uriBuilder->uriFor( - 'index', - ['resourceIdentifier' => $resource], - 'Resource', - 'Sitegeist.MagicWand' - ); - } + protected $configurationService; } diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemTarget.php new file mode 100644 index 0000000..d6d2746 --- /dev/null +++ b/Classes/ResourceManagement/ProxyAwareFileSystemTarget.php @@ -0,0 +1,40 @@ +bootstrap->getActiveRequestHandler(); + if ($requestHandler instanceof HttpRequestHandlerInterface) { + $request = ActionRequest::fromHttpRequest($requestHandler->getComponentContext()->getHttpRequest()); + $this->uriBuilder->setRequest($request); + } + parent::initializeObject(); + } + + /** + * @param CollectionInterface $collection The collection to publish + * @param callable $callback Function called after each resource publishing + * @return void + */ + public function publishCollection(CollectionInterface $collection, callable $callback = null) + { + if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { + parent::publishCollection($collection, $callback); + return; + } + + /** + * @var ProxyAwareWritableFileSystemStorage $storage + */ + $storage = $collection->getStorage(); + if (!$storage instanceof ProxyAwareStorageInterface) { + parent::publishCollection($collection, $callback); + return; + } + + foreach ($collection->getObjects($callback) as $object) { + /** @var StorageObject $object */ + if ($storage->resourceIsPresentInStorage($object) === false) { + // this storage ignores resources that are not yet in the filesystem as they + // are optimistically created during read operations + continue; + } + $sourceStream = $object->getStream(); + $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($object)); + fclose($sourceStream); + } + } + + /** + * @param PersistentResource $resource + * @return string + * @throws Exception + * @throws HttpException + * @throws MissingActionNameException + */ + public function getPublicPersistentResourceUri(PersistentResource $resource) + { + if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { + return parent::getPublicPersistentResourceUri($resource); + } + + $collection = $this->resourceManager->getCollection($resource->getCollectionName()); + $storage = $collection->getStorage(); + + if (!$storage instanceof ProxyAwareStorageInterface) { + return parent::getPublicPersistentResourceUri($resource); + } + + if ($storage->resourceIsPresentInStorage($resource)) { + return parent::getPublicPersistentResourceUri($resource); + } + + // build uri to resource controller that will fetch and publish + // the resource asynchronously + return $this->uriBuilder->uriFor( + 'index', + ['resourceIdentifier' => $resource], + 'Resource', + 'Sitegeist.MagicWand' + ); + } +} From 83d62393b7e9dce1efc1269f5a83b339a9e8fa0c Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Mon, 12 Oct 2020 12:33:14 +0200 Subject: [PATCH 4/5] TASK: Raise minimal Flow to 6.0 Needs a Flow version > 6.0 due to new ActionRequest creation. --- composer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/composer.json b/composer.json index b643b0d..6ae87e9 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ } ], "require": { - "neos/flow": "~4.0 || ~5.0 || ~6.0 || dev-master" + "neos/flow": "~6.0 || dev-master" }, "autoload": { "psr-4": { From 999244beaaa81de22768ab83fae08964a8fb4a96 Mon Sep 17 00:00:00 2001 From: Daniel Lienert Date: Wed, 21 Oct 2020 20:05:57 +0200 Subject: [PATCH 5/5] TASK: Add some documentation about resource proxies --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 4485ff7..c84c58a 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ section of your composer.json**. * Wilhelm Behncke - behncke@sitegeist.de * Martin Ficzel - ficzel@sitegeist.de +* ... and others *The development and the public-releases of this package is generously sponsored by our employer https://www.sitegeist.de.* @@ -108,6 +109,15 @@ the commands.** ``` **Note:** Use this command on a regular basis, because your stash tends to grow **very** large. +## Resource proxies + +While cloning the database to your local dev system is manageable even for larger projects, downloading all the assets is often not an option. + +For this case the package offers the concept of resource proxies. Once activated, only the resources that are actually used are downloaded just at the moment they are rendered. +This is done by custom implementations of `WritableFileSystemStorage` and `ProxyAwareFileSystemSymlinkTarget` and works out of the box if you use this storage and target in you local development environment. +If you use other local storages, for example a local S3 storage, you can easily build your own proxy aware versions implementing the interfaces `ProxyAwareStorageInterface` and `ProxyAwareTargetInterface`of this package. + + ## Installation Sitegeist.Magicwand is available via packagist. Just add `"sitegeist/magicwand" : "~1.0"` to the require-dev section of the composer.json or run `composer require --dev sitegeist/magicwand`. We use semantic-versioning so every breaking change will increase the major-version number.