From 9e5a90e2264e34cb75783a8a36e5a0079dddb724 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 16 Jan 2019 13:58:15 +0100 Subject: [PATCH 01/16] TASK: Introduce proxy resource storage and target Internally the default storage and target of neos is used. --- .../Storage/ProxyStorage.php | 98 +++++++++++++++++++ .../ResourceManagement/Target/ProxyTarget.php | 93 ++++++++++++++++++ Configuration/Settings.yaml | 26 +++++ 3 files changed, 217 insertions(+) create mode 100644 Classes/ResourceManagement/Storage/ProxyStorage.php create mode 100644 Classes/ResourceManagement/Target/ProxyTarget.php diff --git a/Classes/ResourceManagement/Storage/ProxyStorage.php b/Classes/ResourceManagement/Storage/ProxyStorage.php new file mode 100644 index 0000000..a71be1f --- /dev/null +++ b/Classes/ResourceManagement/Storage/ProxyStorage.php @@ -0,0 +1,98 @@ +name = $name; + if (!isset($options['localStorage'])) { + throw new Exception(sprintf('localStorage-option is required in storage %s', $name), 1547635803); + } + $this->localStorageName = $options['localStorage']; + } + + public function initializeObject() { + $localStorageDefinition = Arrays::getValueByPath($this->settings, 'storages.' . $this->localStorageName ); + + if (!isset($localStorageDefinition['storage'])) { + throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has no valid "storage" option. Please check the configuration syntax and make sure to specify a valid storage class name.', $this->localStorageName), 1361467211); + } + if (!class_exists($localStorageDefinition['storage'])) { + throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has not defined a valid "storage" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $this->localStorageName, $localStorageDefinition['storage']), 1361467212); + } + $localStorageOptions = (isset($localStorageDefinition['storageOptions']) ? $localStorageDefinition['storageOptions'] : []); + $this->localStorage = new $localStorageDefinition['storage']($this->localStorageName, $localStorageOptions); + } + + /** + * @return string + */ + public function getName() + { + return $this->name; + } + + public function getStreamByResource(PersistentResource $resource) + { + return $this->localStorage->getStreamByResourcePath($resource); + } + + public function getStreamByResourcePath($relativePath) + { + return $this->localStorage->getStreamByResourcePath($relativePath); + } + + public function getObjects() + { + return $this->localStorage->getObjects(); + } + + public function getObjectsByCollection(CollectionInterface $collection) + { + return $this->localStorage->getObjectsByCollection($collection); + } + + public function importResource($source, $collectionName) + { + return $this->localStorage->importResource($source, $collectionName); + } + + public function importResourceFromContent($content, $collectionName) + { + return $this->localStorage->importResourceFromContent($content, $collectionName); + } + + public function deleteResource(PersistentResource $resource) + { + return $this->localStorage->deleteResource($resource); + } +} diff --git a/Classes/ResourceManagement/Target/ProxyTarget.php b/Classes/ResourceManagement/Target/ProxyTarget.php new file mode 100644 index 0000000..1adb122 --- /dev/null +++ b/Classes/ResourceManagement/Target/ProxyTarget.php @@ -0,0 +1,93 @@ +name = $name; + + if (!isset($options['localTarget'])) { + throw new Exception(sprintf('localTarget-option is required in target %s', $name), 1547635863); + } + + $this->localTargetName = $options['localTarget']; + } + + public function initializeObject() { + $localTargetDefinition = Arrays::getValueByPath($this->settings, 'targets.' . $this->localTargetName ); + if (!isset($localTargetDefinition['target'])) { + throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has no valid "target" option. Please check the configuration syntax and make sure to specify a valid target class name.', $targetName), 1361467838); + } + if (!class_exists($localTargetDefinition['target'])) { + throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has not defined a valid "target" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $targetName, $targetDefinition['target']), 1361467839); + } + $localTargetOptions = (isset($localTargetDefinition['targetOptions']) ? $localTargetDefinition['targetOptions'] : []); + + $this->localTarget = new $localTargetDefinition['target']($this->localTargetName, $localTargetOptions); + } + + public function getName() + { + return $this->name; + } + + public function publishCollection(CollectionInterface $collection) + { + return $this->localTarget->publishCollection($collection); + } + + public function publishResource(PersistentResource $resource, CollectionInterface $collection) + { + return $this->localTarget->publishResource($resource, $collection); + } + + public function unpublishResource(PersistentResource $resource) + { + return $this->localTarget->unpublishResource($resource); + } + + public function getPublicStaticResourceUri($relativePathAndFilename) + { + return $this->localTarget->getPublicStaticResourceUri($relativePathAndFilename); + } + + public function getPublicPersistentResourceUri(PersistentResource $resource) + { + return $this->localTarget->getPublicPersistentResourceUri($resource); + } + +} diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 643d138..568f490 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -25,3 +25,29 @@ Sitegeist: # # commands to execute after cloning # postClone: # - './flow help' + +Neos: + Flow: + resource: + + storages: + magicWandProxyStorage: + storage: 'Sitegeist\MagicWand\ResourceManagement\Storage\ProxyStorage' + storageOptions: + localStorage: defaultPersistentResourcesStorage + + targets: + magicWandProxyTarget: + target: 'Sitegeist\MagicWand\ResourceManagement\Target\ProxyTarget' + targetOptions: + localTarget: localWebDirectoryPersistentResourcesTarget + + # + # alter the settings for the persistent collection + # to use the proxy storage and target instead + # + collections: + persistent: + storage: 'magicWandProxyStorage' + target: 'magicWandProxyTarget' + From c1e9c65ea85013681c88a5f5c61fbdcfa64f4838 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 16 Jan 2019 15:09:00 +0100 Subject: [PATCH 02/16] TASK: Extract intialization of targets into seperate method --- .../Storage/ProxyStorage.php | 34 +++++++++++----- .../ResourceManagement/Target/ProxyTarget.php | 40 +++++++++++-------- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/Classes/ResourceManagement/Storage/ProxyStorage.php b/Classes/ResourceManagement/Storage/ProxyStorage.php index a71be1f..7a9f9b9 100644 --- a/Classes/ResourceManagement/Storage/ProxyStorage.php +++ b/Classes/ResourceManagement/Storage/ProxyStorage.php @@ -2,6 +2,7 @@ namespace Sitegeist\MagicWand\ResourceManagement\Storage; use Neos\Flow\Annotations as Flow; +use Neos\Flow\ResourceManagement\Storage\StorageInterface; use Neos\Utility\Arrays; use Neos\Flow\ResourceManagement\CollectionInterface; use Neos\Flow\ResourceManagement\PersistentResource; @@ -40,17 +41,9 @@ public function __construct($name, $options = []) $this->localStorageName = $options['localStorage']; } - public function initializeObject() { - $localStorageDefinition = Arrays::getValueByPath($this->settings, 'storages.' . $this->localStorageName ); - - if (!isset($localStorageDefinition['storage'])) { - throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has no valid "storage" option. Please check the configuration syntax and make sure to specify a valid storage class name.', $this->localStorageName), 1361467211); - } - if (!class_exists($localStorageDefinition['storage'])) { - throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has not defined a valid "storage" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $this->localStorageName, $localStorageDefinition['storage']), 1361467212); - } - $localStorageOptions = (isset($localStorageDefinition['storageOptions']) ? $localStorageDefinition['storageOptions'] : []); - $this->localStorage = new $localStorageDefinition['storage']($this->localStorageName, $localStorageOptions); + public function initializeObject() + { + $this->localStorage = $this->initializeStorage($this->localStorageName); } /** @@ -95,4 +88,23 @@ public function deleteResource(PersistentResource $resource) { return $this->localStorage->deleteResource($resource); } + + /** + * @param string $name + * @return StorageInterface + * @see \Neos\Flow\ResourceManagement\ResourceManager::initializeStorages + */ + protected function initializeStorage($name) + { + $storageDefinition = Arrays::getValueByPath($this->settings, 'storages.' . $name); + + if (!isset($storageDefinition['storage'])) { + throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has no valid "storage" option. Please check the configuration syntax and make sure to specify a valid storage class name.', $name), 1361467211); + } + if (!class_exists($storageDefinition['storage'])) { + throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has not defined a valid "storage" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $name, $storageDefinition['storage']), 1361467212); + } + $options = (isset($storageDefinition['storageOptions']) ? $storageDefinition['storageOptions'] : []); + return new $storageDefinition['storage']($this->localStorageName, $options); + } } diff --git a/Classes/ResourceManagement/Target/ProxyTarget.php b/Classes/ResourceManagement/Target/ProxyTarget.php index 1adb122..3d04f96 100644 --- a/Classes/ResourceManagement/Target/ProxyTarget.php +++ b/Classes/ResourceManagement/Target/ProxyTarget.php @@ -22,14 +22,14 @@ class ProxyTarget implements TargetInterface protected $name; /** - * @var string + * @var TargetInterface */ - protected $localTargetName; + protected $localTarget; /** - * @var TargetInterface + * @var string */ - protected $localTarget; + protected $localTargetName; /** * ProxyTarget constructor. @@ -39,25 +39,14 @@ class ProxyTarget implements TargetInterface public function __construct($name, $options) { $this->name = $name; - if (!isset($options['localTarget'])) { throw new Exception(sprintf('localTarget-option is required in target %s', $name), 1547635863); } - $this->localTargetName = $options['localTarget']; } public function initializeObject() { - $localTargetDefinition = Arrays::getValueByPath($this->settings, 'targets.' . $this->localTargetName ); - if (!isset($localTargetDefinition['target'])) { - throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has no valid "target" option. Please check the configuration syntax and make sure to specify a valid target class name.', $targetName), 1361467838); - } - if (!class_exists($localTargetDefinition['target'])) { - throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has not defined a valid "target" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $targetName, $targetDefinition['target']), 1361467839); - } - $localTargetOptions = (isset($localTargetDefinition['targetOptions']) ? $localTargetDefinition['targetOptions'] : []); - - $this->localTarget = new $localTargetDefinition['target']($this->localTargetName, $localTargetOptions); + $this->localTarget = $this->initalizeTarget($this->localTargetName); } public function getName() @@ -90,4 +79,23 @@ public function getPublicPersistentResourceUri(PersistentResource $resource) return $this->localTarget->getPublicPersistentResourceUri($resource); } + /** + * @param string $name + * @return TargetInterface + * @throws Exception + * @see \Neos\Flow\ResourceManagement\ResourceManager::initializeTargets + */ + protected function initalizeTarget($name) + { + $targetDefinition = Arrays::getValueByPath($this->settings, 'targets.' . $name); + if (!isset($targetDefinition['target'])) { + throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has no valid "target" option. Please check the configuration syntax and make sure to specify a valid target class name.', $name), 1361467838); + } + if (!class_exists($targetDefinition['target'])) { + throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has not defined a valid "target" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $name, $targetDefinition['target']), 1361467839); + } + $options = (isset($targetDefinition['targetOptions']) ? $targetDefinition['targetOptions'] : []); + return new $targetDefinition['target']($this->localTargetName, $options); + } + } From bb2dad1a2bf7b221246b75c3bbac30151ebf2110 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Wed, 16 Jan 2019 20:11:52 +0100 Subject: [PATCH 03/16] TASK: Implement proxy resources --- Classes/Controller/ResourceController.php | 36 ++++++++ .../ResourceNotFoundException.php | 15 ++++ .../Storage/ProxyStorage.php | 85 ++++++++++++++++++- .../ResourceManagement/Target/ProxyTarget.php | 57 ++++++++++++- Configuration/Policy.yaml | 11 +++ Configuration/Routes.yaml | 11 +++ Configuration/Settings.yaml | 5 ++ 7 files changed, 217 insertions(+), 3 deletions(-) create mode 100644 Classes/Controller/ResourceController.php create mode 100644 Classes/ResourceManagement/ResourceNotFoundException.php create mode 100644 Configuration/Policy.yaml create mode 100644 Configuration/Routes.yaml diff --git a/Classes/Controller/ResourceController.php b/Classes/Controller/ResourceController.php new file mode 100644 index 0000000..074b8e3 --- /dev/null +++ b/Classes/Controller/ResourceController.php @@ -0,0 +1,36 @@ +resourceRepository->findByIdentifier($resourceIdentifier); + if ($resource) { + $headers = $this->response->getHeaders(); + $headers->set('Content-Type', $resource->getMediaType(), true); + $this->response->setHeaders($headers); + return stream_get_contents($resource->getStream()); + } else { + throw new ResourceNotFoundException("Unkonwn Resource"); + } + } +} diff --git a/Classes/ResourceManagement/ResourceNotFoundException.php b/Classes/ResourceManagement/ResourceNotFoundException.php new file mode 100644 index 0000000..2828942 --- /dev/null +++ b/Classes/ResourceManagement/ResourceNotFoundException.php @@ -0,0 +1,15 @@ +name = $name; @@ -54,39 +62,112 @@ public function getName() return $this->name; } + /** + * @param PersistentResource $resource + * @return bool|resource + */ public function getStreamByResource(PersistentResource $resource) { - return $this->localStorage->getStreamByResourcePath($resource); + $localResourceStream = $this->localStorage->getStreamByResource($resource); + if ($localResourceStream) { + return $localResourceStream; + } else { + + $curlEngine = new CurlEngine(); + + $browser = new Browser(); + $browser->setRequestEngine($curlEngine); + + $uri = $baseUri . '/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + + $response = $browser->request($uri); + if ($response->getStatusCode() == 200 ) { + $response->getContent(); + + $stream = fopen('php://memory', 'r+'); + fwrite($stream, $response->getContent()); + rewind($stream); + + $this->localStorage->importResource($stream, $resource->getCollectionName()); + + return $stream; + } else { + + \Neos\Flow\var_dump($uri); + throw new ResourceNotFoundException('Kennichnich'); + } + } } + /** + * @param string $relativePath + * @return bool|resource + */ public function getStreamByResourcePath($relativePath) { return $this->localStorage->getStreamByResourcePath($relativePath); } + /** + * @return \Generator + */ public function getObjects() { return $this->localStorage->getObjects(); } + /** + * @param CollectionInterface $collection + * @return \Generator + */ public function getObjectsByCollection(CollectionInterface $collection) { return $this->localStorage->getObjectsByCollection($collection); } + /** + * @param resource|string $source + * @param string $collectionName + * @return PersistentResource + * @throws StorageException + */ public function importResource($source, $collectionName) { return $this->localStorage->importResource($source, $collectionName); } + /** + * @param string $content + * @param string $collectionName + * @return PersistentResource + * @throws StorageException + */ public function importResourceFromContent($content, $collectionName) { return $this->localStorage->importResourceFromContent($content, $collectionName); } + /** + * @param PersistentResource $resource + * @return bool + * @throws \Exception + */ public function deleteResource(PersistentResource $resource) { - return $this->localStorage->deleteResource($resource); + if ($this->isLocal($resource)) { + return $this->localStorage->deleteResource($resource); + } else { + throw new \Exception('proxy-resources cannot be deleted'); + } + } + + /** + * @param PersistentResource $resource + * @return bool + */ + protected function isLocal(PersistentResource $resource) + { + return true; } /** diff --git a/Classes/ResourceManagement/Target/ProxyTarget.php b/Classes/ResourceManagement/Target/ProxyTarget.php index 3d04f96..07380f1 100644 --- a/Classes/ResourceManagement/Target/ProxyTarget.php +++ b/Classes/ResourceManagement/Target/ProxyTarget.php @@ -2,20 +2,38 @@ namespace Sitegeist\MagicWand\ResourceManagement\Target; use Neos\Flow\Annotations as Flow; +use Neos\Flow\Http\Request; +use Neos\Flow\Mvc\ActionRequest; +use Neos\Flow\Mvc\Routing\UriBuilder; use Neos\Utility\Arrays; use Neos\Flow\ResourceManagement\CollectionInterface; use Neos\Flow\ResourceManagement\PersistentResource; use Neos\Flow\ResourceManagement\Target\Exception; use Neos\Flow\ResourceManagement\Target\TargetInterface; +use Neos\Flow\Core\Bootstrap; +use Neos\Flow\Http\HttpRequestHandlerInterface; + class ProxyTarget implements TargetInterface { + /** + * @var Bootstrap + * @Flow\Inject + */ + protected $bootstrap; + /** * @var * @Flow\InjectConfiguration(package="Neos.Flow", path="resource") */ protected $settings; + /** + * @var UriBuilder + * @Flow\Inject + */ + protected $uriBuilder; + /** * @var string */ @@ -46,37 +64,74 @@ public function __construct($name, $options) } public function initializeObject() { + // intialize uribuilder with request + $requestHandler = $this->bootstrap->getActiveRequestHandler(); + if ($requestHandler instanceof HttpRequestHandlerInterface) { + $request = new ActionRequest($requestHandler->getHttpRequest()); + $this->uriBuilder->setRequest($request); + } + + // initialize targets $this->localTarget = $this->initalizeTarget($this->localTargetName); } + /** + * @return string + */ public function getName() { return $this->name; } + /** + * @param CollectionInterface $collection + */ public function publishCollection(CollectionInterface $collection) { return $this->localTarget->publishCollection($collection); } + /** + * @param PersistentResource $resource + * @param CollectionInterface $collection + * @throws Exception + */ public function publishResource(PersistentResource $resource, CollectionInterface $collection) { return $this->localTarget->publishResource($resource, $collection); } + /** + * @param PersistentResource $resource + */ public function unpublishResource(PersistentResource $resource) { return $this->localTarget->unpublishResource($resource); } + /** + * @param string $relativePathAndFilename + * @return string + */ public function getPublicStaticResourceUri($relativePathAndFilename) { return $this->localTarget->getPublicStaticResourceUri($relativePathAndFilename); } + /** + * @param PersistentResource $resource + * @return string + * @throws Exception + */ public function getPublicPersistentResourceUri(PersistentResource $resource) { - return $this->localTarget->getPublicPersistentResourceUri($resource); + //return $this->localTarget->getPublicPersistentResourceUri($resource); + return $this->uriBuilder->uriFor( + 'index', + ['resourceIdentifier' => $resource], + 'Resource', + 'Sitegeist.MagicWand' + ); } /** diff --git a/Configuration/Policy.yaml b/Configuration/Policy.yaml new file mode 100644 index 0000000..90dce83 --- /dev/null +++ b/Configuration/Policy.yaml @@ -0,0 +1,11 @@ +privilegeTargets: + 'Neos\Flow\Security\Authorization\Privilege\Method\MethodPrivilege': + 'Sitegeist.MagicWand:ProxyAssets': + matcher: 'method(Sitegeist\MagicWand\Controller\ResourceController->indexAction())' + +roles: + 'Neos.Flow:Everybody': + privileges: + - + privilegeTarget: 'Sitegeist.MagicWand:ProxyAssets' + permission: GRANT diff --git a/Configuration/Routes.yaml b/Configuration/Routes.yaml new file mode 100644 index 0000000..3cdc8db --- /dev/null +++ b/Configuration/Routes.yaml @@ -0,0 +1,11 @@ +## +# MagicWand-ResourceProxy + +- + name: 'MagicWand-ResourceProxy' + uriPattern: 'magicwand/resourceproxy/{resourceIdentifier}' + defaults: + '@package': 'Sitegeist.MagicWand' + '@controller': 'Resource' + '@action': 'index' + httpMethods: ['GET'] diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 568f490..13a271c 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -1,5 +1,6 @@ Sitegeist: MagicWand: + # the local flow command flowCommand: './flow' # # preset which is used by the clone:default command @@ -51,3 +52,7 @@ Neos: storage: 'magicWandProxyStorage' target: 'magicWandProxyTarget' + mvc: + routes: + Sitegeist.MagicWand: + position: 'before Neos.Neos' From 7c17d9110fa295aa1e1cd0a008619c9db03eee27 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Thu, 17 Jan 2019 11:56:22 +0100 Subject: [PATCH 04/16] TASK: Refactor to centralized configuration of proxyConfigurations --- Classes/Command/CloneCommandController.php | 51 +++-- Classes/Controller/ResourceController.php | 5 +- .../ResourceProxyConfigurationService.php | 38 ++++ .../ResourceProxyConfiguration.php | 51 +++++ .../ProxyAwareFileSystemSymlinkTarget.php | 119 +++++++++++ .../ProxyAwareWritableFileSystemStorage.php | 68 +++++++ .../Storage/ProxyStorage.php | 191 ------------------ .../ResourceManagement/Target/ProxyTarget.php | 156 -------------- Configuration/Caches.yaml | 7 + Configuration/Objects.yaml | 9 + Configuration/Settings.yaml | 39 ++-- 11 files changed, 357 insertions(+), 377 deletions(-) create mode 100644 Classes/Domain/Service/ResourceProxyConfigurationService.php create mode 100644 Classes/Domain/ValueObjects/ResourceProxyConfiguration.php create mode 100644 Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php create mode 100644 Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php delete mode 100644 Classes/ResourceManagement/Storage/ProxyStorage.php delete mode 100644 Classes/ResourceManagement/Target/ProxyTarget.php create mode 100644 Configuration/Caches.yaml create mode 100644 Configuration/Objects.yaml diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 0c56527..714c134 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -10,6 +10,7 @@ use Neos\Utility\Arrays; use Neos\Flow\Core\Bootstrap; use Sitegeist\MagicWand\DBAL\SimpleDBAL; +use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; /** * @Flow\Scope("singleton") @@ -41,6 +42,12 @@ class CloneCommandController extends AbstractCommandController */ protected $dbal; + /** + * @Flow\Inject + * @var ResourceProxyConfigurationService + */ + protected $resourceProxyConfigurationService; + /** * Show the list of predefined clone configurations */ @@ -93,6 +100,9 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) { if (count($this->clonePresets) > 0) { if ($this->clonePresets && array_key_exists($presetName, $this->clonePresets)) { + + $this->resourceProxyConfigurationService->setCurrentResourceProxyConfiguration(Arrays::getValueByPath($this->clonePresets, $presetName . '.resourceProxy')); + $this->outputLine('Clone by preset ' . $presetName); $this->remoteHostCommand( $this->clonePresets[$presetName]['host'], @@ -295,18 +305,35 @@ public function remoteHostCommand( # Transfer Files # ################## - $this->outputHeadLine('Transfer Files'); - $this->executeLocalShellCommand( - 'rsync -e "ssh -p %s %s" -kLr %s@%s:%s/* %s', - [ - $port, - addslashes($sshOptions), - $user, - $host, - $remoteDataPersistentPath, - $localDataPersistentPath - ] - ); + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + + if (!$resourceProxyConfiguration) { + $this->outputHeadLine('Transfer Files'); + $this->executeLocalShellCommand( + 'rsync -e "ssh -p %s %s" -kLr %s@%s:%s/* %s', + [ + $port, + addslashes($sshOptions), + $user, + $host, + $remoteDataPersistentPath, + $localDataPersistentPath + ] + ); + } else { + $this->outputHeadLine('Transfer Files - without Resources because a resourceProxyConfiguration is found'); + $this->executeLocalShellCommand( + 'rsync -e "ssh -p %s %s" --exclude "Resources/*" -kLr %s@%s:%s/* %s', + [ + $port, + addslashes($sshOptions), + $user, + $host, + $remoteDataPersistentPath, + $localDataPersistentPath + ] + ); + } ######################### # Transfer Translations # diff --git a/Classes/Controller/ResourceController.php b/Classes/Controller/ResourceController.php index 074b8e3..2d8ab0b 100644 --- a/Classes/Controller/ResourceController.php +++ b/Classes/Controller/ResourceController.php @@ -28,7 +28,10 @@ public function indexAction(string $resourceIdentifier) { $headers = $this->response->getHeaders(); $headers->set('Content-Type', $resource->getMediaType(), true); $this->response->setHeaders($headers); - return stream_get_contents($resource->getStream()); + $sourceStream = $resource->getStream(); + $streamContent = stream_get_contents($sourceStream); + fclose($sourceStream); + return $streamContent; } else { throw new ResourceNotFoundException("Unkonwn Resource"); } diff --git a/Classes/Domain/Service/ResourceProxyConfigurationService.php b/Classes/Domain/Service/ResourceProxyConfigurationService.php new file mode 100644 index 0000000..1ebf2fe --- /dev/null +++ b/Classes/Domain/Service/ResourceProxyConfigurationService.php @@ -0,0 +1,38 @@ +resourceProxyConfigurationCache->get('current'); + if ($configuration) { + return new ResourceProxyConfiguration( + $configuration['baseUri'], + $configuration['curlOptions'] ?? [] , + $configuration['subdivideHashPathSegment'] ?? false + ); + } else { + return null; + } + } + + /** + * + */ + public function setCurrentResourceProxyConfiguration($configuration) { + $this->resourceProxyConfigurationCache->set('current', $configuration); + } +} diff --git a/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php b/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php new file mode 100644 index 0000000..2e88c44 --- /dev/null +++ b/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php @@ -0,0 +1,51 @@ +baseUri = $baseUri; + $this->curlOptions = $curlOptions; + $this->subdivideHashPathSegment = $subdivideHashPathSegment; + } + + /** + * @return string + */ + public function getBaseUri(): string + { + return $this->baseUri; + } + + /** + * @return array + */ + public function getCurlOptions(): array + { + return $this->curlOptions; + } + + /** + * @return bool + */ + public function isSubdivideHashPathSegment(): bool + { + return $this->subdivideHashPathSegment; + } +} diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php new file mode 100644 index 0000000..f66b269 --- /dev/null +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -0,0 +1,119 @@ +bootstrap->getActiveRequestHandler(); + if ($requestHandler instanceof HttpRequestHandlerInterface) { + $request = new ActionRequest($requestHandler->getHttpRequest()); + $this->uriBuilder->setRequest($request); + } + parent::initializeObject(); + } + + /** + * Publishes the whole collection to this target + * + * @param CollectionInterface $collection The collection to publish + * @param callable $callback Function called after each resource publishing + * @return void + */ + public function publishCollection(CollectionInterface $collection, callable $callback = null) + { + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + if ($resourceProxyConfiguration === null) { + return parent::publishCollection($collection, $callback); + } + + foreach ($collection->getObjects($callback) as $object) { + /** @var StorageObject $object */ + $sourceStream = $object->getStream(); + if ($sourceStream !== false) { + $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($object)); + fclose($sourceStream); + } + + } + } + + /** + * @param PersistentResource $resource + * @param CollectionInterface $collection + */ + public function publishResource(PersistentResource $resource, CollectionInterface $collection) + { + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + if ($resourceProxyConfiguration === null) { + return parent::publishResource($resource, $collection); + } + + $sourceStream = $resource->getStream(); + if ($sourceStream !== false) { + return parent::publishResource($resource, $collection); + fclose($sourceStream); + } + } + + /** + * @param resource $sourceStream + * @param string $relativeTargetPathAndFilename + * @throws Exception + */ + protected function publishFile($sourceStream, $relativeTargetPathAndFilename) + { + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + if ($resourceProxyConfiguration === null) { + return parent::publishFile($sourceStream, $relativeTargetPathAndFilename); + } + + if ($sourceStream === false) { + return; + } else { + parent::publishFile($sourceStream, $relativeTargetPathAndFilename); + } + } + + /** + * @param PersistentResource $resource + * @return string + * @throws Exception + */ + public function getPublicPersistentResourceUri(PersistentResource $resource) + { + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + if ($resourceProxyConfiguration === null) { + return parent::getPublicPersistentResourceUri($resource); + } + + return $this->uriBuilder->uriFor( + 'index', + ['resourceIdentifier' => $resource], + 'Resource', + 'Sitegeist.MagicWand' + ); + } +} diff --git a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php new file mode 100644 index 0000000..ac33883 --- /dev/null +++ b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php @@ -0,0 +1,68 @@ +resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + if (!$resourceProxyConfiguration) { + return parent::getStreamByResource($resource); + } + + $localResourceStream = parent::getStreamByResource($resource); + if ($localResourceStream) { + return $localResourceStream; + } else { + $curlEngine = new CurlEngine(); + foreach($resourceProxyConfiguration->getCurlOptions() as $key => $value) { + $curlEngine->setOption(constant($key), $value); + } + + $browser = new Browser(); + $browser->setRequestEngine($curlEngine); + + $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + + $response = $browser->request($uri); + + if ($response->getStatusCode() == 200 ) { + $response->getContent(); + + $stream = fopen('php://memory', 'r+'); + fwrite($stream, $response->getContent()); + rewind($stream); + + parent::importResource($stream, $resource->getCollectionName()); + + return $stream; + } else { + throw new ResourceNotFoundException('Kennichnich'); + } + } + } +} diff --git a/Classes/ResourceManagement/Storage/ProxyStorage.php b/Classes/ResourceManagement/Storage/ProxyStorage.php deleted file mode 100644 index ebe9305..0000000 --- a/Classes/ResourceManagement/Storage/ProxyStorage.php +++ /dev/null @@ -1,191 +0,0 @@ -name = $name; - if (!isset($options['localStorage'])) { - throw new Exception(sprintf('localStorage-option is required in storage %s', $name), 1547635803); - } - $this->localStorageName = $options['localStorage']; - } - - public function initializeObject() - { - $this->localStorage = $this->initializeStorage($this->localStorageName); - } - - /** - * @return string - */ - public function getName() - { - return $this->name; - } - - /** - * @param PersistentResource $resource - * @return bool|resource - */ - public function getStreamByResource(PersistentResource $resource) - { - $localResourceStream = $this->localStorage->getStreamByResource($resource); - if ($localResourceStream) { - return $localResourceStream; - } else { - - $curlEngine = new CurlEngine(); - - $browser = new Browser(); - $browser->setRequestEngine($curlEngine); - - $uri = $baseUri . '/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); - - $response = $browser->request($uri); - if ($response->getStatusCode() == 200 ) { - $response->getContent(); - - $stream = fopen('php://memory', 'r+'); - fwrite($stream, $response->getContent()); - rewind($stream); - - $this->localStorage->importResource($stream, $resource->getCollectionName()); - - return $stream; - } else { - - \Neos\Flow\var_dump($uri); - throw new ResourceNotFoundException('Kennichnich'); - } - } - } - - /** - * @param string $relativePath - * @return bool|resource - */ - public function getStreamByResourcePath($relativePath) - { - return $this->localStorage->getStreamByResourcePath($relativePath); - } - - /** - * @return \Generator - */ - public function getObjects() - { - return $this->localStorage->getObjects(); - } - - /** - * @param CollectionInterface $collection - * @return \Generator - */ - public function getObjectsByCollection(CollectionInterface $collection) - { - return $this->localStorage->getObjectsByCollection($collection); - } - - /** - * @param resource|string $source - * @param string $collectionName - * @return PersistentResource - * @throws StorageException - */ - public function importResource($source, $collectionName) - { - return $this->localStorage->importResource($source, $collectionName); - } - - /** - * @param string $content - * @param string $collectionName - * @return PersistentResource - * @throws StorageException - */ - public function importResourceFromContent($content, $collectionName) - { - return $this->localStorage->importResourceFromContent($content, $collectionName); - } - - /** - * @param PersistentResource $resource - * @return bool - * @throws \Exception - */ - public function deleteResource(PersistentResource $resource) - { - if ($this->isLocal($resource)) { - return $this->localStorage->deleteResource($resource); - } else { - throw new \Exception('proxy-resources cannot be deleted'); - } - } - - /** - * @param PersistentResource $resource - * @return bool - */ - protected function isLocal(PersistentResource $resource) - { - return true; - } - - /** - * @param string $name - * @return StorageInterface - * @see \Neos\Flow\ResourceManagement\ResourceManager::initializeStorages - */ - protected function initializeStorage($name) - { - $storageDefinition = Arrays::getValueByPath($this->settings, 'storages.' . $name); - - if (!isset($storageDefinition['storage'])) { - throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has no valid "storage" option. Please check the configuration syntax and make sure to specify a valid storage class name.', $name), 1361467211); - } - if (!class_exists($storageDefinition['storage'])) { - throw new Exception(sprintf('The configuration for the resource storage "%s" defined in your settings has not defined a valid "storage" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $name, $storageDefinition['storage']), 1361467212); - } - $options = (isset($storageDefinition['storageOptions']) ? $storageDefinition['storageOptions'] : []); - return new $storageDefinition['storage']($this->localStorageName, $options); - } -} diff --git a/Classes/ResourceManagement/Target/ProxyTarget.php b/Classes/ResourceManagement/Target/ProxyTarget.php deleted file mode 100644 index 07380f1..0000000 --- a/Classes/ResourceManagement/Target/ProxyTarget.php +++ /dev/null @@ -1,156 +0,0 @@ -name = $name; - if (!isset($options['localTarget'])) { - throw new Exception(sprintf('localTarget-option is required in target %s', $name), 1547635863); - } - $this->localTargetName = $options['localTarget']; - } - - public function initializeObject() { - // intialize uribuilder with request - $requestHandler = $this->bootstrap->getActiveRequestHandler(); - if ($requestHandler instanceof HttpRequestHandlerInterface) { - $request = new ActionRequest($requestHandler->getHttpRequest()); - $this->uriBuilder->setRequest($request); - } - - // initialize targets - $this->localTarget = $this->initalizeTarget($this->localTargetName); - } - - /** - * @return string - */ - public function getName() - { - return $this->name; - } - - /** - * @param CollectionInterface $collection - */ - public function publishCollection(CollectionInterface $collection) - { - return $this->localTarget->publishCollection($collection); - } - - /** - * @param PersistentResource $resource - * @param CollectionInterface $collection - * @throws Exception - */ - public function publishResource(PersistentResource $resource, CollectionInterface $collection) - { - return $this->localTarget->publishResource($resource, $collection); - } - - /** - * @param PersistentResource $resource - */ - public function unpublishResource(PersistentResource $resource) - { - return $this->localTarget->unpublishResource($resource); - } - - /** - * @param string $relativePathAndFilename - * @return string - */ - public function getPublicStaticResourceUri($relativePathAndFilename) - { - return $this->localTarget->getPublicStaticResourceUri($relativePathAndFilename); - } - - /** - * @param PersistentResource $resource - * @return string - * @throws Exception - */ - public function getPublicPersistentResourceUri(PersistentResource $resource) - { - //return $this->localTarget->getPublicPersistentResourceUri($resource); - return $this->uriBuilder->uriFor( - 'index', - ['resourceIdentifier' => $resource], - 'Resource', - 'Sitegeist.MagicWand' - ); - } - - /** - * @param string $name - * @return TargetInterface - * @throws Exception - * @see \Neos\Flow\ResourceManagement\ResourceManager::initializeTargets - */ - protected function initalizeTarget($name) - { - $targetDefinition = Arrays::getValueByPath($this->settings, 'targets.' . $name); - if (!isset($targetDefinition['target'])) { - throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has no valid "target" option. Please check the configuration syntax and make sure to specify a valid target class name.', $name), 1361467838); - } - if (!class_exists($targetDefinition['target'])) { - throw new Exception(sprintf('The configuration for the resource target "%s" defined in your settings has not defined a valid "target" option. Please check the configuration syntax and make sure that the specified class "%s" really exists.', $name, $targetDefinition['target']), 1361467839); - } - $options = (isset($targetDefinition['targetOptions']) ? $targetDefinition['targetOptions'] : []); - return new $targetDefinition['target']($this->localTargetName, $options); - } - -} diff --git a/Configuration/Caches.yaml b/Configuration/Caches.yaml new file mode 100644 index 0000000..25ac639 --- /dev/null +++ b/Configuration/Caches.yaml @@ -0,0 +1,7 @@ + +Sitegeist_Magicwand_ResourceProxyConfigurations: + persistent: true + frontend: Neos\Cache\Frontend\VariableFrontend + backend: Neos\Cache\Backend\FileBackend + backendOptions: + defaultLifetime: 0 diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml new file mode 100644 index 0000000..e8ca3a5 --- /dev/null +++ b/Configuration/Objects.yaml @@ -0,0 +1,9 @@ +'Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService': + properties: + resourceProxyConfigurationCache: + object: + factoryObjectName: Neos\Flow\Cache\CacheManager + factoryMethodName: getCache + arguments: + 1: + value: Sitegeist_Magicwand_ResourceProxyConfigurations diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 13a271c..fc6e1a0 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -8,49 +8,54 @@ Sitegeist: clonePresets: [] # # the name of the preset for referencing on the clone:preset command # master: +# # # hostname or ip of the server to clone from # host: ~ +# # # ssh username # user: ~ +# # # ssh port # port: ~ +# # # ssh options # sshOptions: ~ +# # # path on the remote server # path: ~ +# # # flow-context on the remote server # context: Production +# # # the flow cli command on the remote server # # default is the main flowCommand-Setting # flowCommand: ~ +# # # commands to execute after cloning # postClone: # - './flow help' +# +# # informations to acess the resources of the cloned setup via http +# # if this is confiured the rsync of the persistent resources is skipped +# # and instead resources are fetched and imported on the fly once they +# # are read +# resourceProxy: +# baseUri: http://vour.server.tld +# curlOptions: +# CURLOPT_USERPWD: very:secure +# subdivideHashPathSegment: false Neos: Flow: resource: storages: - magicWandProxyStorage: - storage: 'Sitegeist\MagicWand\ResourceManagement\Storage\ProxyStorage' - storageOptions: - localStorage: defaultPersistentResourcesStorage + defaultPersistentResourcesStorage: + storage: 'Sitegeist\MagicWand\ResourceManagement\ProxyAwareWritableFileSystemStorage' targets: - magicWandProxyTarget: - target: 'Sitegeist\MagicWand\ResourceManagement\Target\ProxyTarget' - targetOptions: - localTarget: localWebDirectoryPersistentResourcesTarget - - # - # alter the settings for the persistent collection - # to use the proxy storage and target instead - # - collections: - persistent: - storage: 'magicWandProxyStorage' - target: 'magicWandProxyTarget' + localWebDirectoryPersistentResourcesTarget: + target: 'Sitegeist\MagicWand\ResourceManagement\ProxyAwareFileSystemSymlinkTarget' mvc: routes: From 112143ead5fb5d258c05c72620003670f96e8247 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Fri, 18 Jan 2019 12:30:18 +0100 Subject: [PATCH 05/16] TASK: Send redirect after the resource was read and this imported from proxy --- Classes/Controller/ResourceController.php | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/Classes/Controller/ResourceController.php b/Classes/Controller/ResourceController.php index 2d8ab0b..a20345b 100644 --- a/Classes/Controller/ResourceController.php +++ b/Classes/Controller/ResourceController.php @@ -5,6 +5,7 @@ use Neos\Flow\Mvc\Controller\ActionController; use Neos\Flow\ResourceManagement\PersistentResource; use Neos\Flow\ResourceManagement\ResourceRepository; +use Neos\Flow\ResourceManagement\ResourceManager; use Sitegeist\MagicWand\ResourceManagement\ResourceNotFoundException; class ResourceController extends ActionController @@ -16,6 +17,12 @@ class ResourceController extends ActionController */ protected $resourceRepository; + /** + * @var ResourceManager + * @Flow\Inject + */ + protected $resourceManager; + /** * @param string $resourceIdentifier */ @@ -25,15 +32,12 @@ public function indexAction(string $resourceIdentifier) { */ $resource = $this->resourceRepository->findByIdentifier($resourceIdentifier); if ($resource) { - $headers = $this->response->getHeaders(); - $headers->set('Content-Type', $resource->getMediaType(), true); - $this->response->setHeaders($headers); $sourceStream = $resource->getStream(); - $streamContent = stream_get_contents($sourceStream); - fclose($sourceStream); - return $streamContent; - } else { - throw new ResourceNotFoundException("Unkonwn Resource"); + if ($sourceStream !== false) { + fclose($sourceStream); + $this->redirectToUri($this->resourceManager->getPublicPersistentResourceUri($resource), 0, 302); + } } + throw new ResourceNotFoundException("Unknown resource"); } } From 9e555bf7a6fb7de6967b6e86f8e82b33243e9254 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Fri, 18 Jan 2019 12:30:51 +0100 Subject: [PATCH 06/16] TASK: Centralize configuration of resource proxies --- .../ResourceProxyConfigurationService.php | 10 +- .../ResourceProxyConfiguration.php | 16 +-- .../ProxyAwareFileSystemSymlinkTarget.php | 102 +++++++++--------- .../ProxyAwareWritableFileSystemStorage.php | 36 +++++-- Configuration/Settings.yaml | 8 +- README.md | 9 ++ 6 files changed, 103 insertions(+), 78 deletions(-) diff --git a/Classes/Domain/Service/ResourceProxyConfigurationService.php b/Classes/Domain/Service/ResourceProxyConfigurationService.php index 1ebf2fe..b0f9612 100644 --- a/Classes/Domain/Service/ResourceProxyConfigurationService.php +++ b/Classes/Domain/Service/ResourceProxyConfigurationService.php @@ -21,14 +21,20 @@ public function getCurrentResourceProxyConfiguration() { if ($configuration) { return new ResourceProxyConfiguration( $configuration['baseUri'], - $configuration['curlOptions'] ?? [] , - $configuration['subdivideHashPathSegment'] ?? false + $configuration['curlOptions'] ?? [] ); } else { return null; } } + /** + * @return boolean + */ + public function hasCurrentResourceProxyConfiguration() { + return $this->resourceProxyConfigurationCache->has('current'); + } + /** * */ diff --git a/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php b/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php index 2e88c44..6db2287 100644 --- a/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php +++ b/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php @@ -13,16 +13,10 @@ class ResourceProxyConfiguration */ protected $curlOptions; - /** - * @var bool - */ - protected $subdivideHashPathSegment; - - public function __construct(string $baseUri, array $curlOptions = [], bool $subdivideHashPathSegment = false) + public function __construct(string $baseUri, array $curlOptions = []) { $this->baseUri = $baseUri; $this->curlOptions = $curlOptions; - $this->subdivideHashPathSegment = $subdivideHashPathSegment; } /** @@ -40,12 +34,4 @@ public function getCurlOptions(): array { return $this->curlOptions; } - - /** - * @return bool - */ - public function isSubdivideHashPathSegment(): bool - { - return $this->subdivideHashPathSegment; - } } diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php index f66b269..fe23e48 100644 --- a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -4,12 +4,16 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\ResourceManagement\CollectionInterface; use Neos\Flow\ResourceManagement\PersistentResource; +use Neos\Flow\ResourceManagement\ResourceManager; +use Neos\Flow\ResourceManagement\ResourceRepository; use Neos\Flow\ResourceManagement\Target\Exception; use Neos\Flow\ResourceManagement\Target\FileSystemSymlinkTarget; -use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; use Neos\Flow\Mvc\Routing\UriBuilder; use Neos\Flow\Http\HttpRequestHandlerInterface; use Neos\Flow\Mvc\ActionRequest; +use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; +use Sitegeist\MagicWand\ResourceManagement\ProxyAwareWritableFileSystemStorage; +use Neos\Flow\ResourceManagement\Storage\StorageObject; class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget { @@ -25,6 +29,18 @@ class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget */ protected $uriBuilder; + /** + * @var ResourceRepository + * @Flow\Inject + */ + protected $resourceRepository; + + /** + * @var ResourceManager + * @Flow\Inject + */ + protected $resourceManager; + public function initializeObject() { // intialize uribuilder with request $requestHandler = $this->bootstrap->getActiveRequestHandler(); @@ -44,76 +60,64 @@ public function initializeObject() { */ public function publishCollection(CollectionInterface $collection, callable $callback = null) { - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - if ($resourceProxyConfiguration === null) { + if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { + return parent::publishCollection($collection, $callback); + } + + /** + * @var ProxyAwareWritableFileSystemStorage $storage + */ + $storage = $collection->getStorage(); + if (!$storage instanceof ProxyAwareWritableFileSystemStorage) { return parent::publishCollection($collection, $callback); } foreach ($collection->getObjects($callback) as $object) { /** @var StorageObject $object */ - $sourceStream = $object->getStream(); - if ($sourceStream !== false) { - $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($object)); - fclose($sourceStream); + $isPresent = $storage->resourceIsPresentInStorage($object); + if ($isPresent === false) { + // this storage ignores resources that are not yet in the filesystem as they + // are optimistically created during read operations + continue; } - - } - } - - /** - * @param PersistentResource $resource - * @param CollectionInterface $collection - */ - public function publishResource(PersistentResource $resource, CollectionInterface $collection) - { - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - if ($resourceProxyConfiguration === null) { - return parent::publishResource($resource, $collection); - } - - $sourceStream = $resource->getStream(); - if ($sourceStream !== false) { - return parent::publishResource($resource, $collection); + $sourceStream = $object->getStream(); + $this->publishFile($sourceStream, $this->getRelativePublicationPathAndFilename($object)); fclose($sourceStream); } } /** - * @param resource $sourceStream - * @param string $relativeTargetPathAndFilename + * @param PersistentResource $resource + * @return string * @throws Exception */ - protected function publishFile($sourceStream, $relativeTargetPathAndFilename) + public function getPublicPersistentResourceUri(PersistentResource $resource) { - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - if ($resourceProxyConfiguration === null) { - return parent::publishFile($sourceStream, $relativeTargetPathAndFilename); + if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { + return parent::getPublicPersistentResourceUri($resource); } - if ($sourceStream === false) { - return; + $collection = $this->resourceManager->getCollection($resource->getCollectionName()); + $storage = $collection->getStorage(); + $isPresent = $storage->resourceIsPresentInStorage($resource); + + if ($isPresent) { + return parent::getPublicPersistentResourceUri($resource); } else { - parent::publishFile($sourceStream, $relativeTargetPathAndFilename); + return $this->uriBuilder->uriFor( + 'index', + ['resourceIdentifier' => $resource], + 'Resource', + 'Sitegeist.MagicWand' + ); } } /** - * @param PersistentResource $resource - * @return string - * @throws Exception + * @return bool */ - public function getPublicPersistentResourceUri(PersistentResource $resource) + public function isSubdivideHashPathSegment(): bool { - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - if ($resourceProxyConfiguration === null) { - return parent::getPublicPersistentResourceUri($resource); - } - - return $this->uriBuilder->uriFor( - 'index', - ['resourceIdentifier' => $resource], - 'Resource', - 'Sitegeist.MagicWand' - ); + return $this->subdivideHashPathSegment; } } diff --git a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php index ac33883..7af1f6c 100644 --- a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php +++ b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php @@ -6,6 +6,7 @@ use Neos\Flow\Http\Client\CurlEngine; use Neos\Flow\ResourceManagement\PersistentResource; use Neos\Flow\ResourceManagement\ResourceManager; +use Neos\Flow\ResourceManagement\ResourceMetaDataInterface; use Neos\Flow\ResourceManagement\Storage\WritableFileSystemStorage; use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; @@ -23,21 +24,36 @@ class ProxyAwareWritableFileSystemStorage extends WritableFileSystemStorage */ protected $resourceManager; + /** + * @param PersistentResource $resource + * @return string + */ + public function resourceIsPresentInStorage(ResourceMetaDataInterface $resource) { + $path = $this->getStoragePathAndFilenameByHash($resource->getSha1()); + return file_exists($path); + } + /** * @param PersistentResource $resource * @return bool|resource */ public function getStreamByResource(PersistentResource $resource) { - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - if (!$resourceProxyConfiguration) { + if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { return parent::getStreamByResource($resource); } - $localResourceStream = parent::getStreamByResource($resource); - if ($localResourceStream) { - return $localResourceStream; + $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + $isPresent = $this->resourceIsPresentInStorage($resource); + if ($isPresent) { + return parent::getStreamByResource($resource); } else { + $collection = $this->resourceManager->getCollection($resource->getCollectionName()); + /** + * @var ProxyAwareFileSystemSymlinkTarget $target + */ + $target = $collection->getTarget(); + $curlEngine = new CurlEngine(); foreach($resourceProxyConfiguration->getCurlOptions() as $key => $value) { $curlEngine->setOption(constant($key), $value); @@ -46,7 +62,12 @@ public function getStreamByResource(PersistentResource $resource) $browser = new Browser(); $browser->setRequestEngine($curlEngine); - $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + if ($target instanceof ProxyAwareFileSystemSymlinkTarget && $target->isSubdivideHashPathSegment()) { + $sha1Hash = $resource->getSha1(); + $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . $object->getFilename(); + } else { + $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + } $response = $browser->request($uri); @@ -57,7 +78,8 @@ public function getStreamByResource(PersistentResource $resource) fwrite($stream, $response->getContent()); rewind($stream); - parent::importResource($stream, $resource->getCollectionName()); + $collection->importResource($stream); + $target->publishResource($resource, $collection); return $stream; } else { diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index fc6e1a0..9f2a66c 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -35,15 +35,13 @@ Sitegeist: # postClone: # - './flow help' # -# # informations to acess the resources of the cloned setup via http -# # if this is confiured the rsync of the persistent resources is skipped -# # and instead resources are fetched and imported on the fly once they -# # are read +# # informations to access the resources of the cloned setup via http +# # if this is configured the rsync of the persistent resources is skipped +# # and instead resources are fetched and imported on the fly once read # resourceProxy: # baseUri: http://vour.server.tld # curlOptions: # CURLOPT_USERPWD: very:secure -# subdivideHashPathSegment: false Neos: Flow: diff --git a/README.md b/README.md index a218b15..982fc8c 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,15 @@ Sitegeist: # commands to execute after cloning like ./flow user:create ... postClone: [] + + # informations to access the resources of the cloned setup via http + # if this is configured the rsync of the persistent resources is skipped + # and instead resources are fetched and imported on the fly once read + resourceProxy: + baseUri: http://vour.server.tld + # curl options + curlOptions: + CURLOPT_USERPWD: very:secure ``` The settings should be added to the global `Settings.yaml` of the project, so that every From 5455a5e5aa85c0eecb39e1231e3f046147405067 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Mon, 21 Jan 2019 09:06:25 +0100 Subject: [PATCH 07/16] TASK: Relax version-constraint to support flow 4.x To support lower neos versions that had different type-annotations the outputLine and outputHeader methods had to be renamed. --- Classes/Command/AbstractCommandController.php | 14 ++-- Classes/Command/CloneCommandController.php | 66 +++++++++---------- Classes/Command/StashCommandController.php | 66 +++++++++---------- composer.json | 2 +- 4 files changed, 74 insertions(+), 74 deletions(-) diff --git a/Classes/Command/AbstractCommandController.php b/Classes/Command/AbstractCommandController.php index f20ae5a..6ef5b3b 100644 --- a/Classes/Command/AbstractCommandController.php +++ b/Classes/Command/AbstractCommandController.php @@ -53,11 +53,11 @@ protected function executeLocalShellCommand($command, $arguments = [], $options { $customizedCommand = call_user_func_array('sprintf', array_merge([$command], $arguments)); if (!in_array(self::HIDE_COMMAND, $options)) { - $this->outputLine($customizedCommand); + $this->renderLine($customizedCommand); } $customizedCommandResult = shell_exec($customizedCommand); if (is_string($customizedCommandResult) && !in_array(self::HIDE_RESULT, $options)) { - $this->outputLine($customizedCommandResult); + $this->renderLine($customizedCommandResult); } return $customizedCommandResult; } @@ -87,19 +87,19 @@ protected function executeLocalFlowCommand($command, $arguments = [], $options = /** * @param $line */ - protected function outputHeadLine($line = '', $arguments = []) + protected function renderHeadLine($line = '', $arguments = []) { $this->headlineNumber++; - $this->outputLine(); - $this->outputLine('' . $this->headlineNumber . '. ' . $line . '', $arguments); - $this->outputLine(); + $this->renderLine(); + $this->renderLine('' . $this->headlineNumber . '. ' . $line . '', $arguments); + $this->renderLine(); } /** * @param string $line * @param array $arguments */ - protected function outputLine(string $line = '', array $arguments = []) + protected function renderLine(string $line = '', array $arguments = []) { $filteredLine = $line; foreach ($this->secrets as $secret) { diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 714c134..5f5f949 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -55,19 +55,19 @@ public function listCommand() { if ($this->clonePresets) { foreach ($this->clonePresets as $presetName => $presetConfiguration) { - $this->outputHeadLine($presetName); + $this->renderHeadLine($presetName); foreach ($presetConfiguration as $key => $value) { if (is_array($value)) { - $this->outputLine(' - ' . $key . ':'); + $this->renderLine(' - ' . $key . ':'); foreach ($value as $line) { - $this->outputLine(' ' . $line); + $this->renderLine(' ' . $line); } continue; } - $this->outputLine(' - ' . $key . ': ' . $value); + $this->renderLine(' - ' . $key . ': ' . $value); } } } @@ -82,7 +82,7 @@ public function listCommand() public function defaultCommand(bool $yes = false, bool $keepDb = false) : void { if ($this->defaultPreset === null || $this->defaultPreset === '') { - $this->outputLine('There is no default preset configured!'); + $this->renderLine('There is no default preset configured!'); $this->quit(1); } @@ -103,7 +103,7 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) $this->resourceProxyConfigurationService->setCurrentResourceProxyConfiguration(Arrays::getValueByPath($this->clonePresets, $presetName . '.resourceProxy')); - $this->outputLine('Clone by preset ' . $presetName); + $this->renderLine('Clone by preset ' . $presetName); $this->remoteHostCommand( $this->clonePresets[$presetName]['host'], $this->clonePresets[$presetName]['user'], @@ -123,11 +123,11 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) ) ); } else { - $this->outputLine('The preset ' . $presetName . ' was not found!'); + $this->renderLine('The preset ' . $presetName . ' was not found!'); $this->quit(1); } } else { - $this->outputLine('No presets found!'); + $this->renderLine('No presets found!'); $this->quit(1); } } @@ -165,12 +165,12 @@ public function remoteHostCommand( } // read local configuration - $this->outputHeadLine('Read local configuration'); + $this->renderHeadLine('Read local configuration'); $localDataPersistentPath = FLOW_PATH_ROOT . 'Data/Persistent'; // read remote configuration - $this->outputHeadLine('Fetch remote configuration'); + $this->renderHeadLine('Fetch remote configuration'); $remotePersistenceConfigurationYaml = $this->executeLocalShellCommand( 'ssh -p %s %s %s@%s "cd %s; FLOW_CONTEXT=%s ' . $remoteFlowCommand @@ -198,15 +198,15 @@ public function remoteHostCommand( ################# if (!$yes) { - $this->outputLine("Are you sure you want to do this? Type 'yes' to continue: "); + $this->renderLine("Are you sure you want to do this? Type 'yes' to continue: "); $handle = fopen("php://stdin", "r"); $line = fgets($handle); if (trim($line) != 'yes') { - $this->outputLine('exit'); + $this->renderLine('exit'); $this->quit(1); } else { - $this->outputLine(); - $this->outputLine(); + $this->renderLine(); + $this->renderLine(); } } @@ -248,7 +248,7 @@ public function remoteHostCommand( ######################## if ($keepDb == false) { - $this->outputHeadLine('Drop and Recreate DB'); + $this->renderHeadLine('Drop and Recreate DB'); $emptyLocalDbSql = $this->dbal->flushDbSql($this->databaseConfiguration['driver'], $this->databaseConfiguration['dbname']); @@ -267,14 +267,14 @@ public function remoteHostCommand( ] ); } else { - $this->outputHeadLine('Skipped (Drop and Recreate DB)'); + $this->renderHeadLine('Skipped (Drop and Recreate DB)'); } ###################### # Transfer Database # ###################### - $this->outputHeadLine('Transfer Database'); + $this->renderHeadLine('Transfer Database'); $this->executeLocalShellCommand( 'ssh -p %s %s %s@%s -- %s | %s', [ @@ -308,7 +308,7 @@ public function remoteHostCommand( $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); if (!$resourceProxyConfiguration) { - $this->outputHeadLine('Transfer Files'); + $this->renderHeadLine('Transfer Files'); $this->executeLocalShellCommand( 'rsync -e "ssh -p %s %s" -kLr %s@%s:%s/* %s', [ @@ -321,7 +321,7 @@ public function remoteHostCommand( ] ); } else { - $this->outputHeadLine('Transfer Files - without Resources because a resourceProxyConfiguration is found'); + $this->renderHeadLine('Transfer Files - without Resources because a resourceProxyConfiguration is found'); $this->executeLocalShellCommand( 'rsync -e "ssh -p %s %s" --exclude "Resources/*" -kLr %s@%s:%s/* %s', [ @@ -339,7 +339,7 @@ public function remoteHostCommand( # Transfer Translations # ######################### - $this->outputHeadLine('Transfer Translations'); + $this->renderHeadLine('Transfer Translations'); $remoteDataTranslationsPath = $path . '/Data/Translations'; $localDataTranslationsPath = FLOW_PATH_ROOT . 'Data/Translations'; @@ -375,14 +375,14 @@ public function remoteHostCommand( # Clear Caches # ################ - $this->outputHeadLine('Clear Caches'); + $this->renderHeadLine('Clear Caches'); $this->executeLocalFlowCommand('flow:cache:flush'); ################## # Set DB charset # ################## - if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4') { - $this->outputHeadLine('Set DB charset'); + if ($this->databaseConfiguration['driver'] == 'pdo_mysql' && $remotePersistenceConfiguration['charset'] != 'utf8mb4' ) { + $this->renderHeadLine('Set DB charset'); $this->executeLocalFlowCommand('database:setcharset'); } @@ -390,14 +390,14 @@ public function remoteHostCommand( # Migrate DB # ############## - $this->outputHeadLine('Migrate cloned DB'); + $this->renderHeadLine('Migrate cloned DB'); $this->executeLocalFlowCommand('doctrine:migrate'); ##################### # Publish Resources # ##################### - $this->outputHeadLine('Publish Resources'); + $this->renderHeadLine('Publish Resources'); $this->executeLocalFlowCommand('resource:publish'); ############## @@ -405,7 +405,7 @@ public function remoteHostCommand( ############## if ($postClone) { - $this->outputHeadLine('Execute post_clone commands'); + $this->renderHeadLine('Execute post_clone commands'); if (is_array($postClone)) { foreach ($postClone as $postCloneCommand) { $this->executeLocalShellCommandWithFlowContext($postCloneCommand); @@ -422,8 +422,8 @@ public function remoteHostCommand( $endTimestamp = time(); $duration = $endTimestamp - $startTimestamp; - $this->outputHeadLine('Done'); - $this->outputLine('Successfully cloned in %s seconds', [$duration]); + $this->renderHeadLine('Done'); + $this->renderLine('Successfully cloned in %s seconds', [$duration]); } /** @@ -433,22 +433,22 @@ public function remoteHostCommand( */ protected function checkConfiguration($remotePersistenceConfiguration) { - $this->outputHeadLine('Check Configuration'); + $this->renderHeadLine('Check Configuration'); if (!$this->dbal->driverIsSupported($remotePersistenceConfiguration['driver']) && !$this->dbal->driverIsSupported($this->databaseConfiguration['driver'])) { - $this->outputLine(sprintf('ERROR: Only pdo_pgsql and pdo_mysql drivers are supported! Remote: "%s" Local: "%s" configured.', $remotePersistenceConfiguration['driver'], $this->databaseConfiguration['driver'])); + $this->renderLine(sprintf('ERROR: Only pdo_pgsql and pdo_mysql drivers are supported! Remote: "%s" Local: "%s" configured.', $remotePersistenceConfiguration['driver'], $this->databaseConfiguration['driver'])); $this->quit(1); } if ($remotePersistenceConfiguration['driver'] !== $this->databaseConfiguration['driver']) { - $this->outputLine('ERROR: Remote and local databases must use the same driver!'); + $this->renderLine('ERROR: Remote and local databases must use the same driver!'); $this->quit(1); } if (in_array($remotePersistenceConfiguration['charset'], ['utf8', 'utf8mb4']) && in_array($this->databaseConfiguration['charset'], ['utf8', 'utf8mb4'])) { // we accept utf8 and utf8mb4 being similar enough } else if ($remotePersistenceConfiguration['charset'] != $this->databaseConfiguration['charset']) { - $this->outputLine(sprintf('ERROR: Remote and local databases must use the same charset! Remote: "%s", Local: "%s" configured.', $remotePersistenceConfiguration['charset'], $this->databaseConfiguration['charset'])); + $this->renderLine(sprintf('ERROR: Remote and local databases must use the same charset! Remote: "%s", Local: "%s" configured.', $remotePersistenceConfiguration['charset'], $this->databaseConfiguration['charset'])); $this->quit(1); } - $this->outputLine(' - Configuration seems ok ...'); + $this->renderLine(' - Configuration seems ok ...'); } } diff --git a/Classes/Command/StashCommandController.php b/Classes/Command/StashCommandController.php index 39ab8c7..4efa509 100644 --- a/Classes/Command/StashCommandController.php +++ b/Classes/Command/StashCommandController.php @@ -59,7 +59,7 @@ public function createCommand($name) # Backup Database # ###################### - $this->outputHeadLine('Backup Database'); + $this->renderHeadLine('Backup Database'); $this->executeLocalShellCommand( 'mysqldump --single-transaction --add-drop-table --host="%s" --user="%s" --password="%s" %s > %s', [ @@ -75,7 +75,7 @@ public function createCommand($name) # Backup Persistent Resources # ############################### - $this->outputHeadLine('Backup Persistent Resources'); + $this->renderHeadLine('Backup Persistent Resources'); $this->executeLocalShellCommand( 'cp -al %s %s', [ @@ -91,8 +91,8 @@ public function createCommand($name) $endTimestamp = time(); $duration = $endTimestamp - $startTimestamp; - $this->outputHeadLine('Done'); - $this->outputLine('Successfuly stashed %s in %s seconds', [$name, $duration]); + $this->renderHeadLine('Done'); + $this->renderLine('Successfuly stashed %s in %s seconds', [$name, $duration]); } /** @@ -105,7 +105,7 @@ public function listCommand() $basePath = sprintf(FLOW_PATH_ROOT . 'Data/MagicWandStash'); if (!is_dir($basePath)) { - $this->outputLine('Stash is empty.'); + $this->renderLine('Stash is empty.'); $this->quit(1); } @@ -114,13 +114,13 @@ public function listCommand() foreach ($baseDir as $entry) { if (!in_array($entry, ['.', '..'])) { - $this->outputLine(' • %s', [$entry->getFilename()]); + $this->renderLine(' • %s', [$entry->getFilename()]); $anyEntry = true; } } if (!$anyEntry) { - $this->outputLine('Stash is empty.'); + $this->renderLine('Stash is empty.'); $this->quit(1); } } @@ -144,8 +144,8 @@ public function clearCommand() $endTimestamp = time(); $duration = $endTimestamp - $startTimestamp; - $this->outputHeadLine('Done'); - $this->outputLine('Cleanup successful in %s seconds', [$duration]); + $this->renderHeadLine('Done'); + $this->renderLine('Cleanup successful in %s seconds', [$duration]); } /** @@ -174,21 +174,21 @@ public function removeCommand($name, $yes = false) $directory = FLOW_PATH_ROOT . 'Data/MagicWandStash/' . $name; if (!is_dir($directory)) { - $this->outputLine('%s does not exist', [$name]); + $this->renderLine('%s does not exist', [$name]); $this->quit(1); } if (!$yes) { - $this->outputLine("Are you sure you want to do this? Type 'yes' to continue: "); + $this->renderLine("Are you sure you want to do this? Type 'yes' to continue: "); $handle = fopen("php://stdin", "r"); $line = fgets($handle); if (trim($line) != 'yes') { - $this->outputLine('exit'); + $this->renderLine('exit'); $this->quit(1); } else { - $this->outputLine(); - $this->outputLine(); + $this->renderLine(); + $this->renderLine(); } } @@ -208,8 +208,8 @@ public function removeCommand($name, $yes = false) $endTimestamp = time(); $duration = $endTimestamp - $startTimestamp; - $this->outputHeadLine('Done'); - $this->outputLine('Cleanup removed stash %s in %s seconds', [$name, $duration]); + $this->renderHeadLine('Done'); + $this->renderLine('Cleanup removed stash %s in %s seconds', [$name, $duration]); } /** @@ -224,7 +224,7 @@ public function removeCommand($name, $yes = false) protected function restoreStashEntry($source, $name, $force = false, $preserve = true, $keepDb = false) { if (!is_dir($source)) { - $this->outputLine('%s does not exist', [$name]); + $this->renderLine('%s does not exist', [$name]); $this->quit(1); } @@ -233,16 +233,16 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = ################# if (!$force) { - $this->outputLine("Are you sure you want to do this? Type 'yes' to continue: "); + $this->renderLine("Are you sure you want to do this? Type 'yes' to continue: "); $handle = fopen("php://stdin", "r"); $line = fgets($handle); if (trim($line) != 'yes') { - $this->outputLine('exit'); + $this->renderLine('exit'); $this->quit(1); } else { - $this->outputLine(); - $this->outputLine(); + $this->renderLine(); + $this->renderLine(); } } @@ -270,7 +270,7 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = ######################## if ($keepDb == false) { - $this->outputHeadLine('Drop and Recreate DB'); + $this->renderHeadLine('Drop and Recreate DB'); $emptyLocalDbSql = 'DROP DATABASE `' . $this->databaseConfiguration['dbname'] @@ -288,14 +288,14 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = ] ); } else { - $this->outputHeadLine('Skipped (Drop and Recreate DB)'); + $this->renderHeadLine('Skipped (Drop and Recreate DB)'); } ###################### # Restore Database # ###################### - $this->outputHeadLine('Restore Database'); + $this->renderHeadLine('Restore Database'); $this->executeLocalShellCommand( 'mysql --host="%s" --user="%s" --password="%s" %s < %s', [ @@ -311,7 +311,7 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = # Restore Persistent Resources # ################################ - $this->outputHeadLine('Restore Persistent Resources'); + $this->renderHeadLine('Restore Persistent Resources'); $this->executeLocalShellCommand( 'rm -rf %s && cp -al %s %1$s', [ @@ -329,7 +329,7 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = # Clear Caches # ################ - $this->outputHeadLine('Clear Caches'); + $this->renderHeadLine('Clear Caches'); $this->executeLocalFlowCommand('flow:cache:flush'); @@ -337,14 +337,14 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = # Migrate DB # ############## - $this->outputHeadLine('Migrate DB'); + $this->renderHeadLine('Migrate DB'); $this->executeLocalFlowCommand('doctrine:migrate'); ##################### # Publish Resources # ##################### - $this->outputHeadLine('Publish Resources'); + $this->renderHeadLine('Publish Resources'); $this->executeLocalFlowCommand('resource:publish'); ################# @@ -354,8 +354,8 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = $endTimestamp = time(); $duration = $endTimestamp - $startTimestamp; - $this->outputHeadLine('Done'); - $this->outputLine('Successfuly restored %s in %s seconds', [$name, $duration]); + $this->renderHeadLine('Done'); + $this->renderLine('Successfuly restored %s in %s seconds', [$name, $duration]); } /** @@ -363,13 +363,13 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = */ protected function checkConfiguration() { - $this->outputHeadLine('Check Configuration'); + $this->renderHeadLine('Check Configuration'); if ($this->databaseConfiguration['driver'] !== 'pdo_mysql') { - $this->outputLine(' only mysql is supported'); + $this->renderLine(' only mysql is supported'); $this->quit(1); } - $this->outputLine(' - Configuration seems ok ...'); + $this->renderLine(' - Configuration seems ok ...'); } } diff --git a/composer.json b/composer.json index 23979aa..fe2bc0f 100644 --- a/composer.json +++ b/composer.json @@ -16,7 +16,7 @@ } ], "require": { - "neos/flow": "^5.0 || dev-master" + "neos/flow": "~4.0 || ~5.0 || dev-master" }, "autoload": { "psr-4": { From ec408aa361da703c18e1ce347d9b93d8203d1c8b Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Fri, 8 Feb 2019 18:34:50 +0100 Subject: [PATCH 08/16] WIP: Cleanup and unify configuration --- Classes/Command/CloneCommandController.php | 33 +++++---- Classes/Controller/ResourceController.php | 5 +- .../Domain/Service/ConfigurationService.php | 60 +++++++++++++++ .../ResourceProxyConfigurationService.php | 44 ----------- .../ResourceProxyConfiguration.php | 37 ---------- .../ProxyAwareFileSystemSymlinkTarget.php | 42 +++++------ .../ProxyAwareWritableFileSystemStorage.php | 73 +++++++++---------- Configuration/Caches.yaml | 2 +- Configuration/Objects.yaml | 6 +- Configuration/Settings.yaml | 1 + README.md | 2 + 11 files changed, 140 insertions(+), 165 deletions(-) create mode 100644 Classes/Domain/Service/ConfigurationService.php delete mode 100644 Classes/Domain/Service/ResourceProxyConfigurationService.php delete mode 100644 Classes/Domain/ValueObjects/ResourceProxyConfiguration.php diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 5f5f949..764d1ed 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -10,7 +10,7 @@ use Neos\Utility\Arrays; use Neos\Flow\Core\Bootstrap; use Sitegeist\MagicWand\DBAL\SimpleDBAL; -use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; +use Sitegeist\MagicWand\Domain\Service\ConfigurationService; /** * @Flow\Scope("singleton") @@ -44,9 +44,9 @@ class CloneCommandController extends AbstractCommandController /** * @Flow\Inject - * @var ResourceProxyConfigurationService + * @var ConfigurationService */ - protected $resourceProxyConfigurationService; + protected $configurationService; /** * Show the list of predefined clone configurations @@ -101,25 +101,26 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) if (count($this->clonePresets) > 0) { if ($this->clonePresets && array_key_exists($presetName, $this->clonePresets)) { - $this->resourceProxyConfigurationService->setCurrentResourceProxyConfiguration(Arrays::getValueByPath($this->clonePresets, $presetName . '.resourceProxy')); + $this->configurationService->setCurrentPreset($presetName); + $configuration = $this->configurationService->getCurrentConfiguration(); $this->renderLine('Clone by preset ' . $presetName); $this->remoteHostCommand( - $this->clonePresets[$presetName]['host'], - $this->clonePresets[$presetName]['user'], - $this->clonePresets[$presetName]['port'], - $this->clonePresets[$presetName]['path'], - $this->clonePresets[$presetName]['context'], - (isset($this->clonePresets[$presetName]['postClone']) ? - $this->clonePresets[$presetName]['postClone'] : null + $configuration['host'], + $configuration['user'], + $configuration['port'], + $configuration['path'], + $configuration['context'], + (isset($configuration['postClone']) ? + $configuration['postClone'] : null ), $yes, $keepDb, - (isset($this->clonePresets[$presetName]['flowCommand']) ? - $this->clonePresets[$presetName]['flowCommand'] : null + (isset($configuration['flowCommand']) ? + $configuration['flowCommand'] : null ), - (isset($this->clonePresets[$presetName]['sshOptions']) ? - $this->clonePresets[$presetName]['sshOptions'] : '' + (isset($configuration['sshOptions']) ? + $configuration['sshOptions'] : '' ) ); } else { @@ -305,7 +306,7 @@ public function remoteHostCommand( # Transfer Files # ################## - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); + $resourceProxyConfiguration = $this->configurationService->getCurrentConfigurationByPath('resourceProxy'); if (!$resourceProxyConfiguration) { $this->renderHeadLine('Transfer Files'); diff --git a/Classes/Controller/ResourceController.php b/Classes/Controller/ResourceController.php index a20345b..d6fe7ec 100644 --- a/Classes/Controller/ResourceController.php +++ b/Classes/Controller/ResourceController.php @@ -36,8 +36,11 @@ public function indexAction(string $resourceIdentifier) { if ($sourceStream !== false) { fclose($sourceStream); $this->redirectToUri($this->resourceManager->getPublicPersistentResourceUri($resource), 0, 302); + } else { + throw new ResourceNotFoundException(sprintf('Could not read stream of resource with id %s ', $resourceIdentifier)); } } - throw new ResourceNotFoundException("Unknown resource"); + + throw new ResourceNotFoundException(sprintf('Could not find any resource with id %s in local database', $resourceIdentifier)); } } diff --git a/Classes/Domain/Service/ConfigurationService.php b/Classes/Domain/Service/ConfigurationService.php new file mode 100644 index 0000000..d48c71e --- /dev/null +++ b/Classes/Domain/Service/ConfigurationService.php @@ -0,0 +1,60 @@ +clonePresetInformationCache->get('current'); + if ($cloneInformations && is_array($this->clonePresets) && array_key_exists($cloneInformations['presetName'], $this->clonePresets)) { + return $this->clonePresets[$cloneInformations['presetName']]; + } else { + return []; + } + } + + /** + * @return mixed + */ + public function getCurrentConfigurationByPath($path) + { + $currentConfiguration = $this->getCurrentConfiguration(); + return Arrays::getValueByPath($currentConfiguration, $path); + } + + /** + * @return boolean + */ + public function hasConfiguration(): bool + { + return $this->clonePresetInformationCache->has('current'); + } + + /** + * @param $presetName string + * @throws \Neos\Cache\Exception + */ + public function setCurrentPreset(string $presetName): void + { + $this->clonePresetInformationCache->set('current', ['presetName' => $presetName, 'timestamp' => time()]); + } +} diff --git a/Classes/Domain/Service/ResourceProxyConfigurationService.php b/Classes/Domain/Service/ResourceProxyConfigurationService.php deleted file mode 100644 index b0f9612..0000000 --- a/Classes/Domain/Service/ResourceProxyConfigurationService.php +++ /dev/null @@ -1,44 +0,0 @@ -resourceProxyConfigurationCache->get('current'); - if ($configuration) { - return new ResourceProxyConfiguration( - $configuration['baseUri'], - $configuration['curlOptions'] ?? [] - ); - } else { - return null; - } - } - - /** - * @return boolean - */ - public function hasCurrentResourceProxyConfiguration() { - return $this->resourceProxyConfigurationCache->has('current'); - } - - /** - * - */ - public function setCurrentResourceProxyConfiguration($configuration) { - $this->resourceProxyConfigurationCache->set('current', $configuration); - } -} diff --git a/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php b/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php deleted file mode 100644 index 6db2287..0000000 --- a/Classes/Domain/ValueObjects/ResourceProxyConfiguration.php +++ /dev/null @@ -1,37 +0,0 @@ -baseUri = $baseUri; - $this->curlOptions = $curlOptions; - } - - /** - * @return string - */ - public function getBaseUri(): string - { - return $this->baseUri; - } - - /** - * @return array - */ - public function getCurlOptions(): array - { - return $this->curlOptions; - } -} diff --git a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php index fe23e48..30f4516 100644 --- a/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php +++ b/Classes/ResourceManagement/ProxyAwareFileSystemSymlinkTarget.php @@ -11,17 +11,17 @@ use Neos\Flow\Mvc\Routing\UriBuilder; use Neos\Flow\Http\HttpRequestHandlerInterface; use Neos\Flow\Mvc\ActionRequest; -use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; +use Sitegeist\MagicWand\Domain\Service\ConfigurationService; use Sitegeist\MagicWand\ResourceManagement\ProxyAwareWritableFileSystemStorage; use Neos\Flow\ResourceManagement\Storage\StorageObject; class ProxyAwareFileSystemSymlinkTarget extends FileSystemSymlinkTarget { /** - * @var ResourceProxyConfigurationService + * @var ConfigurationService * @Flow\Inject */ - protected $resourceProxyConfigurationService; + protected $configurationService; /** * @var UriBuilder @@ -60,7 +60,7 @@ public function initializeObject() { */ public function publishCollection(CollectionInterface $collection, callable $callback = null) { - if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { + if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { return parent::publishCollection($collection, $callback); } @@ -74,8 +74,7 @@ public function publishCollection(CollectionInterface $collection, callable $cal foreach ($collection->getObjects($callback) as $object) { /** @var StorageObject $object */ - $isPresent = $storage->resourceIsPresentInStorage($object); - if ($isPresent === false) { + if ($storage->resourceIsPresentInStorage($object) === false) { // this storage ignores resources that are not yet in the filesystem as they // are optimistically created during read operations continue; @@ -93,31 +92,28 @@ public function publishCollection(CollectionInterface $collection, callable $cal */ public function getPublicPersistentResourceUri(PersistentResource $resource) { - if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { + if (!$this->configurationService->getCurrentConfigurationByPath('resourceProxy')) { return parent::getPublicPersistentResourceUri($resource); } $collection = $this->resourceManager->getCollection($resource->getCollectionName()); $storage = $collection->getStorage(); - $isPresent = $storage->resourceIsPresentInStorage($resource); - if ($isPresent) { + if (!$storage instanceof ProxyAwareWritableFileSystemStorage) { return parent::getPublicPersistentResourceUri($resource); - } else { - return $this->uriBuilder->uriFor( - 'index', - ['resourceIdentifier' => $resource], - 'Resource', - 'Sitegeist.MagicWand' - ); } - } - /** - * @return bool - */ - public function isSubdivideHashPathSegment(): bool - { - return $this->subdivideHashPathSegment; + if ($storage->resourceIsPresentInStorage($resource)) { + return parent::getPublicPersistentResourceUri($resource); + } + + // build uri to resoucre controller that will fetch and publish + // the resource asynchronously + return $this->uriBuilder->uriFor( + 'index', + ['resourceIdentifier' => $resource], + 'Resource', + 'Sitegeist.MagicWand' + ); } } diff --git a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php index 7af1f6c..d844481 100644 --- a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php +++ b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php @@ -8,15 +8,15 @@ use Neos\Flow\ResourceManagement\ResourceManager; use Neos\Flow\ResourceManagement\ResourceMetaDataInterface; use Neos\Flow\ResourceManagement\Storage\WritableFileSystemStorage; -use Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService; +use Sitegeist\MagicWand\Domain\Service\ConfigurationService; class ProxyAwareWritableFileSystemStorage extends WritableFileSystemStorage { /** - * @var ResourceProxyConfigurationService + * @var ConfigurationService * @Flow\Inject */ - protected $resourceProxyConfigurationService; + protected $configurationService; /** * @var ResourceManager @@ -39,52 +39,45 @@ public function resourceIsPresentInStorage(ResourceMetaDataInterface $resource) */ public function getStreamByResource(PersistentResource $resource) { - if ($this->resourceProxyConfigurationService->hasCurrentResourceProxyConfiguration() === false) { + $resourceProxyConfiguration = $this->configurationService->getCurrentConfigurationByPath('resourceProxy'); + if (!$resourceProxyConfiguration) { return parent::getStreamByResource($resource); } - $resourceProxyConfiguration = $this->resourceProxyConfigurationService->getCurrentResourceProxyConfiguration(); - $isPresent = $this->resourceIsPresentInStorage($resource); - if ($isPresent) { + $collection = $this->resourceManager->getCollection($resource->getCollectionName()); + $target = $collection->getTarget(); + if (!$target instanceof ProxyAwareFileSystemSymlinkTarget) { return parent::getStreamByResource($resource); - } else { - $collection = $this->resourceManager->getCollection($resource->getCollectionName()); - /** - * @var ProxyAwareFileSystemSymlinkTarget $target - */ - $target = $collection->getTarget(); - - $curlEngine = new CurlEngine(); - foreach($resourceProxyConfiguration->getCurlOptions() as $key => $value) { - $curlEngine->setOption(constant($key), $value); - } - - $browser = new Browser(); - $browser->setRequestEngine($curlEngine); - - if ($target instanceof ProxyAwareFileSystemSymlinkTarget && $target->isSubdivideHashPathSegment()) { - $sha1Hash = $resource->getSha1(); - $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . $object->getFilename(); - } else { - $uri = $resourceProxyConfiguration->getBaseUri() .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); - } + } - $response = $browser->request($uri); + $curlEngine = new CurlEngine(); + $curlOptions = $resourceProxyConfiguration['curlOptions'] ?? []; + foreach($curlOptions as $key => $value) { + $curlEngine->setOption(constant($key), $value); + } - if ($response->getStatusCode() == 200 ) { - $response->getContent(); + $browser = new Browser(); + $browser->setRequestEngine($curlEngine); - $stream = fopen('php://memory', 'r+'); - fwrite($stream, $response->getContent()); - rewind($stream); + $subdivideHashPathSegment = $resourceProxyConfiguration['subdivideHashPathSegment'] ?? false; + if ($subdivideHashPathSegment) { + $sha1Hash = $resource->getSha1(); + $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . $resource->getSha1() . '/' . $resource->getFilename();; + } else { + $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + } - $collection->importResource($stream); - $target->publishResource($resource, $collection); + $response = $browser->request($uri); - return $stream; - } else { - throw new ResourceNotFoundException('Kennichnich'); - } + if ($response->getStatusCode() == 200 ) { + $stream = $response->getBody()->detach(); + $importedResource = $collection->importResource($stream); + $target->publishResource($importedResource, $collection); + return $importedResource->getStream(); } + + throw new ResourceNotFoundException( + sprintf('Resource from uri %s returned status %s', $uri, $response->getStatusCode()) + ); } } diff --git a/Configuration/Caches.yaml b/Configuration/Caches.yaml index 25ac639..29e18cc 100644 --- a/Configuration/Caches.yaml +++ b/Configuration/Caches.yaml @@ -1,5 +1,5 @@ -Sitegeist_Magicwand_ResourceProxyConfigurations: +Sitegeist_Magicwand_ClonePresetInformation: persistent: true frontend: Neos\Cache\Frontend\VariableFrontend backend: Neos\Cache\Backend\FileBackend diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml index e8ca3a5..2bd6bc1 100644 --- a/Configuration/Objects.yaml +++ b/Configuration/Objects.yaml @@ -1,9 +1,9 @@ -'Sitegeist\MagicWand\Domain\Service\ResourceProxyConfigurationService': +'Sitegeist\MagicWand\Domain\Service\ConfigurationService': properties: - resourceProxyConfigurationCache: + clonePresetInformationCache: object: factoryObjectName: Neos\Flow\Cache\CacheManager factoryMethodName: getCache arguments: 1: - value: Sitegeist_Magicwand_ResourceProxyConfigurations + value: Sitegeist_Magicwand_ClonePresetInformation diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml index 9f2a66c..fe0b015 100644 --- a/Configuration/Settings.yaml +++ b/Configuration/Settings.yaml @@ -40,6 +40,7 @@ Sitegeist: # # and instead resources are fetched and imported on the fly once read # resourceProxy: # baseUri: http://vour.server.tld +# subdivideHashPathSegment: false # curlOptions: # CURLOPT_USERPWD: very:secure diff --git a/README.md b/README.md index 982fc8c..5077c4f 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,8 @@ Sitegeist: # and instead resources are fetched and imported on the fly once read resourceProxy: baseUri: http://vour.server.tld + # define wether or not the remote uses subdivideHashPathSegments + subdivideHashPathSegment: false # curl options curlOptions: CURLOPT_USERPWD: very:secure From 7d6c5df040f6cab1ef4de54144972436fd9531e8 Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Mon, 18 Feb 2019 10:10:07 +0100 Subject: [PATCH 09/16] BUGFIX: Fix storing and publishing of proxied assets --- .../ProxyAwareWritableFileSystemStorage.php | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php index d844481..102c899 100644 --- a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php +++ b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php @@ -9,6 +9,7 @@ use Neos\Flow\ResourceManagement\ResourceMetaDataInterface; use Neos\Flow\ResourceManagement\Storage\WritableFileSystemStorage; use Sitegeist\MagicWand\Domain\Service\ConfigurationService; +use Neos\Utility\Files; class ProxyAwareWritableFileSystemStorage extends WritableFileSystemStorage { @@ -39,6 +40,10 @@ public function resourceIsPresentInStorage(ResourceMetaDataInterface $resource) */ public function getStreamByResource(PersistentResource $resource) { + if ($this->resourceIsPresentInStorage($resource)) { + return parent::getStreamByResource($resource); + } + $resourceProxyConfiguration = $this->configurationService->getCurrentConfigurationByPath('resourceProxy'); if (!$resourceProxyConfiguration) { return parent::getStreamByResource($resource); @@ -71,9 +76,14 @@ public function getStreamByResource(PersistentResource $resource) if ($response->getStatusCode() == 200 ) { $stream = $response->getBody()->detach(); - $importedResource = $collection->importResource($stream); - $target->publishResource($importedResource, $collection); - return $importedResource->getStream(); + $targetPathAndFilename = $this->getStoragePathAndFilenameByHash($resource->getSha1()); + if (!file_exists(dirname($targetPathAndFilename))) { + Files::createDirectoryRecursively(dirname($targetPathAndFilename)); + } + file_put_contents($targetPathAndFilename, stream_get_contents($stream)); + $this->fixFilePermissions($targetPathAndFilename); + $target->publishResource($resource, $collection); + return $stream; } throw new ResourceNotFoundException( From 611d9d6230c69c1987e2a80deb242093e022cd3a Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Mon, 18 Feb 2019 10:35:06 +0100 Subject: [PATCH 10/16] TASK: Implement configuration preview in clone:list as yaml export --- Classes/Command/CloneCommandController.php | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 764d1ed..e7a0012 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -11,6 +11,7 @@ use Neos\Flow\Core\Bootstrap; use Sitegeist\MagicWand\DBAL\SimpleDBAL; use Sitegeist\MagicWand\Domain\Service\ConfigurationService; +use Symfony\Component\Yaml\Yaml; /** * @Flow\Scope("singleton") @@ -56,18 +57,10 @@ public function listCommand() if ($this->clonePresets) { foreach ($this->clonePresets as $presetName => $presetConfiguration) { $this->renderHeadLine($presetName); - foreach ($presetConfiguration as $key => $value) { - if (is_array($value)) { - $this->renderLine(' - ' . $key . ':'); - - foreach ($value as $line) { - $this->renderLine(' ' . $line); - } - - continue; - } - - $this->renderLine(' - ' . $key . ': ' . $value); + $presetConfigurationAsYaml = Yaml::dump($presetConfiguration); + $lines = explode(PHP_EOL, $presetConfigurationAsYaml); + foreach ($lines as $line) { + $this->renderLine($line); } } } From c7b9a172cbf6779efd180c9527ff6e83a83e814e Mon Sep 17 00:00:00 2001 From: Martin Ficzel Date: Mon, 18 Feb 2019 10:38:47 +0100 Subject: [PATCH 11/16] !!! TASK: Remove clone:remoteHost from cli This is not compatible with magicwand now having an understanding of the last cloned preset. --- Classes/Command/CloneCommandController.php | 4 ++-- README.md | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index e7a0012..f50b713 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -98,7 +98,7 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) $configuration = $this->configurationService->getCurrentConfiguration(); $this->renderLine('Clone by preset ' . $presetName); - $this->remoteHostCommand( + $this->cloneRemoteHost( $configuration['host'], $configuration['user'], $configuration['port'], @@ -140,7 +140,7 @@ public function presetCommand($presetName, $yes = false, $keepDb = false) * @param string $remoteFlowCommand the flow command to execute on the remote system * @param string $sshOptions additional options for the ssh command */ - public function remoteHostCommand( + protected function cloneRemoteHost( $host, $user, $port, diff --git a/README.md b/README.md index 5077c4f..4485ff7 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ section of your composer.json**. ## Easy and fast cloning of Flow and Neos Installations -The CLI commands `clone:list`, `clone:preset` and `clone:remotehost` help to +The CLI commands `clone:list`, `clone:preset` to help to clone a remote Flow/Neos setup into the local Flow/Neos installation that executes the command. **Attention: These commands will empty the local database and resources of your local Flow installation. @@ -28,9 +28,6 @@ using the commands.** # clone from remote host with the information stored in the master preset ./flow clone:preset master - -# clone remote host with the information stored in the master preset -./flow clone:remotehost --host=host --user=user --port=port --path=path --context=context ``` ### Settings.yaml From 44340623484eb34ba4161e0b372dad5ad65504d3 Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Mon, 18 Feb 2019 11:15:33 +0100 Subject: [PATCH 12/16] BUGFIX: Add rawurlencode processing while building resource proxy urls --- .../ProxyAwareWritableFileSystemStorage.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php index 102c899..54652da 100644 --- a/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php +++ b/Classes/ResourceManagement/ProxyAwareWritableFileSystemStorage.php @@ -67,9 +67,9 @@ public function getStreamByResource(PersistentResource $resource) $subdivideHashPathSegment = $resourceProxyConfiguration['subdivideHashPathSegment'] ?? false; if ($subdivideHashPathSegment) { $sha1Hash = $resource->getSha1(); - $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . $resource->getSha1() . '/' . $resource->getFilename();; + $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $sha1Hash[0] . '/' . $sha1Hash[1] . '/' . $sha1Hash[2] . '/' . $sha1Hash[3] . '/' . $sha1Hash . '/' . $resource->getSha1() . '/' . rawurlencode($resource->getFilename()); } else { - $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $resource->getSha1() . '/' . $resource->getFilename(); + $uri = $resourceProxyConfiguration['baseUri'] .'/_Resources/Persistent/' . $resource->getSha1() . '/' . rawurlencode($resource->getFilename()); } $response = $browser->request($uri); From 2fe42ea705ae641b02ba1ca53ad8bb7949c6dfae Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Tue, 19 Feb 2019 09:18:11 +0100 Subject: [PATCH 13/16] TASK: Move ConfigurationService Injection to AbstractCommandController --- Classes/Command/AbstractCommandController.php | 6 ++++++ Classes/Command/CloneCommandController.php | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Classes/Command/AbstractCommandController.php b/Classes/Command/AbstractCommandController.php index 6ef5b3b..a7a7730 100644 --- a/Classes/Command/AbstractCommandController.php +++ b/Classes/Command/AbstractCommandController.php @@ -44,6 +44,12 @@ abstract class AbstractCommandController extends CommandController */ protected $flowCommand; + /** + * @Flow\Inject + * @var ConfigurationService + */ + protected $configurationService; + /** * @param string $commands * @param array $arguments diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index f50b713..82c25ff 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -43,12 +43,6 @@ class CloneCommandController extends AbstractCommandController */ protected $dbal; - /** - * @Flow\Inject - * @var ConfigurationService - */ - protected $configurationService; - /** * Show the list of predefined clone configurations */ From 3caccf2ee8fc8389b7e84167ddf781afb4706eb9 Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Tue, 19 Feb 2019 09:19:02 +0100 Subject: [PATCH 14/16] TASK: Fix code style and typos --- Classes/Command/StashCommandController.php | 5 +---- Classes/Domain/Service/ConfigurationService.php | 6 +++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Classes/Command/StashCommandController.php b/Classes/Command/StashCommandController.php index 4efa509..5ccad09 100644 --- a/Classes/Command/StashCommandController.php +++ b/Classes/Command/StashCommandController.php @@ -15,9 +15,6 @@ */ class StashCommandController extends AbstractCommandController { - - - /** * Creates a new stash entry with the given name. * @@ -277,7 +274,7 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = . '`; CREATE DATABASE `' . $this->databaseConfiguration['dbname'] . '` collate utf8_unicode_ci;'; - + $this->executeLocalShellCommand( 'echo %s | mysql --host=%s --user=%s --password=%s', [ diff --git a/Classes/Domain/Service/ConfigurationService.php b/Classes/Domain/Service/ConfigurationService.php index d48c71e..c39fb29 100644 --- a/Classes/Domain/Service/ConfigurationService.php +++ b/Classes/Domain/Service/ConfigurationService.php @@ -24,9 +24,9 @@ class ConfigurationService */ public function getCurrentConfiguration(): array { - $cloneInformations = $this->clonePresetInformationCache->get('current'); - if ($cloneInformations && is_array($this->clonePresets) && array_key_exists($cloneInformations['presetName'], $this->clonePresets)) { - return $this->clonePresets[$cloneInformations['presetName']]; + $cloneInformation = $this->clonePresetInformationCache->get('current'); + if ($cloneInformation && is_array($this->clonePresets) && array_key_exists($cloneInformation['presetName'], $this->clonePresets)) { + return $this->clonePresets[$cloneInformation['presetName']]; } else { return []; } From 7742ec283dd234f0c7d5fb89b6d2c0dcf94a4984 Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Tue, 19 Feb 2019 11:58:13 +0100 Subject: [PATCH 15/16] TASK: Fix namespaces between Abstract and Clone CommandControllers --- Classes/Command/AbstractCommandController.php | 2 +- Classes/Command/CloneCommandController.php | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Classes/Command/AbstractCommandController.php b/Classes/Command/AbstractCommandController.php index a7a7730..1069a22 100644 --- a/Classes/Command/AbstractCommandController.php +++ b/Classes/Command/AbstractCommandController.php @@ -9,10 +9,10 @@ use Neos\Flow\Annotations as Flow; use Neos\Flow\Core\Bootstrap; use Neos\Flow\Cli\CommandController; +use Sitegeist\MagicWand\Domain\Service\ConfigurationService; abstract class AbstractCommandController extends CommandController { - const HIDE_RESULT = 1; const HIDE_COMMAND = 2; diff --git a/Classes/Command/CloneCommandController.php b/Classes/Command/CloneCommandController.php index 82c25ff..2ee5052 100644 --- a/Classes/Command/CloneCommandController.php +++ b/Classes/Command/CloneCommandController.php @@ -10,7 +10,6 @@ use Neos\Utility\Arrays; use Neos\Flow\Core\Bootstrap; use Sitegeist\MagicWand\DBAL\SimpleDBAL; -use Sitegeist\MagicWand\Domain\Service\ConfigurationService; use Symfony\Component\Yaml\Yaml; /** From e7be8636c72cc9239370d286a4ad314c2219a63f Mon Sep 17 00:00:00 2001 From: Wilhelm Behncke Date: Tue, 19 Feb 2019 11:59:00 +0100 Subject: [PATCH 16/16] FEATURE: Write manifest files in stash entries and restore preset states --- Classes/Command/StashCommandController.php | 100 ++++++++++++++++-- .../Domain/Service/ConfigurationService.php | 84 +++++++++++++-- 2 files changed, 170 insertions(+), 14 deletions(-) diff --git a/Classes/Command/StashCommandController.php b/Classes/Command/StashCommandController.php index 5ccad09..81f5c63 100644 --- a/Classes/Command/StashCommandController.php +++ b/Classes/Command/StashCommandController.php @@ -29,10 +29,7 @@ public function createCommand($name) # Build Paths # ####################### - $basePath = sprintf( - FLOW_PATH_ROOT . 'Data/MagicWandStash/%s', - $name - ); + $basePath = $this->getStashEntryPath($name); $databaseDestination = $basePath . '/database.sql'; $persistentDestination = $basePath . '/persistent/'; @@ -52,6 +49,24 @@ public function createCommand($name) $this->addSecret($this->databaseConfiguration['user']); $this->addSecret($this->databaseConfiguration['password']); + ###################### + # Write Manifest # + ###################### + $this->renderHeadLine('Write Manifest'); + $presetName = $this->configurationService->getCurrentPreset(); + $presetConfiguration = $this->configurationService->getCurrentConfiguration(); + $cloneTimestamp = $this->configurationService->getMostRecentCloneTimeStamp(); + $stashTimestamp = time(); + + $this->writeStashEntryManifest($name, [ + 'preset' => [ + 'name' => $presetName, + 'configuration' => $presetConfiguration + ], + 'cloned_at' => $cloneTimestamp, + 'stashed_at' => $stashTimestamp + ]); + ###################### # Backup Database # ###################### @@ -99,6 +114,8 @@ public function createCommand($name) */ public function listCommand() { + $head = ['Name', 'Stashed At', 'From Preset', 'Cloned At']; + $rows = []; $basePath = sprintf(FLOW_PATH_ROOT . 'Data/MagicWandStash'); if (!is_dir($basePath)) { @@ -111,7 +128,16 @@ public function listCommand() foreach ($baseDir as $entry) { if (!in_array($entry, ['.', '..'])) { - $this->renderLine(' • %s', [$entry->getFilename()]); + $stashEntryName = $entry->getFilename(); + $manifest = $this->readStashEntryManifest($stashEntryName) ?: []; + + $rows[] = [ + $stashEntryName, + $manifest['stashed_at'] ? date('Y-m-d H:i:s', $manifest['stashed_at']) : 'N/A', + isset($manifest['preset']['name']) ? $manifest['preset']['name'] : 'N/A', + $manifest['cloned_at'] ? date('Y-m-d H:i:s', $manifest['cloned_at']) : 'N/A', + ]; + $anyEntry = true; } } @@ -120,6 +146,8 @@ public function listCommand() $this->renderLine('Stash is empty.'); $this->quit(1); } + + $this->output->outputTable($rows, $head); } /** @@ -155,7 +183,7 @@ public function clearCommand() */ public function restoreCommand($name, $yes = false, $keepDb = false) { - $basePath = sprintf(FLOW_PATH_ROOT . 'Data/MagicWandStash/%s', $name); + $basePath = $this->getStashEntryPath($name); $this->restoreStashEntry($basePath, $name, $yes, true, $keepDb); } @@ -344,6 +372,13 @@ protected function restoreStashEntry($source, $name, $force = false, $preserve = $this->renderHeadLine('Publish Resources'); $this->executeLocalFlowCommand('resource:publish'); + ############################# + # Restore Clone Information # + ############################# + if($manifest = $this->readStashEntryManifest($name)) { + $this->configurationService->setCurrentStashEntry($name, $manifest); + } + ################# # Final Message # ################# @@ -369,4 +404,57 @@ protected function checkConfiguration() $this->renderLine(' - Configuration seems ok ...'); } + + /** + * @param string $stashEntryName + * @return string + */ + protected function getStashEntryPath(string $stashEntryName): string + { + return sprintf( + FLOW_PATH_ROOT . 'Data/MagicWandStash/%s', + $stashEntryName + ); + } + + /** + * @param string $stashEntryName + * @return array|null + */ + protected function readStashEntryManifest(string $stashEntryName): ?array + { + $manifestDestination = $this->getStashEntryPath($stashEntryName) . '/manifest.json'; + + if (file_exists($manifestDestination)) { + if ($manifest = json_decode(file_get_contents($manifestDestination), true)) { + if (is_array($manifest)) { + return $manifest; + } + } + + $this->outputLine('Manifest file has been corrupted.'); + } + + return null; + } + + /** + * @param string $stashEntryName + * @param array $manifest + * @return void + */ + protected function writeStashEntryManifest(string $stashEntryName, array $manifest): void + { + $manifestDestination = $this->getStashEntryPath($stashEntryName) . '/manifest.json'; + + // Create directory, if not exists + if (!file_exists(dirname($manifestDestination))) { + FileUtils::createDirectoryRecursively(dirname($manifestDestination)); + } + + // Write manifest file + file_put_contents($manifestDestination, json_encode($manifest, JSON_PRETTY_PRINT)); + + $this->outputLine('Wrote "%s"', [$manifestDestination]); + } } diff --git a/Classes/Domain/Service/ConfigurationService.php b/Classes/Domain/Service/ConfigurationService.php index c39fb29..b461d46 100644 --- a/Classes/Domain/Service/ConfigurationService.php +++ b/Classes/Domain/Service/ConfigurationService.php @@ -19,17 +19,46 @@ class ConfigurationService */ protected $clonePresets; + /** + * @return string + */ + public function getCurrentPreset(): ?string + { + $clonePresetInformation = $this->clonePresetInformationCache->get('current'); + + if ($clonePresetInformation && is_array($clonePresetInformation) && isset($clonePresetInformation['presetName'])) { + return $clonePresetInformation['presetName']; + } + + return null; + } + + /** + * @return integer + */ + public function getMostRecentCloneTimeStamp(): ?int + { + $clonePresetInformation = $this->clonePresetInformationCache->get('current'); + + if ($clonePresetInformation && is_array($clonePresetInformation) && isset($clonePresetInformation['cloned_at'])) { + return intval($clonePresetInformation['cloned_at']); + } + + return null; + } + /** * @return array */ public function getCurrentConfiguration(): array { - $cloneInformation = $this->clonePresetInformationCache->get('current'); - if ($cloneInformation && is_array($this->clonePresets) && array_key_exists($cloneInformation['presetName'], $this->clonePresets)) { - return $this->clonePresets[$cloneInformation['presetName']]; - } else { - return []; + if ($presetName = $this->getCurrentPreset()) { + if (is_array($this->clonePresets) && array_key_exists($presetName, $this->clonePresets)) { + return $this->clonePresets[$presetName]; + } } + + return []; } /** @@ -44,17 +73,56 @@ public function getCurrentConfigurationByPath($path) /** * @return boolean */ - public function hasConfiguration(): bool + public function hasCurrentPreset(): bool { - return $this->clonePresetInformationCache->has('current'); + if ($this->clonePresetInformationCache->has('current')) { + return true; + } + + $clonePresetInformation = $this->clonePresetInformationCache->get('current'); + + if ($clonePresetInformation && is_array($clonePresetInformation) && isset($clonePresetInformation['presetName'])) { + return true; + } + + return false; } /** * @param $presetName string + * @return void * @throws \Neos\Cache\Exception */ public function setCurrentPreset(string $presetName): void { - $this->clonePresetInformationCache->set('current', ['presetName' => $presetName, 'timestamp' => time()]); + $this->clonePresetInformationCache->set('current', [ + 'presetName' => $presetName, + 'cloned_at' => time() + ]); + } + + /** + * @param string $stashEntryName + * @param array $stashEntryManifest + * @return void + * @throws \Neos\Cache\Exception + */ + public function setCurrentStashEntry(string $stashEntryName, array $stashEntryManifest): void + { + if (!isset($stashEntryManifest['preset']['name'])) { + return; + } + + if (!isset($stashEntryManifest['cloned_at'])) { + return; + } + + $presetName = $stashEntryManifest['preset']['name']; + $clonedAt = $stashEntryManifest['cloned_at']; + + $this->clonePresetInformationCache->set('current', [ + 'presetName' => $presetName, + 'cloned_at' => $clonedAt + ]); } }