diff --git a/Classes/Aspect/EventLogAspect.php b/Classes/Aspect/EventLogAspect.php
index 4521d8b..501700d 100644
--- a/Classes/Aspect/EventLogAspect.php
+++ b/Classes/Aspect/EventLogAspect.php
@@ -1,10 +1,6 @@
settings = $settings;
}
+ /**
+ * Reset the mapping between external identifier and local nodes
+ *
+ * @param string $preset
+ * @param string $parts
+ */
+ public function initCommand($preset, $parts = null)
+ {
+ $parts = Arrays::trimExplode(',', $parts);
+ $presetSettings = $this->loadPreset($preset);
+ array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts) {
+ $this->outputLine();
+ $this->outputPartTitle($partSetting, $partName);
+
+ if ($parts !== array() && !in_array($partName, $parts)) {
+ $this->outputLine('~ Skipped');
+ return;
+ }
+
+ if (!isset($partSetting['importerClassName'])) {
+ $this->outputLine('Missing importerClassName in the current preset part (%s/%s), check your settings', [$preset, $partName]);
+ return;
+ }
+
+ $identifier = $partSetting['importerClassName'] . '@' . $preset . '/' . $partName;
+ /** @var RecordMapping $recordMapper */
+ foreach ($this->recordMapperRepository->findByImporterClassName($identifier) as $recordMapper) {
+ $this->recordMapperRepository->remove($recordMapper);
+ }
+ });
+ $vault = new Vault($preset);
+ $vault->flush();
+ }
+
+ /**
+ * Show the different pars of the preset
+ *
+ * @param string $preset
+ * @param string $parts
+ */
+ public function showCommand($preset)
+ {
+ $presetSettings = $this->loadPreset($preset);
+ array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset) {
+ $this->outputLine();
+ $this->outputPartTitle($partSetting, $partName);
+ });
+ }
+
/**
* Run batch import
*
@@ -113,14 +167,14 @@ public function injectSettings(array $settings)
* @param string $preset Name of the preset which holds the configuration for the import
* @param string $parts Optional comma separated names of parts. If no parts are specified, all parts will be imported.
* @param integer $batchSize Number of records to import at a time. If not specified, the batch size defined in the preset will be used.
- * @param string $externalImportIdentifier External identifier which is used for checking if an import of the same data has already been executed earlier.
+ * @param string $identifier External identifier which is used for checking if an import of the same data has already been executed earlier.
* @param boolean $force If set, an import will even be executed if it ran earlier with the same external import identifier.
* @return void
*/
- public function batchCommand($preset, $parts = null, $batchSize = null, $externalImportIdentifier = null, $force = false)
+ public function batchCommand($preset, $parts = null, $batchSize = null, $identifier = null, $force = false)
{
try {
- $this->importService->start($externalImportIdentifier, $force);
+ $this->importService->start($identifier, $force);
} catch (ImportAlreadyExecutedException $e) {
$this->outputLine($e->getMessage());
$this->outputLine('Import skipped. You can force running this import again by specifying --force.');
@@ -130,21 +184,15 @@ public function batchCommand($preset, $parts = null, $batchSize = null, $externa
$this->startTime = microtime(true);
$parts = Arrays::trimExplode(',', $parts);
- $this->outputLine('Start import ...');
- $presetSettings = Arrays::getValueByPath($this->settings, array('presets', $preset));
- if (!is_array($presetSettings)) {
- $this->outputLine(sprintf('Preset "%s" not found ...', $preset));
- $this->quit(1);
- }
-
- $this->checkForPartsSettingsOrQuit($presetSettings, $preset);
-
+ $identifier = $this->importService->getCurrentImportIdentifier();
+ $this->outputLine('Start import with identifier %s', [$identifier]);
- array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts, $batchSize) {
+ $presetSettings = $this->loadPreset($preset);
+ array_walk($presetSettings['parts'], function ($partSetting, $partName) use ($preset, $parts, $batchSize, $identifier) {
$this->elapsedTime = 0;
$this->batchCounter = 0;
$this->outputLine();
- $this->outputFormatted(sprintf('%s', $partSetting['label']));
+ $this->outputPartTitle($partSetting, $partName);
$partSetting['__currentPresetName'] = $preset;
$partSetting['__currentPartName'] = $partName;
@@ -152,9 +200,9 @@ public function batchCommand($preset, $parts = null, $batchSize = null, $externa
$partSetting['batchSize'] = $batchSize;
}
- $partSetting = new PresetPartDefinition($partSetting, $this->importService->getCurrentImportIdentifier());
+ $partSetting = new PresetPartDefinition($partSetting, $identifier);
if ($parts !== array() && !in_array($partName, $parts)) {
- $this->outputLine('Skipped');
+ $this->outputLine('~ Skipped');
return;
}
@@ -173,14 +221,31 @@ public function batchCommand($preset, $parts = null, $batchSize = null, $externa
$import = $this->importService->getLastImport();
$this->outputLine();
- $this->outputLine('Import finished.');
- $this->outputLine(sprintf(' Started %s', $import->getStartTime()->format(DATE_RFC2822)));
- $this->outputLine(sprintf(' Finished %s', $import->getEndTime()->format(DATE_RFC2822)));
- $this->outputLine(sprintf(' Runtime %d seconds', $import->getElapsedTime()));
+ $this->outputLine('Import finished');
+ $this->outputLine(sprintf('- Started %s', $import->getStartTime()->format(DATE_RFC2822)));
+ $this->outputLine(sprintf('- Finished %s', $import->getEndTime()->format(DATE_RFC2822)));
+ $this->outputLine(sprintf('- Runtime %d seconds', $import->getElapsedTime()));
$this->outputLine();
$this->outputLine('See log for more details and possible errors.');
}
+ /**
+ * @param string $preset
+ * @return array
+ */
+ protected function loadPreset($preset)
+ {
+ $presetSettings = Arrays::getValueByPath($this->settings, ['presets', $preset]);
+ if (!is_array($presetSettings)) {
+ $this->outputLine(sprintf('Preset "%s" not found ...', $preset));
+ $this->quit(1);
+ }
+
+ $this->checkForPartsSettingsOrQuit($presetSettings, $preset);
+
+ return $presetSettings;
+ }
+
/**
* Execute a sub process which imports a batch as specified by the part definition.
*
@@ -196,25 +261,46 @@ protected function executeCommand(PresetPartDefinition $partSetting)
$startTime = microtime(true);
++$this->batchCounter;
- ob_start();
- $status = Scripts::executeCommand('ttree.contentrepositoryimporter:import:executebatch', $this->flowSettings, true, $partSetting->getCommandArguments());
+ ob_start(NULL, 1<<20);
+ $commandIdentifier = 'ttree.contentrepositoryimporter:import:executebatch';
+ $status = Scripts::executeCommand($commandIdentifier, $this->flowSettings, true, $partSetting->getCommandArguments());
if ($status !== true) {
- throw new Exception('Sub command failed', 1426767159);
+ throw new Exception(\vsprintf('Command: %s with parameters: %s', [$commandIdentifier, \json_encode($partSetting->getCommandArguments())]), 1426767159);
}
- $count = (integer)ob_get_clean();
- if ($count < 1) {
- return 0;
+ $output = explode(\PHP_EOL, ob_get_clean());
+ if (count($output) > 1) {
+ $this->outputLine('+ Command "%s"', [$commandIdentifier]);
+ $this->outputLine('+ with parameters:');
+ foreach ($partSetting->getCommandArguments() as $argumentName => $argumentValue) {
+ $this->outputLine('+ %s: %s', [$argumentName, $argumentValue]);
+ }
+ foreach ($output as $line) {
+ $line = trim($line);
+ if ($line === '') {
+ continue;
+ }
+ $this->outputLine('+ %s', [$line]);
+ }
}
+ $count = (int)\array_pop($output);
+ $count = $count < 1 ? 0 : $count;
$elapsedTime = (microtime(true) - $startTime) * 1000;
$this->elapsedTime += $elapsedTime;
- $this->outputLine(' #%d %d records in %dms, %d ms per record, %d ms per batch (avg)', [$partSetting->getCurrentBatch(), $count, $elapsedTime, $elapsedTime / $count, $this->elapsedTime / $this->batchCounter]);
+ $this->outputLine('+ #%d %d records in %dms, %d ms per record, %d ms per batch (avg)', [
+ $partSetting->getCurrentBatch(),
+ $count,
+ $elapsedTime,
+ ($count > 0 ? $elapsedTime / $count : $elapsedTime),
+ ($this->batchCounter > 0 ? $this->elapsedTime / $this->batchCounter : $this->elapsedTime)
+ ]);
$this->importService->addEvent(sprintf('%s:Ended', $partSetting->getEventType()), null, $partSetting->getCommandArguments());
$this->importService->persistEntities();
return $count;
} catch (\Exception $exception) {
$this->logger->logException($exception);
- $this->outputLine("Error, please check your logs ...", [$partSetting->getLabel()]);
+ $this->outputLine('Error in parts "%s", please check your logs for more details', [$partSetting->getLabel()]);
+ $this->outputLine('%s', [$exception->getMessage()]);
$this->importService->addEvent(sprintf('%s:Failed', $partSetting->getEventType()), null, $partSetting->getCommandArguments());
$this->quit(1);
}
@@ -239,7 +325,11 @@ protected function executeCommand(PresetPartDefinition $partSetting)
public function executeBatchCommand($presetName, $partName, $dataProviderClassName, $importerClassName, $currentImportIdentifier, $offset = null, $batchSize = null)
{
try {
+ $vault = new Vault($presetName);
+
$dataProviderOptions = Arrays::getValueByPath($this->settings, implode('.', ['presets', $presetName, 'parts', $partName, 'dataProviderOptions']));
+ $dataProviderOptions['__presetName'] = $presetName;
+ $dataProviderOptions['__partName'] = $partName;
/** @var DataProviderInterface $dataProvider */
$dataProvider = $dataProviderClassName::create(is_array($dataProviderOptions) ? $dataProviderOptions : [], $offset, $batchSize);
@@ -247,7 +337,10 @@ public function executeBatchCommand($presetName, $partName, $dataProviderClassNa
$importerOptions = Arrays::getValueByPath($this->settings, ['presets', $presetName, 'parts', $partName, 'importerOptions']);
/** @var AbstractImporter $importer */
- $importer = $this->objectManager->get($importerClassName, is_array($importerOptions) ? $importerOptions : [], $currentImportIdentifier);
+ $importerOptions = is_array($importerOptions) ? $importerOptions : [];
+ $importerOptions['__presetName'] = $presetName;
+ $importerOptions['__partName'] = $partName;
+ $importer = $this->objectManager->get($importerClassName, $importerOptions, $currentImportIdentifier, $vault);
$importer->getImportService()->addEventMessage(sprintf('%s:Batch:Started', $importerClassName), sprintf('%s batch started (%s)', $importerClassName, $dataProviderClassName));
$importer->initialize($dataProvider);
$importer->process();
@@ -255,7 +348,8 @@ public function executeBatchCommand($presetName, $partName, $dataProviderClassNa
$this->output($importer->getProcessedRecords());
} catch (\Exception $exception) {
$this->logger->logException($exception);
- $this->quit(1);
+ $this->outputLine('%s', [$exception->getMessage()]);
+ $this->sendAndExit(1);
}
}
@@ -271,6 +365,15 @@ public function flushEventLogCommand()
$this->eventLogRepository->removeAll();
}
+ /**
+ * @param array $partSetting
+ * @param string $partName
+ */
+ protected function outputPartTitle(array $partSetting, $partName)
+ {
+ $this->outputFormatted(sprintf('+ %s (%s)', $partSetting['label'], $partName));
+ }
+
/**
* Checks if the preset settings contain a "parts" segment and quits if it does not.
*
diff --git a/Classes/DataProvider/AbstractDataProvider.php b/Classes/DataProvider/AbstractDataProvider.php
index 21642f1..76ace81 100644
--- a/Classes/DataProvider/AbstractDataProvider.php
+++ b/Classes/DataProvider/AbstractDataProvider.php
@@ -1,13 +1,11 @@
options = $options;
+ $this->presetName = $options['__presetName'];
+ $this->partName = $options['__partName'];
+ $this->vault = $vault;
}
/**
@@ -75,7 +92,7 @@ public function __construct(array $options)
*/
public static function create(array $options = [], $offset = null, $limit = null)
{
- $dataProvider = new static($options);
+ $dataProvider = new static($options, new Vault($options['__presetName']));
$dataProvider->setOffset($offset);
$dataProvider->setLimit($limit);
diff --git a/Classes/DataProvider/AbstractDatabaseDataProvider.php b/Classes/DataProvider/AbstractDatabaseDataProvider.php
index 46df8be..0108b95 100644
--- a/Classes/DataProvider/AbstractDatabaseDataProvider.php
+++ b/Classes/DataProvider/AbstractDatabaseDataProvider.php
@@ -1,10 +1,6 @@
options['skipHeader']) && $this->options['skipHeader'] === true) {
+ $skipLines = 1;
+ } elseif (isset($this->options['skipHeader']) && \is_numeric($this->options['skipHeader'])) {
+ $skipLines = (int)$this->options['skipHeader'];
+ } else {
+ $skipLines = 0;
+ }
if (($handle = fopen($this->csvFilePath, 'r')) !== false) {
while (($data = fgetcsv($handle, 65534, $this->csvDelimiter, $this->csvEnclosure)) !== false) {
// skip header (maybe is better to set the first offset position instead)
- if (isset($this->options['skipHeader']) && $this->options['skipHeader'] === true && $currentLine === 0) {
+ if ($currentLine < $skipLines) {
$currentLine++;
continue;
}
@@ -94,7 +97,6 @@ public function fetch()
}
fclose($handle);
}
-
$this->logger->log(sprintf('%s: read %s lines and found %s records.', $this->csvFilePath, $currentLine, count($dataResult)), LOG_DEBUG);
return $dataResult;
}
diff --git a/Classes/DataProvider/DataProviderInterface.php b/Classes/DataProvider/DataProviderInterface.php
index 6640cc0..be081b8 100644
--- a/Classes/DataProvider/DataProviderInterface.php
+++ b/Classes/DataProvider/DataProviderInterface.php
@@ -1,10 +1,6 @@
batchSize) {
$arguments['batchSize'] = (integer)$this->batchSize;
+ } else {
+ $arguments['batchSize'] = 100000;
}
if ($this->offset) {
$arguments['offset'] = (integer)$this->offset;
diff --git a/Classes/Domain/Model/ProviderPropertyValidity.php b/Classes/Domain/Model/ProviderPropertyValidity.php
new file mode 100644
index 0000000..a057614
--- /dev/null
+++ b/Classes/Domain/Model/ProviderPropertyValidity.php
@@ -0,0 +1,39 @@
+nodeOrTemplate = $nodeOrTemplate;
+ }
+
+ /**
+ * @param string $propertyName
+ * @return bool
+ */
+ public function isValid($propertyName)
+ {
+ $availableProperties = $this->nodeOrTemplate->getNodeType()->getProperties();
+ return !(\in_array(substr($propertyName, 0, 1), ['_', '@']) || !isset($availableProperties[$propertyName]));
+ }
+}
diff --git a/Classes/Domain/Model/RecordMapping.php b/Classes/Domain/Model/RecordMapping.php
index 375883e..7f8c10e 100644
--- a/Classes/Domain/Model/RecordMapping.php
+++ b/Classes/Domain/Model/RecordMapping.php
@@ -1,10 +1,6 @@
entityManager->getConnection();
- $connection->query('SET FOREIGN_KEY_CHECKS=0');
- $connection->query('DELETE FROM typo3_neos_eventlog_domain_model_event WHERE dtype = "ttree_contentrepositoryimporter_event"');
- $connection->query('SET FOREIGN_KEY_CHECKS=1');
+
+ $isMySQL = $connection->getDriver()->getName() === 'pdo_mysql';
+ if ($isMySQL) {
+ $connection->query('SET FOREIGN_KEY_CHECKS=0');
+ }
+
+ $connection->query("DELETE FROM neos_neos_eventlog_domain_model_event WHERE dtype = 'ttree_contentrepositoryimporter_event'");
+
+ if ($isMySQL) {
+ $connection->query('SET FOREIGN_KEY_CHECKS=1');
+ }
}
}
diff --git a/Classes/Domain/Repository/ImportRepository.php b/Classes/Domain/Repository/ImportRepository.php
index f5a4c44..8a1c776 100644
--- a/Classes/Domain/Repository/ImportRepository.php
+++ b/Classes/Domain/Repository/ImportRepository.php
@@ -1,10 +1,6 @@
currentImport instanceof Import) {
throw new Exception('Unable to start a new import, please stop the current import first', 1426638560);
}
- if ($externalImportIdentifier !== null) {
- $existingImport = $this->importRepository->findOneByExternalImportIdentifier($externalImportIdentifier);
+ if ($identifier !== null) {
+ $existingImport = $this->importRepository->findOneByExternalImportIdentifier($identifier);
if (!$force && $existingImport instanceof Import) {
- throw new ImportAlreadyExecutedException(sprintf('An import referring to the external identifier "%s" has already been executed on %s.', $externalImportIdentifier, $existingImport->getStartTime()->format('d.m.Y h:m:s')), 1464028408403);
+ throw new ImportAlreadyExecutedException(sprintf('An import referring to the external identifier "%s" has already been executed on %s.', $identifier, $existingImport->getStartTime()->format('d.m.Y h:m:s')), 1464028408403);
}
}
$this->currentImport = new Import();
- $this->currentImport->setExternalImportIdentifier($externalImportIdentifier);
+ $this->currentImport->setExternalImportIdentifier($identifier);
if ($force && isset($existingImport)) {
- $this->addEventMessage(sprintf('ImportService:start', 'Forcing re-import of data set with external identifier "%s".', $externalImportIdentifier), LOG_NOTICE);
+ $this->addEventMessage(sprintf('ImportService:start', 'Forcing re-import of data set with external identifier "%s".', $identifier), LOG_NOTICE);
}
$this->importRepository->add($this->currentImport);
diff --git a/Classes/Exception/ImportAlreadyExecutedException.php b/Classes/Exception/ImportAlreadyExecutedException.php
index bd8182a..6e3e144 100644
--- a/Classes/Exception/ImportAlreadyExecutedException.php
+++ b/Classes/Exception/ImportAlreadyExecutedException.php
@@ -1,10 +1,5 @@
options = $options;
+ $this->presetName = $options['__presetName'];
+ $this->partName = $options['__partName'];
+ $this->vault = $vault;
+ unset($this->options['__presetName'], $this->options['__partName']);
$this->currentImportIdentifier = $currentImportIdentifier;
}
@@ -273,17 +312,26 @@ public function initialize(DataProviderInterface $dataProvider)
$context = $this->contextFactory->create($contextConfiguration);
$this->rootNode = $context->getRootNode();
- if (isset($this->options['siteNodePath'])) {
- $siteNodePath = $this->options['siteNodePath'];
- $this->siteNode = $this->rootNode->getNode($siteNodePath);
+ $this->applyOption($this->storageNodeNodePath, 'storageNodeNodePath');
+ $this->applyOption($this->nodeTypeName, 'nodeTypeName');
+
+ if (isset($this->options['siteNodePath']) || isset($this->options['siteNodeIdentifier'])) {
+ $siteNodePath = isset($this->options['siteNodePath']) ? trim($this->options['siteNodePath']) : null;
+ $siteNodeIdentifier = isset($this->options['siteNodeIdentifier']) ? trim($this->options['siteNodeIdentifier']) : null;
+ $this->siteNode = $this->rootNode->getNode($siteNodePath) ?: $context->getNodeByIdentifier($siteNodeIdentifier);
if ($this->siteNode === null) {
- throw new Exception(sprintf('Site node not found (%s)', $siteNodePath), 1425077201);
+ throw new Exception(sprintf('Site node not found (%s)', $siteNodePath ?: $siteNodeIdentifier), 1425077201);
}
} else {
$this->log(get_class($this) . ': siteNodePath is not defined. Please make sure to set the target siteNodePath in your importer options.', LOG_WARNING);
}
}
+ protected function applyOption(&$option, $optionName)
+ {
+ $option = isset($this->options[$optionName]) ? $this->options[$optionName] : $option;
+ }
+
/**
* Starts batch processing all commands
*
@@ -316,12 +364,28 @@ protected function processBatch(NodeTemplate $nodeTemplate = null)
$records = $this->preProcessing($records);
array_walk($records, function ($data) use ($nodeTemplate) {
+ if (!\is_array($data)) {
+ $data = $this->propertyMapper->convert($data, 'array');
+ }
$this->processRecord($nodeTemplate, $data);
++$this->processedRecords;
});
$this->postProcessing($records);
}
+ public function withStorageNode(NodeInterface $storageNode, \Closure $closure)
+ {
+ $previousStorageNode = $this->storageNode;
+ try {
+ $this->storageNode = $storageNode;
+ $closure();
+ $this->storageNode = $previousStorageNode;
+ } catch (\Exception $exception) {
+ $this->storageNode = $previousStorageNode;
+ throw $exception;
+ }
+ }
+
/**
* Processes a single record
*
@@ -349,25 +413,32 @@ public function processRecord(NodeTemplate $nodeTemplate, array $data)
if ($node === null) {
throw new \Exception(sprintf('Failed retrieving existing node for update. External identifier: %s Node identifier: %s. Maybe the record mapping in the database does not match the existing (imported) nodes anymore.', $externalIdentifier, $recordMapping->getNodeIdentifier()), 1462971366085);
}
- $somethingChanged = $this->applyProperties($data, $node);
- if ($somethingChanged) {
- $this->importService->addEventMessage('Node:Processed:Updated', sprintf('Updating existing node %s (%s)', $node->getPath(), $node->getIdentifier()), LOG_INFO, $this->currentEvent);
- } else {
- $this->importService->addEventMessage('Node:Processed:Skipped', sprintf('Skipping unchanged node %s (%s)', $node->getPath(), $node->getIdentifier()), LOG_INFO, $this->currentEvent);
- }
+ $this->applyProperties($this->getPropertiesFromDataProviderPayload($data), $node);
} else {
$nodeTemplate->setNodeType($this->nodeType);
$nodeTemplate->setName($nodeName);
- $this->applyProperties($data, $nodeTemplate);
+ $this->applyProperties($this->getPropertiesFromDataProviderPayload($data), $nodeTemplate);
- $node = $this->storageNode->createNodeFromTemplate($nodeTemplate);
+ $node = $this->createNodeFromTemplate($nodeTemplate, $data);
$this->registerNodeProcessing($node, $externalIdentifier);
}
+ $this->dimensionsImporter->process($node, $data, $this->currentEvent);
+
return $node;
}
+ /**
+ * @param NodeTemplate $templace
+ * @param array $data
+ * @return NodeInterface
+ */
+ protected function createNodeFromTemplate(NodeTemplate $templace, array $data)
+ {
+ return $this->storageNode->createNodeFromTemplate($templace);
+ }
+
/**
* @param NodeTemplate $nodeTemplate
* @throws \Neos\ContentRepository\Exception\NodeException
@@ -382,6 +453,15 @@ protected function unsetAllNodeTemplateProperties(NodeTemplate $nodeTemplate)
}
}
+ /**
+ * @param array $data
+ * @return array
+ */
+ protected function getPropertiesFromDataProviderPayload(array $data)
+ {
+ return $data;
+ }
+
/**
* Applies the given properties ($data) to the given Node or NodeTemplate
*
@@ -391,20 +471,7 @@ protected function unsetAllNodeTemplateProperties(NodeTemplate $nodeTemplate)
*/
protected function applyProperties(array $data, $nodeOrTemplate)
{
- if (!$nodeOrTemplate instanceof NodeInterface && !$nodeOrTemplate instanceof NodeTemplate) {
- throw new \InvalidArgumentException(sprintf('$nodeOrTemplate must be either an object implementing NodeInterface or a NodeTemplate, %s given.', (is_object($nodeOrTemplate) ? get_class($nodeOrTemplate) : gettype($nodeOrTemplate))), 1462958554616);
- }
- $nodeChanged = false;
- foreach ($data as $propertyName => $propertyValue) {
- if (substr($propertyName, 0, 1) === '_') {
- continue;
- }
- if ($nodeOrTemplate->getProperty($propertyName) != $propertyValue) {
- $nodeOrTemplate->setProperty($propertyName, $propertyValue);
- $nodeChanged = true;
- }
- }
- return $nodeChanged;
+ return $this->nodePropertyMapper->map($data, $nodeOrTemplate, $this->currentEvent);
}
/**
@@ -468,12 +535,7 @@ protected function skipNodeProcessing($externalIdentifier, $nodeName, NodeInterf
*/
protected function registerNodeProcessing(NodeInterface $node, $externalIdentifier, $externalRelativeUri = null)
{
- if (defined('static::IMPORTER_CLASSNAME') === false) {
- $importerClassName = get_called_class();
- } else {
- $importerClassName = static::IMPORTER_CLASSNAME;
- }
- $this->processedNodeService->set($importerClassName, $externalIdentifier, $externalRelativeUri, $node->getIdentifier(), $node->getPath());
+ $this->processedNodeService->set(get_called_class(), $externalIdentifier, $externalRelativeUri, $node->getIdentifier(), $node->getPath(), $this->presetPath());
}
/**
@@ -482,7 +544,15 @@ protected function registerNodeProcessing(NodeInterface $node, $externalIdentifi
*/
protected function getNodeProcessing($externalIdentifier)
{
- return $this->processedNodeService->get(get_called_class(), $externalIdentifier);
+ return $this->processedNodeService->get(get_called_class(), $externalIdentifier, $this->presetPath());
+ }
+
+ /**
+ * @return string
+ */
+ protected function presetPath()
+ {
+ return $this->presetName . '/' . $this->partName;
}
/**
@@ -551,27 +621,48 @@ protected function getLabelFromRecordData(array $data)
*
* The storage node is either created or just retrieved and finally stored in $this->storageNode.
*
- * @param string $nodePath Absolute or relative (to the site node) node path of the storage node
+ * @param string $nodePathOrIdentifier A nodeIdentifier (prefixed with #) or an absolute or relative (to the site node) node path of the storage node
* @param string $title Title for the storage node document
* @return void
- * @throws \Neos\ContentRepository\Exception\NodeTypeNotFoundException
+ * @throws Exception
+ */
+ protected function initializeStorageNode($nodePathOrIdentifier, $title)
+ {
+ if (is_string($nodePathOrIdentifier) && $nodePathOrIdentifier[0] === '#') {
+ $this->storageNode = $this->rootNode->getContext()->getNodeByIdentifier(\substr($nodePathOrIdentifier, 1));
+ } else {
+ $this->storageNode = $this->getSiteNode()->getNode($nodePathOrIdentifier);
+
+ preg_match('|([a-z0-9\-]+/)*([a-z0-9\-]+)$|', $nodePathOrIdentifier, $matches);
+ $nodeName = $matches[2];
+ $uriPathSegment = Slug::create($title)->getValue();
+
+ $storageNodeTemplate = new NodeTemplate();
+ $storageNodeTemplate->setNodeType($this->nodeTypeManager->getNodeType($this->storageNodeTypeName));
+
+ if ($this->storageNode === null) {
+ $storageNodeTemplate->setProperty('title', $title);
+ $storageNodeTemplate->setProperty('uriPathSegment', $uriPathSegment);
+ $storageNodeTemplate->setName($nodeName);
+ $this->storageNode = $this->getSiteNode()->createNodeFromTemplate($storageNodeTemplate);
+ }
+ }
+
+ if (!$this->storageNode instanceof NodeInterface) {
+ throw new Exception('Storage node can not be empty', 1500558744);
+ }
+ }
+
+ /**
+ * @return NodeInterface
+ * @throws SiteNodeEmptyException
*/
- protected function initializeStorageNode($nodePath, $title)
+ protected function getSiteNode()
{
- preg_match('|([a-z0-9\-]+/)*([a-z0-9\-]+)$|', $nodePath, $matches);
- $nodeName = $matches[2];
- $uriPathSegment = Slug::create($title)->getValue();
-
- $storageNodeTemplate = new NodeTemplate();
- $storageNodeTemplate->setNodeType($this->nodeTypeManager->getNodeType($this->storageNodeTypeName));
-
- $this->storageNode = $this->siteNode->getNode($nodePath);
- if ($this->storageNode === null) {
- $storageNodeTemplate->setProperty('title', $title);
- $storageNodeTemplate->setProperty('uriPathSegment', $uriPathSegment);
- $storageNodeTemplate->setName($nodeName);
- $this->storageNode = $this->siteNode->createNodeFromTemplate($storageNodeTemplate);
+ if (!$this->siteNode instanceof NodeInterface) {
+ throw new SiteNodeEmptyException(get_class($this) . ': siteNodePath is not defined. Please make sure to set the target siteNodePath in your importer options.');
}
+ return $this->siteNode;
}
diff --git a/Classes/Importer/ImporterInterface.php b/Classes/Importer/ImporterInterface.php
index 62b6ad3..347e027 100644
--- a/Classes/Importer/ImporterInterface.php
+++ b/Classes/Importer/ImporterInterface.php
@@ -1,10 +1,6 @@
node = $node;
+ }
+
+ /**
+ * @param array $dimensions
+ * @return NodeInterface|null
+ */
+ public function to(array $dimensions)
+ {
+ return (new FlowQuery([$this->node]))->context([
+ 'dimensions' => $dimensions,
+ 'targetDimensions' => array_map(function ($dimensionValues) {
+ return array_shift($dimensionValues);
+ }, $dimensions)
+ ])->get(0);
+ }
+}
diff --git a/Classes/Service/DimensionsImporter.php b/Classes/Service/DimensionsImporter.php
new file mode 100644
index 0000000..68d2dea
--- /dev/null
+++ b/Classes/Service/DimensionsImporter.php
@@ -0,0 +1,53 @@
+isValid($propertyName);
+ };
+ $dimensionsData = $data['@dimensions'];
+ $properties = \array_filter($data, $dataFilter, \ARRAY_FILTER_USE_KEY);
+
+ $contextSwitcher = new ContextSwitcher($node);
+ foreach (array_keys($dimensionsData) as $preset) {
+ $dimensions = $this->settings[\str_replace('@', '', $preset)];
+ $nodeInContext = $contextSwitcher->to($dimensions);
+ $localProperties = \array_filter($dimensionsData[$preset], $dataFilter, \ARRAY_FILTER_USE_KEY);
+ $localProperties = Arrays::arrayMergeRecursiveOverrule($properties, $localProperties);
+ $this->nodePropertyMapper->map($localProperties, $nodeInContext, $event);
+ }
+ }
+}
diff --git a/Classes/Service/NodePropertyMapper.php b/Classes/Service/NodePropertyMapper.php
new file mode 100644
index 0000000..b4bf1fb
--- /dev/null
+++ b/Classes/Service/NodePropertyMapper.php
@@ -0,0 +1,63 @@
+ $propertyValue) {
+ if (!$propertyValidity->isValid($propertyName)) {
+ continue;
+ }
+ if ($nodeOrTemplate->getProperty($propertyName) != $propertyValue) {
+ $nodeOrTemplate->setProperty($propertyName, $propertyValue);
+ $nodeChanged = true;
+ }
+ }
+
+ if (isset($data['__identifier']) && \is_string($data['__identifier']) && $nodeOrTemplate instanceof NodeTemplate) {
+ $nodeOrTemplate->setIdentifier(trim($data['__identifier']));
+ }
+
+ if ($nodeOrTemplate instanceof NodeInterface) {
+ $path = $nodeOrTemplate->getContextPath();
+ if ($nodeChanged) {
+ $this->importService->addEventMessage('Node:Processed:Updated', sprintf('Updating existing node "%s" %s (%s)', $nodeOrTemplate->getLabel(), $path, $nodeOrTemplate->getIdentifier()), \LOG_INFO, $currentEvent);
+ } else {
+ $this->importService->addEventMessage('Node:Processed:Skipped', sprintf('Skipping unchanged node "%s" %s (%s)', $nodeOrTemplate->getLabel(), $path, $nodeOrTemplate->getIdentifier()), \LOG_NOTICE, $currentEvent);
+ }
+ }
+
+ return $nodeChanged;
+ }
+}
diff --git a/Classes/Service/ProcessedNodeService.php b/Classes/Service/ProcessedNodeService.php
index 86d1425..648cb22 100644
--- a/Classes/Service/ProcessedNodeService.php
+++ b/Classes/Service/ProcessedNodeService.php
@@ -1,10 +1,6 @@
importService->addOrUpdateRecordMapping($importerClassName, $externalIdentifier, $externalRelativeUri, $nodeIdentifier, $nodePath);
+ $this->importService->addOrUpdateRecordMapping($this->buildImporterClassName($importerClassName, $presetPath), $externalIdentifier, $externalRelativeUri, $nodeIdentifier, $nodePath);
}
/**
@@ -46,8 +43,13 @@ public function set($importerClassName, $externalIdentifier, $externalRelativeUr
* @param string $externalIdentifier
* @return RecordMapping
*/
- public function get($importerClassName, $externalIdentifier)
+ public function get($importerClassName, $externalIdentifier, $presetPath)
+ {
+ return $this->recordMappingRepository->findOneByImporterClassNameAndExternalIdentifier($this->buildImporterClassName($importerClassName, $presetPath), $externalIdentifier);
+ }
+
+ protected function buildImporterClassName($importerClassName, $presetPath)
{
- return $this->recordMappingRepository->findOneByImporterClassNameAndExternalIdentifier($importerClassName, $externalIdentifier);
+ return $importerClassName . '@' . $presetPath;
}
}
diff --git a/Classes/Service/Vault.php b/Classes/Service/Vault.php
new file mode 100644
index 0000000..a1f231e
--- /dev/null
+++ b/Classes/Service/Vault.php
@@ -0,0 +1,59 @@
+preset = (string)$preset;
+ }
+
+ /**
+ * @param string $key
+ * @param mixed $value
+ */
+ public function set($key, $value)
+ {
+ $this->storage->set(md5($this->preset . $key), $value, [$this->preset]);
+ }
+
+ /**
+ * @param string $key
+ * @return mixed
+ */
+ public function get($key)
+ {
+ return $this->storage->get(md5($this->preset . $key));
+ }
+
+ /**
+ * @param string $key
+ * @return bool
+ */
+ public function has($key)
+ {
+ return $this->storage->has(md5($this->preset . $key));
+ }
+
+ /**
+ * @return void
+ */
+ public function flush()
+ {
+ $this->storage->flushByTag($this->preset);
+ }
+}
diff --git a/Configuration/Caches.yaml b/Configuration/Caches.yaml
new file mode 100644
index 0000000..7105e16
--- /dev/null
+++ b/Configuration/Caches.yaml
@@ -0,0 +1,3 @@
+Ttree_ContentRepositoryImporter_Vault:
+ frontend: Neos\Cache\Frontend\VariableFrontend
+ backend: Neos\Cache\Backend\FileBackend
diff --git a/Configuration/Objects.yaml b/Configuration/Objects.yaml
index 4d5c14b..79b8c6c 100644
--- a/Configuration/Objects.yaml
+++ b/Configuration/Objects.yaml
@@ -1,29 +1,9 @@
-Ttree\ContentRepositoryImporter\Service\ProcessedNodeService:
+Ttree\ContentRepositoryImporter\Service\Vault:
properties:
- cache:
+ storage:
object:
factoryObjectName: Neos\Flow\Cache\CacheManager
factoryMethodName: getCache
arguments:
1:
- value: Ttree_ContentRepositoryImporter_Processed_Node
-
-Ttree\ContentRepositoryImporter\Command\ImportCommandController:
- properties:
- cache:
- object:
- factoryObjectName: Neos\Flow\Cache\CacheManager
- factoryMethodName: getCache
- arguments:
- 1:
- value: Ttree_ContentRepositoryImporter_Processed_Node
-
-Ttree\ContentRepositoryImporter\DataType\ExternalResource:
- properties:
- downloadCache:
- object:
- factoryObjectName: Neos\Flow\Cache\CacheManager
- factoryMethodName: getCache
- arguments:
- 1:
- value: Ttree_ContentRepositoryImporter_External_Resource
\ No newline at end of file
+ value: Ttree_ContentRepositoryImporter_Vault
diff --git a/Configuration/Settings.yaml b/Configuration/Settings.yaml
index fc30928..8709020 100644
--- a/Configuration/Settings.yaml
+++ b/Configuration/Settings.yaml
@@ -10,6 +10,9 @@ Ttree:
user: ''
password: ''
+ dimensionsImporter:
+ presets: []
+
dataTypeOptions:
'Ttree\ContentRepositoryImporter\DataType\ExternalResource':
downloadDirectory: '%FLOW_PATH_DATA%Persistent/Ttree.ContentRepositoryImporter/Downloads/'
@@ -31,7 +34,7 @@ Ttree:
# 'base':
# parts:
# 'news':
-# label: 'News Import'
+# label: 'News Import'
# dataProviderClassName: 'Your\Package\Importer\DataProvider\NewsDataProvider'
# importerClassName: 'Your\Package\Importer\Importer\NewsImporter'
#
diff --git a/LICENSE.txt b/LICENSE.txt
new file mode 100644
index 0000000..bdeff91
--- /dev/null
+++ b/LICENSE.txt
@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Neos project contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/README.md b/README.md
index 166b9bf..e8cf08b 100644
--- a/README.md
+++ b/README.md
@@ -1,7 +1,7 @@
ContentRepositoryImporter
=========================
-This package contains generic utility to help importing data in the TYPO3 Content Repository (TYPO3CR) used by Neos.
+This package contains generic utility to help importing data in the Neos Content Repository.
What's included ?
-----------------
@@ -11,6 +11,7 @@ What's included ?
* DataProvider: used to prepare and cleanup data from the external source
* Importer: get the data from the DataProvider and push everything in the CR
* DataType: Simple object used to cleanup value and share code between DataProvider
+* Split your import in multiple sub commands to avoid high memory usage
* No big magic, you can always take control by overriding the default configuration and methods
A basic DataProvider
@@ -23,6 +24,86 @@ to discover.
It's important to update the ```count``` property when you process data from the external source. During the processing,
you can decide to skip some data (invalid data, missing values, ...) so we can not use the SQL count feature.
+Try to do most of the data cleaning up in the data provider, so the data would arrive to the importer ready for insertion.
+Basically the array build by the provider should contains the data with the property name that match your node type property name.
+If you need to transport value that will not match the node properties, please prefix them with '_'.
+
+There is some magic value, those values MUST be on the first level of the array:
+
+- **__identifier** (optional) This UUID will be used in the imported node, you should use ```AbstractImporter::applyProperties``` to have this feature, used by default
+- **__externalIdentifier** (required) The external identifier of the data, this one is really important. The package keep track of imported data
+- **__label** (required) The label of this record used by the importer mainly for logging (this value is not imported, but useful to follow the process)
+if you run twice the same import, the imported node will be updated and not created.
+
+**Tips**: If the properties of your nodes are not at the first level of the array, you can override the method ```AbstractImporter::getPropertiesFromDataProviderPayload```
+
+### Output of the provider
+
+Your provider should output something like this:
+
+```
+ [
+ '__label' => 'The external content lable, for internal use'
+ '__externalIdentifier' => 'The external external identifier, for internal use'
+ 'title' => 'My title'
+ 'year' => 1999
+ 'text' => '...'
+ ]
+```
+
+**Tips**: If your provider does not return an array, you MUST registrer a TypeConverter to convert it to an array. The property mapper is
+used automatically by the Importer.
+
+### Content Dimensions support
+
+If your data provider follow this convention, the importer can automatically create variants of your nodes:
+
+```
+ [
+ '__label' => 'The external content lable, for internal use'
+ '__externalIdentifier' => 'The external external identifier, for internal use'
+ 'title' => 'My title'
+ 'year' => 1999
+ 'text' => '...',
+
+ '@dimensions' => [
+ '@en' => [
+ '@strategy' => 'merge',
+ 'title' => '...',
+ ],
+ '@fr' => [
+ '@strategy' => 'merge',
+ 'title' => '...',
+ ],
+ ]
+ ]
+```
+
+The ```@en``` is a preset name, you must configuration the presets on your ```Settings.yaml```:
+
+```
+Hsso:
+ Importer:
+ dimensionPresets:
+ fr:
+ language: ['fr', 'en', 'de']
+ en:
+ language: ['en', 'de']
+ de:
+ language: ['de']
+```
+
+### Share data between preset parts
+
+You can split your import in multiple parts. Each parts is executed in a separate request. Sometimes it's useful to share data between parts (ex. in the first
+part you import the taxonomy, and in the second parts you map documents with the taxonomy). Those solve this use case, we integrate a feature called **Vault**. The
+Vault is simply a cache accessible in the importer and data provider by calling ```$this->vault->set($key, $name)``` and ```$this->vault->get($key)```. The
+current preset is the namespace, so you can use simple keys like name, ids, ...
+
+The cache is flushed if you call ```flow import:init --preset your-preset```.
+
+### Basic provider
+
```php
class BasicDataProvider extends DataProvider {
@@ -55,6 +136,12 @@ class BasicDataProvider extends DataProvider {
A basic Importer
----------------
+Every data importer must extend the ``AbstractImporter`` abstract class or implement the interface ```ImporterInterface```.
+
+In the `processRecord` method you handle the processing of every record, such as creating Content Repository node for each incoming data record.
+
+Do not forget to register the processed nodes with `registerNodeProcessing`. The method will handle feature like logging and tracking of imported node to decide if the local node need to be created or updated.
+
```php
class ProductImporter extends AbstractImporter
{
@@ -151,6 +238,8 @@ Ttree:
Start your import process
-------------------------
+**Tips**: Do not forget to require this package from the package in which you do the importing, to ensure the correct loading order, so the settings would get overriden correctly.
+
From the CLI:
```
diff --git a/composer.json b/composer.json
index 8232a62..adea2d0 100644
--- a/composer.json
+++ b/composer.json
@@ -1,11 +1,12 @@
{
+ "description": "Helper package to import data in the Neos content repository",
"name": "ttree/contentrepositoryimporter",
"type": "neos-package",
- "description": "Helper package to import data in the Neos content repository",
+ "license": "MIT",
"require": {
"neos/neos": ">3.0",
"ezyang/htmlpurifier": "*",
- "cocur/slugify": "*"
+ "cocur/slugify": "^2.5"
},
"autoload": {
"psr-4": {