Skip to content

Commit

Permalink
update retention rules
Browse files Browse the repository at this point in the history
  • Loading branch information
mpern committed Jun 9, 2021
1 parent 643e468 commit 422e2b7
Show file tree
Hide file tree
Showing 14 changed files with 195 additions and 107 deletions.
41 changes: 40 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

TBD

## [3.3.0] - 2021-06-09

### Added

- Retention periods are now configurable via properties
(check `project.properties` for details)

### Changed

- Retention rules and cron jobs that depend on each other are now grouped and
executed in the correct order via composite cronjobs

### Upgrade Guide

Remove all triggers of cronjobs that are now part of a composite job

```impex
REMOVE Trigger; cronJob(code)[unique = true]
; emailMessageCleanupCronJob
; emailAddressCleanupCronJob
; emailAttachmentCleanupCronJob
; cartCleanupCronJob
; anonymousCartCleanupCronJob
; distributedImpexCronJobCleanupCronJob
; distributedImpexJobCleanupCronJob
; distributedImportProcessCleanupCronJob
; importBatchCleanupCronJob
; importBatchContentCleanupCronJob
; businessProcessCleanupCronJob
; failedBusinessProcessCleanupCronJob
; progressBusinessProcessCleanupCronJob
; orphanedTaskConditionCleanupCronJob
; orphanedProcessTaskCleanupCronJob
; orphanedBusinessProcessParameterCleanupCronJob
; orphanedProcessTaskLogCleanupCronJob
```


## [3.2.0] - 2021-05-06

### Added
Expand Down Expand Up @@ -109,7 +147,8 @@ Initial release
- Carts


[Unreleased]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.2.0...HEAD
[Unreleased]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.3.0...HEAD
[3.3.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.2.0...v3.3.0
[3.2.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.1.0...v.3.2.0
[3.1.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v3.0.0...v3.1.0
[3.0.0]: https://github.com/sap-commerce-tools/sanecleanup/compare/v2.0.0...v3.0.0
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ Sensible defaults for data retention and cleanup for SAP Commerce, based on my C
````xml
<extension name="sanecleanup" />
````
1. :red_circle: Adapt the retention rules to your project requirements :red_circle:
1. :red_circle: Adapt the retention rules to your project requirements :red_circle: \
(check the available properties in `project.properties`)
1. Build and deploy.\
(The rules will be automatically imported during system update)

Expand Down
19 changes: 19 additions & 0 deletions project.properties
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,22 @@ sanecleanup.application-context=sanecleanup-spring.xml
version.gc.enabled=false

sanecleanup.jobs.sessionlanguage=en

# 7200 = 2 hours
# 86400 = 1 day
# 1209600 = 2 weeks
# 2419200 = 1 month / 4 weeks
# 15778476 = 6 months
sanecleanup.retentiontimeseconds.emailmessage=1209600
sanecleanup.retentiontimeseconds.cart.regular=2419200
sanecleanup.retentiontimeseconds.cart.anonymous=1209600
sanecleanup.retentiontimeseconds.storedhttpsession=7200
sanecleanup.retentiontimeseconds.impexmedia.generated=1209600
sanecleanup.retentiontimeseconds.impeximportcronjob.distributed=1209600
sanecleanup.retentiontimeseconds.cronjob.generated=1209600
sanecleanup.retentiontimeseconds.solrindexerjob.generated=86400
sanecleanup.retentiontimeseconds.businessprocess.succeeded=2419200
sanecleanup.retentiontimeseconds.businessprocess.failed=15778476
sanecleanup.retentiontimeseconds.businessprocess.running=15778476
sanecleanup.retentiontimeseconds.taskcondition.premature=15778476
sanecleanup.retentiontimeseconds.solrindexoperation=86400
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Import config properties into impex macros
UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage

$twoWeeks = 1209600
$retentionTime = $config-sanecleanup.retentiontimeseconds.emailmessage

# @readme EmailMessage
# - Are there more than a handful sent/unsent messages?
Expand All @@ -27,19 +26,16 @@ INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; ret
SELECT {m:pk}, {m:itemtype}
FROM {EmailMessage AS m LEFT JOIN BusinessProcess AS bp ON {m:process} = {bp:pk}}
WHERE {bp:pk} IS NULL
AND {m:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $twoWeeks ; basicRemoveCleanupAction ;
AND {m:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ;
INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize
; emailMessageCleanupJob ; emailMessageRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; emailMessageCleanupCronJob ; emailMessageCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 02:00
; emailMessageCleanupCronJob ; 0 0 2 * * ?


INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ;
; emailAddressRule ; "
SELECT DISTINCT({a:pk}), {a:itemType} from {
SELECT DISTINCT {a:pk}, {a:itemType} from {
EmailAddress AS a
LEFT JOIN EmailMessage2ToAddressesRel AS to ON {to:target} = {a:pk}
LEFT JOIN EmailMessage2CcAddressesRel AS cc ON {cc:target} = {a:pk}
Expand All @@ -54,9 +50,7 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; emailAddressCleanupJob ; emailAddressRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; emailAddressCleanupCronJob ; emailAddressCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 02:00
; emailAddressCleanupCronJob ; 0 0 2 * * ?



# EmailAttachment
Expand All @@ -71,9 +65,24 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; emailAttachmentCleanupJob ; emailAttachmentRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; emailAttachmentCleanupCronJob ; emailAttachmentCleanupJob ;


###

INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage]
;emailRetentionCompositeCronJob;compositeJobPerformable;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 02:00
; emailAttachmentCleanupCronJob ; 0 0 2 * * ?

; emailRetentionCompositeCronJob ; 0 0 2 * * ?

INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='emailRetentionCompositeCronJob']
;emailMessageCleanupCronJobEntry;emailMessageCleanupCronJob;
;emailAddressCleanupJobEntry;emailAddressCleanupCronJob;
;emailAttachmentCleanupCronJobEntry;emailAttachmentCleanupCronJob;

# ensure correct order of execution
UPDATE CompositeCronJob;code[unique=true];compositeEntries(code)
;emailRetentionCompositeCronJob;"emailMessageCleanupCronJobEntry,
emailAddressCleanupJobEntry,
emailAttachmentCleanupCronJobEntry
"
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@
# Import config properties into impex macros
UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage

$twoWeeks = 1209600
$fourWeeks = 2419200
$retentionTimeRegular=$config-sanecleanup.retentiontimeseconds.cart.regular
$retentionTimeAnonymous =$config-sanecleanup.retentiontimeseconds.cart.anonymous

# @readme Cart
# - Are there excessive amount of carts per site or per user?
Expand Down Expand Up @@ -56,21 +55,29 @@ INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; ret
FROM { Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk} }
WHERE {c:saveTime} IS NULL
AND {u:uid} <> 'anonymous'
AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $fourWeeks ; basicRemoveCleanupAction ;
AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeRegular ; basicRemoveCleanupAction ;
; anonymousCartRule ; "
SELECT {c:pk}, {c:itemtype}
FROM {Cart AS c LEFT JOIN User AS u ON {c:user} = {u:pk}}
WHERE {c:saveTime} IS NULL
AND ( {u:uid} = 'anonymous' OR {u:uid} IS NULL )
AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $twoWeeks ; basicRemoveCleanupAction ;
AND {c:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeAnonymous ; basicRemoveCleanupAction ;

INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize
; cartCleanupJob ; cartRule ; 1000
; anonymousCartCleanupJob ; anonymousCartRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; cartCleanupCronJob ; cartCleanupJob ;
; anonymousCartCleanupCronJob ; anonymousCartCleanupJob ;

###

INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage]
;cartCleanupCompositeCronJob;compositeJobPerformable;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 03:00
; cartCleanupCronJob ; 0 0 3 * * ?
; anonymousCartCleanupCronJob ; 0 0 3 * * ?
; cartCleanupCompositeCronJob ; 0 0 3 * * ?

INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='cartCleanupCompositeCronJob']
;cartCleanupCronJobEntry;cartCleanupCronJob;
;anonymousCartCleanupCronJobEntry;anonymousCartCleanupCronJob;
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,7 @@
# Import config properties into impex macros
UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true]
$sessionLanguage = $config-sanecleanup.jobs.sessionlanguage

$oneDay = 86400
$retentionTime = $config-sanecleanup.retentiontimeseconds.storedhttpsession

# @readme StoredHttpSession
# Excessive amount of session? This is hard to generalize as it highly depends on your site's traffic, but if you are near or over 5 digits, it's probably too much.
Expand All @@ -18,7 +17,7 @@ $oneDay = 86400
INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true] ; searchQuery; retentionTimeSeconds; actionReference ;
; storedHttpSessionRule ; "select {s:pk}, {s:itemtype}
from {StoredHttpSession as s}
where {s:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $oneDay ; basicRemoveCleanupAction ;
where {s:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ;
INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code) ; batchSize
; storedHttpSessionCleanupJob ; storedHttpSessionRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
Expand Down
5 changes: 2 additions & 3 deletions resources/impex/sanecleanup/impex/001-cleanup-impex.impex
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Import config properties into impex macros
UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage

$twoWeeks = 1209600
$retentionTime = $config-sanecleanup.retentiontimeseconds.impexmedia.generated

# @readme ImpexMedia
# Are there more than a handful (>100) of generated impex medias?
Expand All @@ -20,7 +19,7 @@ INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; ret
SELECT {i:pk}, {i:itemtype}
FROM {ImpexMedia AS i}
WHERE ( {i:code} LIKE '0_______' OR {i:code} LIKE 'generated impex media - %' )
AND {i:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $twoWeeks ; basicRemoveCleanupAction ;
AND {i:modifiedTime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ;
INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize
; impexMediaCleanupJob ; impexMediaRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
Expand Down
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
# Import config properties into impex macros
UPDATE GenericItem[processor=de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor];pk[unique=true]
$sessionLanguage=$config-sanecleanup.jobs.sessionlanguage

$twoWeeks = 1209600
$retentionTime = $config-sanecleanup.retentiontimeseconds.impeximportcronjob.distributed

# @readme ImpExImportCronJob (distributed impex)
# - More than ~10 `FINISHED` distributed impex jobs?
Expand All @@ -26,14 +25,11 @@ INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; ret
SELECT {i:pk}, {i:itemType}
FROM {ImpExImportCronJob AS i}
WHERE {i:code} LIKE 'distributed-impex-%'
AND {i:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $twoWeeks ; basicRemoveCleanupAction ;
AND {i:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $retentionTime ; basicRemoveCleanupAction ;
INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSize
; distributedImpexCronJobCleanupJob ; distributedImpexCronJobRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; distributedImpexCronJobCleanupCronJob ; distributedImpexCronJobCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 04:30
; distributedImpexCronJobCleanupCronJob ; 0 30 4 * * ?

INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ;
; distributedImpexJobRule ; "
Expand All @@ -45,9 +41,6 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; distributedImpexJobCleanupJob ; distributedImpexJobRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; distributedImpexJobCleanupCronJob ; distributedImpexJobCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 05:00
; distributedImpexJobCleanupCronJob ; 0 0 5 * * ?

INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ;
; distributedImportProcessRule ; "
Expand All @@ -58,9 +51,6 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; distributedImportProcessCleanupJob ; distributedImportProcessRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; distributedImportProcessCleanupCronJob ; distributedImportProcessCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 05:00
; distributedImportProcessCleanupCronJob ; 0 0 5 * * ?

INSERT_UPDATE FlexibleSearchRetentionRule; code[unique = true]; searchQuery; retentionTimeSeconds; actionReference ;
; importBatchRule ; "
Expand All @@ -71,9 +61,6 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; importBatchCleanupJob ; importBatchRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; importBatchCleanupCronJob ; importBatchCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 05:30
; importBatchCleanupCronJob ; 0 30 5 * * ?

# @readme ImportBatchContent
# Are there any left-over distributed import batches?
Expand All @@ -97,6 +84,27 @@ INSERT_UPDATE RetentionJob; code[unique = true] ; retentionRule(code); batchSiz
; importBatchContentCleanupJob ; importBatchContentRule ; 1000
INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; sessionLanguage(isoCode)[default = $sessionLanguage]
; importBatchContentCleanupCronJob ; importBatchContentCleanupJob ;

###

INSERT_UPDATE CompositeCronJob;code[unique=true];job(code);sessionLanguage(isoCode)[default = $sessionLanguage]
;distributedImpexRetentionCompositeCronJob;compositeJobPerformable;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at 06:00
; importBatchContentCleanupCronJob ; 0 0 6 * * ?
# every day at 04:30
; distributedImpexRetentionCompositeCronJob ; 0 30 4 * * ?

INSERT_UPDATE CompositeEntry;code[unique=true];executableCronJob(code);compositeCronJob(code)[default='distributedImpexRetentionCompositeCronJob']
;distributedImpexCronJobCleanupCronJobEntry;distributedImpexCronJobCleanupCronJob;
;distributedImpexJobCleanupCronJobEntry;distributedImpexJobCleanupCronJob;
;distributedImportProcessCleanupCronJobEntry;distributedImportProcessCleanupCronJob;
;importBatchCleanupCronJobEntry;importBatchCleanupCronJob;
;importBatchContentCleanupCronJobEntry;importBatchContentCleanupCronJob;

UPDATE CompositeCronJob;code[unique=true];compositeEntries(code)
;distributedImpexRetentionCompositeCronJob;"
distributedImpexCronJobCleanupCronJobEntry,
distributedImpexJobCleanupCronJobEntry,
distributedImportProcessCleanupCronJobEntry,
importBatchCleanupCronJobEntry,
importBatchContentCleanupCronJobEntry
"
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,4 @@ INSERT_UPDATE CronJob; code[unique = true] ; job(code) ; session
; orphanedLogsCleanupCronJob ; orphanedLogsCleanupJob ;
INSERT Trigger; cronJob(code)[unique = true]; cronExpression
# every day at midnight
; impexMediaCleanupCronJob ; 0 0 0 * * ?
; orphanedLogsCleanupCronJob ; 0 0 0 * * ?
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,8 @@
# Import config properties into impex macros
UPDATE GenericItem[processor = de.hybris.platform.commerceservices.impex.impl.ConfigPropertyImportProcessor]; pk[unique = true]
$sessionLanguage = $config-sanecleanup.jobs.sessionlanguage

$twoWeeks = 1209600
$oneDay = 86400
$retentionTimeCronjobs=$config-sanecleanup.retentiontimeseconds.cronjob.generated
$retentionTimeSolrIndexerJob =$config-sanecleanup.retentiontimeseconds.solrindexerjob.generated

# @readme CronJob (auto-generated)
# Are there too many (>10) outdated, auto-geneated jobs in your system?
Expand Down Expand Up @@ -42,13 +41,13 @@ SELECT {c:pk}, {c:itemType}
WHERE {trg:pk} IS NULL
AND {c:code} LIKE '00%'
AND {t:code} IN ( 'ImpExImportCronJob', 'CatalogVersionSyncCronJob', 'SolrIndexerCronJob' )
AND {c:endTime} < ?CALC_RETIREMENT_TIME" ; $twoWeeks ; basicRemoveCleanupAction ;
AND {c:endTime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeCronjobs ; basicRemoveCleanupAction ;
; solrJobRule ; "
SELECT {j:pk},{j:itemType}
FROM {ServicelayerJob AS j LEFT JOIN Trigger AS trg on {trg:job} = {j:pk} }
WHERE {trg:pk} IS NULL
AND ({j:code} LIKE 'solrIndexerJob_full_%' OR {j:code} LIKE 'solrIndexerJob_update_%')
AND {j:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $oneDay ; basicRemoveCleanupAction ;
AND {j:modifiedtime} < ?CALC_RETIREMENT_TIME" ; $retentionTimeSolrIndexerJob ; basicRemoveCleanupAction ;
INSERT_UPDATE RetentionJob; code[unique = true]; retentionRule(code); batchSize
; cronJobCleanupJob ; cronJobRule ; 1000
; solrJobCleanupJob ; solrJobRule ; 1000
Expand Down
Loading

0 comments on commit 422e2b7

Please sign in to comment.