Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/1292 enhance readability of dataextension retention policy #1295

Merged
merged 8 commits into from
May 3, 2024
Merged

Feature/1292 enhance readability of dataextension retention policy #1295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ad874d6
#1292: add c__retentionPolicy and c__retainUntil
JoernBerkefeld May 3, 2024
ce3d85a
#1292: add c__dataRetentionPeriodUnitOfMeasure
JoernBerkefeld May 3, 2024
53209d8
#1292: updated dataExtension tests for new retention policy handling
JoernBerkefeld May 3, 2024
201e897
#1292: fix eventDefinition test cases on Windows machines
JoernBerkefeld May 3, 2024
848015c
#1292: fix case in path of eventDefinition test response
JoernBerkefeld May 3, 2024
8c0a2f5
Delete test/resources/9999999/interaction/v1/EventDefinitions/get-res…
JoernBerkefeld May 3, 2024
cd46d35
#1292: fix weird git behavior by removing & re-adding this
JoernBerkefeld May 3, 2024
648d31b
Revert "#1292: fix weird git behavior by removing & re-adding this"
JoernBerkefeld May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions lib/metadataTypes/DataExtension.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -950,6 +950,52 @@ class DataExtension extends MetadataType {
delete metadata.CreatedDate;
delete metadata.ModifiedDate;

// Retention policy
if (
metadata.RowBasedRetention === false &&
metadata.DeleteAtEndOfRetentionPeriod === false &&
metadata.RetainUntil === ''
) {
// Note: RetainUntil expected to NOT have a value
metadata.c__retentionPolicy = 'none';
delete metadata.RetainUntil;
delete metadata.ResetRetentionPeriodOnImport;
} else if (
metadata.RowBasedRetention === false &&
metadata.DeleteAtEndOfRetentionPeriod === false
) {
// Note: RetainUntil expected to have a value
metadata.c__retentionPolicy = 'allRecordsAndDataextension';
} else if (
metadata.RowBasedRetention === false &&
metadata.DeleteAtEndOfRetentionPeriod === true
) {
// Note: RetainUntil expected to have a value
metadata.c__retentionPolicy = 'allRecords';
} else if (
metadata.RowBasedRetention === true &&
metadata.DeleteAtEndOfRetentionPeriod === false
) {
// Note: RetainUntil expected to NOT have a value
metadata.c__retentionPolicy = 'individialRecords';
delete metadata.RetainUntil;
}
delete metadata.RowBasedRetention;
delete metadata.DeleteAtEndOfRetentionPeriod;

if (metadata.RetainUntil) {
const retainUntil = new Date(metadata.RetainUntil);
metadata.c__retainUntil = `${retainUntil.getFullYear()}-${retainUntil.getMonth() + 1}-${retainUntil.getDate()}`;
}
delete metadata.RetainUntil;
if (metadata.DataRetentionPeriodUnitOfMeasure) {
metadata.c__dataRetentionPeriodUnitOfMeasure = Util.inverseGet(
this.definition.dataRetentionPeriodUnitOfMeasureMapping,
metadata.DataRetentionPeriodUnitOfMeasure
);
delete metadata.DataRetentionPeriodUnitOfMeasure;
}

return metadata;
}

Expand Down Expand Up @@ -1095,6 +1141,55 @@ class DataExtension extends MetadataType {
metadata.SendableSubscriberField.Name = 'Subscriber Key';
}

// Retention policy
switch (metadata.c__retentionPolicy) {
case 'none': {
metadata.RowBasedRetention = false;
metadata.DeleteAtEndOfRetentionPeriod = false;
metadata.ResetRetentionPeriodOnImport = false;
break;
}
case 'allRecordsAndDataextension': {
metadata.RowBasedRetention = false;
metadata.DeleteAtEndOfRetentionPeriod = false;
break;
}
case 'allRecords': {
metadata.RowBasedRetention = false;
metadata.DeleteAtEndOfRetentionPeriod = true;
break;
}
case 'individialRecords': {
metadata.RowBasedRetention = true;
metadata.DeleteAtEndOfRetentionPeriod = false;
break;
}
default: {
Util.logger.debug(
` - dataExtension '${
metadata[this.definition.nameField]
}': Unknown retention policy: ${metadata.c__retentionPolicy}`
);
}
}
delete metadata.c__retentionPolicy;

if (metadata.c__retainUntil) {
const retainUntil = new Date(metadata.c__retainUntil);
metadata.RetainUntil = `${retainUntil.getMonth() + 1}/${retainUntil.getDate()}/${retainUntil.getFullYear()} 12:00:00 AM`;
delete metadata.c__retainUntil;
} else {
metadata.RetainUntil = '';
}

if (metadata.c__dataRetentionPeriodUnitOfMeasure) {
metadata.DataRetentionPeriodUnitOfMeasure =
this.definition.dataRetentionPeriodUnitOfMeasureMapping[
metadata.c__dataRetentionPeriodUnitOfMeasure
];
delete metadata.c__dataRetentionPeriodUnitOfMeasure;
}

return metadata;
}

Expand Down
14 changes: 7 additions & 7 deletions lib/metadataTypes/Event.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ import cache from '../util/cache.js';
class Event extends MetadataType {
/**
* Retrieves Metadata of Event Definition.
* Endpoint /interaction/v1/EventDefinitions return all Event Definitions with all details.
* Currently it is not needed to loop over Imports with endpoint /interaction/v1/EventDefinitions/{id}
* Endpoint /interaction/v1/eventDefinitions return all Event Definitions with all details.
* Currently it is not needed to loop over Imports with endpoint /interaction/v1/eventDefinitions/{id}
*
* @param {string} retrieveDir Directory where retrieved metadata directory will be saved
* @param {void | string[]} [_] unused parameter
Expand All @@ -39,7 +39,7 @@ class Event extends MetadataType {
Util.logBeta(this.definition.type);
return super.retrieveREST(
retrieveDir,
`/interaction/v1/EventDefinitions${
`/interaction/v1/eventDefinitions${
key ? '/key:' + encodeURIComponent(key) : ''
}?extras=all`,
null,
Expand All @@ -53,7 +53,7 @@ class Event extends MetadataType {
* @returns {Promise.<MetadataTypeMapObj>} Promise of metadata
*/
static retrieveForCache() {
return super.retrieveREST(null, '/interaction/v1/EventDefinitions/');
return super.retrieveREST(null, '/interaction/v1/eventDefinitions/');
}

/**
Expand All @@ -67,7 +67,7 @@ class Event extends MetadataType {
static async retrieveAsTemplate(templateDir, name, templateVariables) {
Util.logBeta(this.definition.type);
const res = await this.client.rest.get(
'/interaction/v1/EventDefinitions?name=' + encodeURIComponent(name)
'/interaction/v1/eventDefinitions?name=' + encodeURIComponent(name)
);
const event = res.items.filter((item) => item.name === name);
try {
Expand Down Expand Up @@ -162,7 +162,7 @@ class Event extends MetadataType {
* @returns {Promise} Promise
*/
static create(metadata) {
return super.createREST(metadata, '/interaction/v1/EventDefinitions/');
return super.createREST(metadata, '/interaction/v1/eventDefinitions/');
}

/**
Expand All @@ -174,7 +174,7 @@ class Event extends MetadataType {
static async update(metadataEntry) {
return super.updateREST(
metadataEntry,
'/interaction/v1/EventDefinitions/' + metadataEntry.id,
'/interaction/v1/eventDefinitions/' + metadataEntry.id,
'put'
);
}
Expand Down
15 changes: 15 additions & 0 deletions lib/metadataTypes/definitions/DataExtension.definition.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,12 @@ export default {
SMSSubscriptionLog: null, // CustomerKey cannot be retrieved
TriggeredSendDataExtension: ['SubscriberKey', 'EmailAddress'],
},
dataRetentionPeriodUnitOfMeasureMapping: {
Days: 3,
Weeks: 4,
Months: 5,
Years: 6,
},
hasExtended: false,
idField: 'ObjectID',
keyIsFixed: false,
Expand Down Expand Up @@ -284,5 +290,14 @@ export default {
r__dataExtensionTemplate_Name: {
skipValidation: true,
},
c__retentionPolicy: {
skipValidation: true,
},
c__retainUntil: {
skipValidation: true,
},
c__dataRetentionPeriodUnitOfMeasure: {
skipValidation: true,
},
},
};
7 changes: 3 additions & 4 deletions ...nstance/_ParentBU_/dataExtension/testExisting_dataExtensionShared.dataExtension-meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@
"SendableDataExtensionField": { "Name": "ContactKey" },
"SendableSubscriberField": { "Name": "Subscriber Key" },
"DataRetentionPeriodLength": 6,
"DataRetentionPeriodUnitOfMeasure": 5,
"RowBasedRetention": true,
"ResetRetentionPeriodOnImport": false,
"DeleteAtEndOfRetentionPeriod": false,
"RetainUntil": "",
"Fields": [
{
"Name": "FirstName",
Expand Down Expand Up @@ -54,6 +50,9 @@
"FieldType": "Text"
}
],
"c__dataRetentionPeriodUnitOfMeasure": "Days",
"c__retainUntil": "2024-5-9",
"c__retentionPolicy": "allRecords",
"r__folder_ContentType": "shared_dataextension",
"r__folder_Path": "Shared Items/Shared Data Extensions"
}
5 changes: 1 addition & 4 deletions ...testInstance/_ParentBU_/dataExtension/testNew_dataExtensionShared.dataExtension-meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@
"Description": "",
"IsSendable": false,
"IsTestable": false,
"RowBasedRetention": false,
"ResetRetentionPeriodOnImport": false,
"DeleteAtEndOfRetentionPeriod": false,
"RetainUntil": "",
"Fields": [
{
"Name": "testField",
Expand All @@ -18,6 +14,7 @@
"FieldType": "Text"
}
],
"c__retentionPolicy": "none",
"r__folder_ContentType": "shared_dataextension",
"r__folder_Path": "Shared Items/Shared Data Extensions"
}
7 changes: 3 additions & 4 deletions ...ploy/testInstance/testBU/dataExtension/testExisting_dataExtension.dataExtension-meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@
"SendableDataExtensionField": { "Name": "ContactKey" },
"SendableSubscriberField": { "Name": "Subscriber Key" },
"DataRetentionPeriodLength": 6,
"DataRetentionPeriodUnitOfMeasure": 5,
"RowBasedRetention": true,
"ResetRetentionPeriodOnImport": false,
"DeleteAtEndOfRetentionPeriod": false,
"RetainUntil": "",
"Fields": [
{
"Name": "FirstName",
Expand Down Expand Up @@ -55,6 +51,9 @@
"FieldType": "Text"
}
],
"c__dataRetentionPeriodUnitOfMeasure": "Days",
"c__retainUntil": "2024-5-9",
"c__retentionPolicy": "allRecords",
"r__folder_ContentType": "dataextension",
"r__folder_Path": "Data Extensions"
}
5 changes: 1 addition & 4 deletions ...ot/deploy/testInstance/testBU/dataExtension/testNew_dataExtension.dataExtension-meta.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,6 @@
"Description": "",
"IsSendable": false,
"IsTestable": false,
"RowBasedRetention": false,
"ResetRetentionPeriodOnImport": false,
"DeleteAtEndOfRetentionPeriod": false,
"RetainUntil": "",
"Fields": [
{
"Name": "testField",
Expand All @@ -18,6 +14,7 @@
"FieldType": "Text"
}
],
"c__retentionPolicy": "none",
"r__folder_ContentType": "dataextension",
"r__folder_Path": "Data Extensions"
}
11 changes: 4 additions & 7 deletions test/resources/1111111/dataExtension/create-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
{
"CustomerKey": "testNew_dataExtensionShared",
"DeleteAtEndOfRetentionPeriod": false,
"Name": "testNew_dataExtensionShared",
"Description": "",
"IsSendable": false,
"IsTestable": false,
"Fields": [
{
"DefaultValue": "",
Expand All @@ -12,12 +14,7 @@
"Name": "testField"
}
],
"IsSendable": false,
"IsTestable": false,
"Name": "testNew_dataExtensionShared",
"ResetRetentionPeriodOnImport": false,
"RetainUntil": "",
"RowBasedRetention": false,
"c__retentionPolicy": "none",
"r__folder_ContentType": "shared_dataextension",
"r__folder_Path": "Shared Items/Shared Data Extensions"
}
6 changes: 2 additions & 4 deletions test/resources/1111111/dataExtension/retrieve-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
{
"CustomerKey": "testExisting_dataExtensionShared",
"DataRetentionPeriodLength": 6,
"DataRetentionPeriodUnitOfMeasure": 5,
"DeleteAtEndOfRetentionPeriod": false,
"Description": "bla bla",
"Fields": [
{
Expand Down Expand Up @@ -42,14 +40,14 @@
"IsTestable": true,
"Name": "testExisting_dataExtensionShared",
"ResetRetentionPeriodOnImport": false,
"RetainUntil": "",
"RowBasedRetention": true,
"SendableDataExtensionField": {
"Name": "ContactKey"
},
"SendableSubscriberField": {
"Name": "Subscriber Key"
},
"c__dataRetentionPeriodUnitOfMeasure": "Months",
"c__retentionPolicy": "individialRecords",
"r__folder_ContentType": "shared_dataextension",
"r__folder_Path": "Shared Items/Shared Data Extensions"
}
14 changes: 7 additions & 7 deletions test/resources/1111111/dataExtension/update-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
{
"CustomerKey": "testExisting_dataExtensionShared",
"DeleteAtEndOfRetentionPeriod": false,
"Name": "testExisting_dataExtensionShared",
"Description": "Container for my test emails",
"IsSendable": false,
"IsTestable": false,
"DataRetentionPeriodLength": 6,
"ResetRetentionPeriodOnImport": false,
"Fields": [
{
"DefaultValue": "",
Expand Down Expand Up @@ -44,12 +48,8 @@
"Name": "newField"
}
],
"IsSendable": false,
"IsTestable": false,
"Name": "testExisting_dataExtensionShared",
"ResetRetentionPeriodOnImport": false,
"RetainUntil": "",
"RowBasedRetention": true,
"c__dataRetentionPeriodUnitOfMeasure": "Months",
"c__retentionPolicy": "individialRecords",
"r__folder_ContentType": "shared_dataextension",
"r__folder_Path": "Shared Items/Shared Data Extensions"
}
4 changes: 3 additions & 1 deletion test/resources/1111111/dataExtension/update-response.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@
<Description>Container for my test emails</Description>
<IsSendable>false</IsSendable>
<IsTestable>false</IsTestable>
<DataRetentionPeriodLength>6</DataRetentionPeriodLength>
<DataRetentionPeriodUnitOfMeasure>5</DataRetentionPeriodUnitOfMeasure>
<RowBasedRetention>false</RowBasedRetention>
<ResetRetentionPeriodOnImport>false</ResetRetentionPeriodOnImport>
<DeleteAtEndOfRetentionPeriod>false</DeleteAtEndOfRetentionPeriod>
<DeleteAtEndOfRetentionPeriod>true</DeleteAtEndOfRetentionPeriod>
<RetainUntil />
<Fields>
<Field>
Expand Down
7 changes: 3 additions & 4 deletions test/resources/9999999/dataExtension/build-expected.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,7 @@
"SendableDataExtensionField": { "Name": "ContactKey" },
"SendableSubscriberField": { "Name": "Subscriber Key" },
"DataRetentionPeriodLength": 6,
"DataRetentionPeriodUnitOfMeasure": 5,
"RowBasedRetention": true,
"ResetRetentionPeriodOnImport": false,
"DeleteAtEndOfRetentionPeriod": false,
"RetainUntil": "",
"Fields": [
{
"Name": "FirstName",
Expand Down Expand Up @@ -62,6 +58,9 @@
"FieldType": "Number"
}
],
"c__dataRetentionPeriodUnitOfMeasure": "Months",
"c__retainUntil": "2024-5-9",
"c__retentionPolicy": "allRecords",
"r__folder_ContentType": "dataextension",
"r__folder_Path": "Data Extensions"
}
Loading
Loading