Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bug/2011 throttle createDeltaPkg to prevent it from failing for 100+ changes #2012

Merged
merged 1 commit into from
Jan 27, 2025
Merged

bug/2011 throttle createDeltaPkg to prevent it from failing for 100+ changes #2012

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion @types/lib/util/devops.d.ts.map
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

140 changes: 83 additions & 57 deletions lib/util/devops.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ const git = simpleGit();
import Builder from '../Builder.js';
import MetadataType from '../MetadataTypeInfo.js';
import jsonToTable from 'json-to-table';
import pLimit from 'p-limit';
import cliProgress from 'cli-progress';

/**
* @typedef {import('../../types/mcdev.d.js').AuthObject} AuthObject
Expand Down Expand Up @@ -97,11 +99,13 @@ const DevOps = {
'add/update': 0,
move: 0,
};
const diffSummary = await git.diffSummary([range]);
Util.logger.info(`Found ${diffSummary.files.length} changed files in the commit range`);
/**
* @type {DeltaPkgItem[]}
*/
// first, process everything that can be done synchronously here
const deltaSync = (await git.diffSummary([range])).files
const deltaSync = diffSummary.files
// populate additional info for all changed files
.map((/** @type {DeltaPkgItem} */ file) => {
// If file was moved it's path needs to be parsed
Expand Down Expand Up @@ -149,72 +153,94 @@ const DevOps = {
return false;
}
});

Util.logger.info(
`Identified ${deltaSync.length} relevant file changes after filtering file types and folders. Processing related metadata now.`
);
// second, we need to do asynchronous file operations

const extendedBar = new cliProgress.SingleBar(
{
format: ' Processing changes [{bar}] {percentage}% | {value}/{total}',
},
cliProgress.Presets.shades_classic
);
extendedBar.start(deltaSync.length, 0);
const rateLimit = pLimit(10);

/**
* @type {DeltaPkgItem[]}
*/
const delta = await Promise.all(
deltaSync.map(async (/** @type {DeltaPkgItem} */ file) => {
// Gets external key based on file name und the assumption that filename = externalKey
if (file.type === 'folder') {
file.externalKey = null;
file.name = path.basename(file.file).split('.').shift();
} else {
// find the key in paths like:
// - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html
// - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html
// - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql
file.externalKey =
file.type === 'asset'
? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype
: file.file.split('/')[4].split('.')[0];
file.name = null;
}

// Check if file doesn't exist in reported path, that means it was a git deletion
// TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing
if (!(await File.pathExists(file.file))) {
file.gitAction = 'delete';
} else if (file.moved) {
file.gitAction = 'move';
} else {
file.gitAction = 'add/update';
}
gitActionsCounter[file.gitAction]++;
file._credential = file.file.split('/')[1];
file._businessUnit = file.file.split('/')[2];

// Parse retrieve directory to also populate the name field (not possible for deleted files)
if (file.gitAction !== 'delete' && file.type !== 'folder') {
// folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders
const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`;
if (!metadata[file._credential]) {
metadata[file._credential] = {};
deltaSync.map((/** @type {DeltaPkgItem} */ file) =>
rateLimit(async () => {
// Gets external key based on file name und the assumption that filename = externalKey
if (file.type === 'folder') {
file.externalKey = null;
file.name = path.basename(file.file).split('.').shift();
} else {
// find the key in paths like:
// - retrieve/cred/bu/asset/block/016aecc7-7063-4b78-93f4-aa119ea933c7.asset-block-meta.html
// - retrieve/cred/bu/asset/message/003c1ef5-f538-473a-91da-26942024a64a/blocks/views.html.slots.[bottom-8frq7iw2k99].asset-message-meta.html
// - retrieve/cred/bu/query/03efd5f1-ba1f-487a-9c9a-36aeb2ae5192.query-meta.sql
file.externalKey =
file.type === 'asset'
? file.file.split('/')[5].split('.')[0] // assets have an additional folder level for their subtype
: file.file.split('/')[4].split('.')[0];
file.name = null;
}
if (!metadata[file._credential][file._businessUnit]) {
metadata[file._credential][file._businessUnit] = {};

// Check if file doesn't exist in reported path, that means it was a git deletion
// TODO: improve git action detection by switching from diffSummary to diff with --summary result parsing
if (!(await File.pathExists(file.file))) {
file.gitAction = 'delete';
} else if (file.moved) {
file.gitAction = 'move';
} else {
file.gitAction = 'add/update';
}
if (!metadata[file._credential][file._businessUnit][file.type]) {
try {
await MetadataType[file.type].readBUMetadataForType(
buPath,
false,
metadata[file._credential][file._businessUnit]
);
} catch (ex) {
// silently catch directory-not-found errors here
Util.logger.debug(ex.message);
gitActionsCounter[file.gitAction]++;
file._credential = file.file.split('/')[1];
file._businessUnit = file.file.split('/')[2];

// Parse retrieve directory to also populate the name field (not possible for deleted files)
if (file.gitAction !== 'delete' && file.type !== 'folder') {
// folders are saved with their name as file-name, not with their key, hence this section can be skipped for folders
const buPath = `${properties.directories.retrieve}/${file._credential}/${file._businessUnit}/`;
if (!metadata[file._credential]) {
metadata[file._credential] = {};
}
if (!metadata[file._credential][file._businessUnit]) {
metadata[file._credential][file._businessUnit] = {};
}
if (!metadata[file._credential][file._businessUnit][file.type]) {
try {
await MetadataType[file.type].readBUMetadataForType(
buPath,
false,
metadata[file._credential][file._businessUnit]
);
} catch (ex) {
// silently catch directory-not-found errors here
Util.logger.debug(ex.message);
}
}
const fileContent =
metadata[file._credential][file._businessUnit][file.type][
file.externalKey
];
if (fileContent) {
file.name = fileContent[MetadataType[file.type].definition.nameField];
}
}
const fileContent =
metadata[file._credential][file._businessUnit][file.type][file.externalKey];
if (fileContent) {
file.name = fileContent[MetadataType[file.type].definition.nameField];
}
}
return file;
})
extendedBar.increment();
return file;
})
)
);
// stop the progress bar
extendedBar.stop();

if (
!gitActionsCounter['add/update'] &&
!gitActionsCounter.move &&
Expand Down
Loading