Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat!: Support Frame Chunking for tasks #77

Open
wants to merge 5 commits into
base: mainline
Choose a base branch
from
Open

feat!: Support Frame Chunking for tasks #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3471379
Update AfterEffects environment variable naming
Ahuge Dec 2, 2024
9c41d77
[Fonts] Generate local font references
Ahuge Dec 10, 2024
aa7b65f
Implement Frame Chunking logic
Ahuge Dec 7, 2024
38e8303
Save FramesPerTask as a setting
Ahuge Dec 9, 2024
b67edc5
Error handling around the batch file
Ahuge Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/deadline/ae_adaptor/AEClient/ae_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(self, server_path: str) -> None:
client_path_abs = client_path_abs.replace("\\", "\\\\")
startup_script_inline = f"var x = new File('{client_path_abs}') ; x.open(); eval(x.read()); app.exitAfterLaunchAndEval = false;"

ae_exe = os.environ.get("AFTEREFFECTS_ADAPTOR_AEFX_EXECUTABLE", "afterfx")
ae_exe = os.environ.get("AFTERFX_EXECUTABLE", "afterfx")

# flag -noui for no ui doesn't close properly when running in monitor
cmd_args = [ae_exe, "-noui", "-s", startup_script_inline]
Expand Down
2 changes: 2 additions & 0 deletions src/deadline/ae_submitter/Imports.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@

#include "data/InitData.jsx"

#include "data/RenderScript.jsx"

#include "submission/AssetReferenceTemplate.jsx"

#include "submission/DataTemplate.jsx"
Expand Down
18 changes: 18 additions & 0 deletions src/deadline/ae_submitter/UI/AESubmitterUI.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ function __generateSubmitterUI() {
} else {
initFirstAndLast = dcUtil.parseBool(dcSettings.getIniSetting("FirstAndLast", "false"));
}
var initFramesPerTask = parseInt(dcSettings.getIniSetting("FramesPerTask", "10"));

var initIgnoreMissingLayers = dcUtil.parseBool(dcSettings.getIniSetting("MissingLayers", "false"));
var initIgnoreMissingEffects = dcUtil.parseBool(dcSettings.getIniSetting("MissingEffects", "false"));
Expand Down Expand Up @@ -672,6 +673,12 @@ function __generateSubmitterUI() {
frameListGroup.useCompFrameList.value = initUseCompFrameRange;
frameListGroup.useCompFrameList.helpTip = 'If enabled, the Comp\'s frame list will be used instead of the frame list in this submitter.';

frameListGroup.framesPerTask = frameListGroup.add('edittext', undefined, initFramesPerTask);
frameListGroup.framesPerTask.size = SHORT_TEXT_SIZE;
frameListGroup.framesPerTaskLabel = frameListGroup.add('statictext', undefined, 'Frames Per Task');
frameListGroup.framesPerTaskLabel.size = LABEL_SIZE;
frameListGroup.framesPerTaskLabel.helpTip = 'This is the number of frames to be rendered per Job Task.';

// Create Comp submission group and widgets (Select One, Use Selected in RQ, All as separate)
compSubmissionGroup = deadlineCloud.aeOptionsPanel.add('group', undefined);
compSubmissionGroup.alignment = [ScriptUI.Alignment.FILL, ScriptUI.Alignment.TOP];
Expand Down Expand Up @@ -1222,6 +1229,17 @@ function __generateSubmitterUI() {
}
}

frameListGroup.framesPerTask.onChange = function() {
currentValue = frameListGroup.framesPerTask.text.replace(/[^\d]/g, '');

if (parseInt(currentValue) > 9999) {
frameListGroup.framesPerTask.text = "9999"
}
if (parseInt(currentValue) < 1) {
frameListGroup.framesPerTask.text = "1"
}
}

frameListGroup.frameListLabel.enabled = !initSubmitEntireQueue && !initMultiMachine;
frameListGroup.frameList.enabled = !initUseCompFrameRange && !initSubmitEntireQueue && !initMultiMachine;
frameListGroup.useCompFrameList.enabled = !initSubmitEntireQueue && !initMultiMachine;
Expand Down
70 changes: 46 additions & 24 deletions src/deadline/ae_submitter/UI/SubmitButton.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,26 +79,40 @@ function __generateSubmitButton() {
function generateStep(basicTemplate, itemName, stepsTemplate, stepID, renderQueueItem)
{
var compNameToCheck = dcUtil.removeIllegalCharacters(renderQueueItem.comp.name);
// logger.debug("[generateStep] itemName: " + itemName + " stepID: " + stepID + " compNameToCheck: " + compNameToCheck, _submitButtonFileName);
stepsTemplate[0].name = itemName;
stepsTemplate[0].parameterSpace.taskParameterDefinitions[0].range = "{{Param." + itemName + "_Frames}}";
stepsTemplate[0].script.embeddedFiles[0].data = "frame: {{Task.Param.Frame}}";
if(itemName != compNameToCheck)
{
stepsTemplate[0].stepEnvironments[0].script.embeddedFiles[0].data += "comp_name: {{Param." + compNameToCheck + "_CompName}} \n";
}
else
{
stepsTemplate[0].stepEnvironments[0].script.embeddedFiles[0].data += "comp_name: {{Param." + itemName + "_CompName}} \n";
}
stepsTemplate[0].stepEnvironments[0].script.embeddedFiles[0].data += "output_file_path: {{Param." + itemName + "_OutputFilePath}} \n";
stepsTemplate[0].stepEnvironments[0].script.embeddedFiles[0].data += "output_pattern: {{Param." + itemName + "_OutputPattern}} \n";
stepsTemplate[0].stepEnvironments[0].script.embeddedFiles[0].data += "output_format: {{Param." + itemName + "_OutputFormat}} \n";
stepsTemplate[0].parameterSpace.taskParameterDefinitions[0].range = "{{Param." + itemName + "_FrameStart}} - {{Param." + itemName + "_FrameEnd}} : {{Param." + itemName + "_ChunkSize}}";
stepsTemplate[0].parameterSpace.taskParameterDefinitions[1].range = "{{Param." + itemName + "_FrameStartPlusChunkSizeMinusOne}}-{{Param." + itemName + "_FrameEndMinusOne}}:{{Param." + itemName + "_ChunkSize}},{{Param." + itemName + "_FrameEnd}}";

stepsTemplate[0].script.embeddedFiles[0].data = dcRenderScript.generateRenderCommand("{{Param." + itemName + "_CompName}}")

basicTemplate.steps[stepID-1] = stepsTemplate[0];
// logger.debug("[generateStep] basicTemplate: " + JSON.stringify(basicTemplate), _submitButtonFileName);
return basicTemplate;
}

function applyFrameChunkingDataToTemplate(itemName, frameList)
{
var output = {
"parameterDefinitions": [],
"parameterValues": [],
}

var chunkParametersList = dcUtil.getFrameChunkParameters(frameList, frameListGroup.framesPerTask.text);
// Currently only support a single frame range
output.parameterDefinitions.push(applyDataToTemplate(itemName + "_FrameStart", dcUtil.deepCopy(dcDataTemplate.FrameStart)));
output.parameterDefinitions.push(applyDataToTemplate(itemName + "_FrameEnd", dcUtil.deepCopy(dcDataTemplate.FrameEnd)));
output.parameterDefinitions.push(applyDataToTemplate(itemName + "_ChunkSize", dcUtil.deepCopy(dcDataTemplate.ChunkSize)));
output.parameterDefinitions.push(applyDataToTemplate(itemName + "_FrameStartPlusChunkSizeMinusOne", dcUtil.deepCopy(dcDataTemplate.FrameStartPlusChunkSizeMinusOne)));
output.parameterDefinitions.push(applyDataToTemplate(itemName + "_FrameEndMinusOne", dcUtil.deepCopy(dcDataTemplate.FrameEndMinusOne)));

output.parameterValues.push(applyDataToParameterTemplate(itemName + "_FrameStart", chunkParametersList[0].frameStart));
output.parameterValues.push(applyDataToParameterTemplate(itemName + "_FrameEnd", chunkParametersList[0].frameEnd));
output.parameterValues.push(applyDataToParameterTemplate(itemName + "_ChunkSize", chunkParametersList[0].chunkSize));
output.parameterValues.push(applyDataToParameterTemplate(itemName + "_FrameStartPlusChunkSizeMinusOne", chunkParametersList[0].frameStartPlusChunkSizeMinusOne));
output.parameterValues.push(applyDataToParameterTemplate(itemName + "_FrameEndMinusOne", chunkParametersList[0].frameEndMinusOne));
return output
}

function applyDataToTemplate(dataName, dataTemplate)
{
if(dataName.indexOf("Frames") >= 0 || dataName.indexOf("OutputFilePath") >= 0)
Expand Down Expand Up @@ -315,6 +329,7 @@ function __generateSubmitButton() {

if (submissionText == useQueue)
totalJobs = 1;
queuedCount = 1;

progressBarPanel.progressBar.value = 0;

Expand Down Expand Up @@ -467,26 +482,27 @@ function __generateSubmitButton() {

var __compName = dcUtil.removeIllegalCharacters(app.project.renderQueue.item(j).comp.name);
var frameList = getFrameList(app.project.renderQueue.item(j));

// logger.debug("[createDataAndParameterTemplateOneJob] __compName: " + __compName, _submitButtonFileName);
// logger.debug("[createDataAndParameterTemplateOneJob] frameList: " + frameList, _submitButtonFileName);


var chunking = applyFrameChunkingDataToTemplate(__compName, frameList)

// Add data to the main template
jobTemplate.parameterDefinitions.push(applyDataToTemplate(__compName + "_Frames", dcUtil.deepCopy(dcDataTemplate.Frames)));
for (var pIdx = 0; pIdx < chunking.parameterDefinitions.length; pIdx++) {
jobTemplate.parameterDefinitions.push(chunking.parameterDefinitions[pIdx]);
}
jobTemplate.parameterDefinitions.push(applyDataToTemplate(__compName + "_OutputPattern", dcUtil.deepCopy(dcDataTemplate.OutputPattern)));
jobTemplate.parameterDefinitions.push(applyDataToTemplate(__compName + "_OutputFormat", dcUtil.deepCopy(dcDataTemplate.OutputFormat)));
jobTemplate.parameterDefinitions.push(applyDataToTemplate(__compName + "_CompName", dcUtil.deepCopy(dcDataTemplate.CompName)));
jobTemplate.parameterDefinitions.push(applyDataToTemplate(__compName + "_OutputFilePath", dcUtil.deepCopy(dcDataTemplate.OutputFilePath)));

// Add steps data per task that needs to be run
jobTemplate = generateStep(jobTemplate, __compName, dcUtil.deepCopy(stepsTemplate), stepIndex, app.project.renderQueue.item(j));
// logger.debug("[createDataAndParameterTemplateOneJob] jobTemplate: " + JSON.stringify(jobTemplate), _submitButtonFileName);

jobTemplate = applyHostReqToTemplate(jobTemplate);
// logger.debug("[createDataAndParameterTemplateOneJob] jobTemplate applyHostReqToTemplate: " + JSON.stringify(jobTemplate), _submitButtonFileName);

// Add data to the parameter template
jobParams.parameterValues.push(applyDataToParameterTemplate(__compName + "_Frames", frameList));
for (var pIdx = 0; pIdx < chunking.parameterValues.length; pIdx++) {
jobParams.parameterValues.push(chunking.parameterValues[pIdx]);
}
jobParams.parameterValues.push(applyDataToParameterTemplate(__compName + "_CompName", app.project.renderQueue.item(j).comp.name));
jobParams.parameterValues.push(applyDataToParameterTemplate(__compName + "_OutputPattern", dcUtil.removePercentageFromFileName(getRenderQueueItemData(app.project.renderQueue.item(j))["fileName"])));
jobParams.parameterValues.push(applyDataToParameterTemplate(__compName + "_OutputFormat", getRenderQueueItemData(app.project.renderQueue.item(j))["extension"]));
Expand All @@ -505,9 +521,13 @@ function __generateSubmitButton() {
{
var compNameToCheck = dcUtil.removeIllegalCharacters(renderQueueItem.comp.name);
var frameList = getFrameList(renderQueueItem);

var chunking = applyFrameChunkingDataToTemplate(itemName, frameList)
jobTemplate = generatePartialTemplate();
// Add data to the main template
jobTemplate.parameterDefinitions.push(applyDataToTemplate(itemName + "_Frames", dcDataTemplate.Frames));
for (var pIdx = 0; pIdx < chunking.parameterDefinitions.length; pIdx++) {
jobTemplate.parameterDefinitions.push(chunking.parameterDefinitions[pIdx]);
}
jobTemplate.parameterDefinitions.push(applyDataToTemplate(itemName + "_OutputPattern", dcDataTemplate.OutputPattern));
jobTemplate.parameterDefinitions.push(applyDataToTemplate(itemName + "_OutputFormat", dcDataTemplate.OutputFormat));
// jobTemplate.parameterDefinitions.push(applyDataToTemplate(itemName + "_CompName", dcDataTemplate.CompName));
Expand All @@ -523,7 +543,9 @@ function __generateSubmitButton() {
jobParams.parameterValues.push(applyDataToParameterTemplate("deadline:maxFailedTasksCount", dcProperties.deadlineJobParameters.maxFailedTasksCount.get()));
jobParams.parameterValues.push(applyDataToParameterTemplate("deadline:maxRetriesPerTask", dcProperties.deadlineJobParameters.maxRetriesPerTask.get()));
jobParams.parameterValues.push(applyDataToParameterTemplate("deadline:priority", dcProperties.deadlineJobParameters.priority.get()));
jobParams.parameterValues.push(applyDataToParameterTemplate(itemName + "_Frames", frameList));
for (var pIdx = 0; pIdx < chunking.parameterValues.length; pIdx++) {
jobParams.parameterValues.push(chunking.parameterValues[pIdx]);
}
// jobParams.parameterValues.push(applyDataToParameterTemplate(itemName + "_CompName", comp));
jobParams.parameterValues.push(applyDataToParameterTemplate(itemName + "_OutputPattern", dcUtil.removePercentageFromFileName((getRenderQueueItemData(renderQueueItem)["fileName"]))));
jobParams.parameterValues.push(applyDataToParameterTemplate(itemName + "_OutputFormat", getRenderQueueItemData(renderQueueItem)["extension"]));
Expand Down
75 changes: 74 additions & 1 deletion src/deadline/ae_submitter/data/InitData.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,82 @@ function __generateInitData()
}
}
}
var fontReferences = generateFontReferences();
logger.debug("Get Fonts: " + key, scriptFileInitDataName);
for (var i = 0; i < fontReferences.length; i++) {
detectedItemsList.push(fontReferences[i]);
}
dcProperties.jobAttachments.autoDetectedInputFiles.set(detectedItemsList);
}

function _generateFontReferences() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems out of the scope of this PR. Could we only have Chunking change in this PR?

var fontLocations = [];
var usedList = app.project.usedFonts;
for (var i = 0; i < usedList.length; i++) {
var font = usedList[i].font;
var fontFamilyName = font.familyName;
var familyStyle = font.styleName;
var fontName = fontFamilyName + " " + familyStyle + ".otf";
var fontLocation = font.location;
fontLocations.push([fontName, fontLocation]);
}
return fontLocations
}

function _oldGenerateFontReferences() {
var fontLocations = [];
var items = app.project.items;
for (var i = items.length; i >= 1; i--) {
var myItem = app.project.item(i);
if (myItem instanceof CompItem) {
for (var j = myItem.layers.length; j >= 1; j--) {
var myLayer = myItem.layers[j];
if (myLayer instanceof TextLayer){
var textDocument = myLayer.text.sourceText.value;
var fontLocation = textDocument.fontLocation;
var fontFamilyName = textDocument.fontFamily;
var familyStyle = textDocument.fontStyle;
var fontName = fontFamilyName + " " + familyStyle + ".otf";
fontLocations.push([fontName, fontLocation]);
}
}
}
}
return fontLocations
}

function generateFontReferences() {
var rawFontPaths = [];
if (parseInt(app.version[0] + app.version[1]) >= 24) {
rawFontPaths = _generateFontReferences();
} else {
rawFontPaths = _oldGenerateFontReferences();
}
var _tempFilePath = dcUtil.normPath(Folder.temp.fsName + "/" + "tempFonts");
var formattedFontPaths = [];
var tempFontPath = new Folder(_tempFilePath);
if (!tempFontPath.exists) {
tempFontPath.create();
}


for (var i = 0; i < rawFontPaths.length; i++) {
var fontName = rawFontPaths[i][0];
var fontLocation = rawFontPaths[i][1];
if (fontLocation.indexOf("C:\\Windows\\Fonts") !== -1) {
// We are filtering out fonts installed in C:\Windows\Fonts potentially here.
// I would make the argument that if a font is installed to the system (not an Adobe Font), that
// it should be managed on a system level not through the submitter.
continue
}
var fontFile = File(fontLocation);
var _filePath = dcUtil.normPath(_tempFilePath + "/" + fontName);
fontFile.copy(_filePath);
formattedFontPaths.push(_filePath);
}
return formattedFontPaths;
}

function initAutoDetectOutputDirectories()
{
var detectedOutputDirectories= [];
Expand Down Expand Up @@ -153,4 +226,4 @@ function __generateInitData()
}
}

dcInitData = __generateInitData();
dcInitData = __generateInitData();
19 changes: 19 additions & 0 deletions src/deadline/ae_submitter/data/RenderScript.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
function __generateRenderScriptTemplate(compParameterName) {
return (
"@ECHO OFF\n" +
"IF \"%AFTEREFFECTS_ADAPTOR_AERENDER_EXECUTABLE%\" == \"\" (\n" +
" set AFTEREFFECTS_ADAPTOR_AERENDER_EXECUTABLE=aerender.exe\n" +
")\n" +
"echo \"Running: \\\"%AFTEREFFECTS_ADAPTOR_AERENDER_EXECUTABLE%\\\" -project \\\"{{Param.AfterEffectsProjectFile}}\\\" -comp \\\"" + compParameterName + "\\\" -s {{Task.Param.FrameChunkStart}} -e {{Task.Param.FrameChunkEnd}}\"\n" +
"\"%AFTEREFFECTS_ADAPTOR_AERENDER_EXECUTABLE%\" -project \"{{Param.AfterEffectsProjectFile}}\" -comp \"" + compParameterName + "\" -s {{Task.Param.FrameChunkStart}} -e {{Task.Param.FrameChunkEnd}}\n" +
"IF %ERRORLEVEL% NEQ 0 (\n" +
" echo \"Return code: %ERRORLEVEL%\"\n" +
" exit %ERRORLEVEL%\n" +
")\n" +
"exit 0\n"
)
}

dcRenderScript = {
"generateRenderCommand": __generateRenderScriptTemplate,
};
63 changes: 58 additions & 5 deletions src/deadline/ae_submitter/submission/DataTemplate.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,65 @@
function __generateDataTemplate() {
var Frames =
var FrameStart =
{
"name": "Frames",
"name": "FrameStart",
"type": "STRING",
"userInterface": {
"control": "LINE_EDIT",
"label": "Frames",
"label": "Starting Frames",
"groupLabel": "After Effects Settings"
},
"description": "The frames to render. E.g. 1,8,11",
"description": "The starting frame to render.",
"minLength": 1
}
var FrameEnd =
{
"name": "FrameEnd",
"type": "STRING",
"userInterface": {
"control": "LINE_EDIT",
"label": "Ending Frames",
"groupLabel": "After Effects Settings"
},
"description": "The ending frame to render.",
"minLength": 1
}
var ChunkSize =
{
"name": "ChunkSize",
"type": "STRING",
"userInterface": {
"control": "LINE_EDIT",
"label": "Frames Per Task",
"groupLabel": "After Effects Settings"
},
"description": "The chunk size of frames per task to render",
"minLength": 1
}
var FrameStartPlusChunkSizeMinusOne =
{
"name": "FrameStartPlusChunkSizeMinusOne",
"type": "STRING",
"userInterface": {
"control": "LINE_EDIT",
"label": "FrameStart + ChunkSize - 1",
"groupLabel": "After Effects Settings"
},
"description": "[Internal] This value needs to equal FrameStart + ChunkSize - 1",
"minLength": 1
}
var FrameEndMinusOne =
{
"name": "FrameEndMinusOne",
"type": "STRING",
"userInterface": {
"control": "LINE_EDIT",
"label": "FrameEnd - 1",
"groupLabel": "After Effects Settings"
},
"description": "[Internal] This value needs to equal FrameEnd - 1",
"minLength": 1
}

var OutputPattern =
{
"name": "OutputPattern",
Expand Down Expand Up @@ -45,7 +94,11 @@ function __generateDataTemplate() {
"description": "The render output path."
}
return {
"Frames": Frames,
"FrameStart": FrameStart,
"FrameEnd": FrameEnd,
"ChunkSize": ChunkSize,
"FrameStartPlusChunkSizeMinusOne": FrameStartPlusChunkSizeMinusOne,
"FrameEndMinusOne": FrameEndMinusOne,
"OutputPattern" : OutputPattern,
"OutputFormat": OutputFormat,
"CompName": CompName,
Expand Down
Loading