diff --git a/build-conf/README.md b/build-conf/README.md index 762c24254..673960513 100644 --- a/build-conf/README.md +++ b/build-conf/README.md @@ -301,6 +301,16 @@ zcee3_shellEnvironment | Shell environment used to run the gradle command zcee3_gradlePath | Path to gradle executable zcee3_gradle_JAVA_OPTS | JAVA Options used with gradle +### zCEE2.properties +Application properties used by zAppBuild/language/zCEE2.groovy + +Property | Description +--- | --- +zcee2_zconbtPath | Absolute path to zconbt executable on z/OS UNIX System Services +zcee2_JAVA_HOME | Java installation used by the zconbt utility +zcee2_inputType | Mapping of input files with types of files +zcee2_ARA_PackageArtifacts | Flag to indicate if artifacts produced for the ARA processing should be packaged + ### CRB.properties Application properties used by zAppBuild/language/CRB.groovy diff --git a/build-conf/build.properties b/build-conf/build.properties index 60d0d5c34..acff34098 100644 --- a/build-conf/build.properties +++ b/build-conf/build.properties @@ -19,7 +19,7 @@ buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.properties,BMS.properties,\ MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\ LinkEdit.properties,PLI.properties,REXX.properties,ZunitConfig.properties,Transfer.properties,\ -CRB.properties,zCEE3.properties +CRB.properties,zCEE3.properties,zCEE2.properties # # Comma separated list of default application configuration property files to load diff --git a/build-conf/zCEE2.properties b/build-conf/zCEE2.properties new file mode 100644 index 000000000..f93d9c9f5 --- /dev/null +++ b/build-conf/zCEE2.properties @@ -0,0 +1,28 @@ +# Releng properties used by language/zCEE2.groovy + +# +# Comma separated list of required build properties for zCEE3.groovy +zcee2_requiredBuildProperties=zcee2_zconbtPath,zcee2_JAVA_HOME + +# +# Absolute path to zconbt executable on z/OS UNIX System Services +# for instance: /var/zosconnect/v359/bin/zconbt.zos +zcee2_zconbtPath= + +# +# Java installation used by the zconbt utility +# for instance: /usr/lpp/java/J8.0_64 +zcee2_JAVA_HOME= + +# +# Mapping of input files with types of files +# PROJECT can be used for SAR and AAR projects +# SAR and ARA can be used for SAR Properties files and ARA properties files +# Can be overridden by file-level properties +zcee2_inputType=PROJECT + +# +# Flag to indicate if artifacts produced for the ARA processing should be packaged (generated copybooks, API information copybook and logs) +# When set to true, the artifacts are located based on the dataStructuresLocation, apiInfoFileLocation and logFileDirectory properties of the ARA properties files +# When not defined, the default value is false and artifacts are packaged +zcee2_ARA_PackageArtifacts=true diff --git a/build.groovy b/build.groovy index 984298f71..d2bb12ad6 100644 --- a/build.groovy +++ b/build.groovy @@ -31,7 +31,15 @@ props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS") println("\n** Build start at $props.startTime") // initialize build -initializeBuildProcess(args) +try { + initializeBuildProcess(args) +} catch ( AssertionError e ) { + String errorMsg = e.getMessage() + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + finalizeBuildProcess(start:startTime, 0) +} // create build list List buildList = new ArrayList() @@ -57,13 +65,21 @@ else { scriptPath = script // Use the ScriptMappings class to get the files mapped to the build script def buildFiles = ScriptMappings.getMappedList(script, buildList) - if (buildFiles.size() > 0) { - if (scriptPath.startsWith('/')) - runScript(new File("${scriptPath}"), ['buildList':buildFiles]) - else - runScript(new File("languages/${scriptPath}"), ['buildList':buildFiles]) + try { + if (buildFiles.size() > 0) { + if (scriptPath.startsWith('/')) + runScript(new File("${scriptPath}"), ['buildList':buildFiles]) + else + runScript(new File("languages/${scriptPath}"), ['buildList':buildFiles]) + } + processCounter = processCounter + buildFiles.size() + } catch (BuildException | AssertionError e ) { + String errorMsg = e.getMessage() + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + finalizeBuildProcess(start:startTime, count:processCounter) } - processCounter = processCounter + buildFiles.size() } } else if(props.scanLoadmodules && props.scanLoadmodules.toBoolean()){ println ("** Scanning load modules.") @@ -77,10 +93,6 @@ if (processCounter == 0) finalizeBuildProcess(start:startTime, count:processCounter) -// if error occurred signal process error -if (props.error) - System.exit(1) - // end script @@ -96,10 +108,17 @@ def initializeBuildProcess(String[] args) { // print and store property dbb toolkit version in use def dbbToolkitVersion = VersionInfo.getInstance().getVersion() - props.dbbToolkitVersion = dbbToolkitVersion def dbbToolkitBuildDate = VersionInfo.getInstance().getDate() - if (props.verbose) println "** zAppBuild running on DBB Toolkit Version ${dbbToolkitVersion} ${dbbToolkitBuildDate} " + props.dbbToolkitVersion = dbbToolkitVersion + props.dbbToolkitBuildDate = dbbToolkitBuildDate + File versionFile = new File("${props.zAppBuildDir}/version.properties") + if (versionFile.exists()) { + props.load(versionFile) + if (props.zappbuild_version) println "** Running zAppBuild Version ${props.zappbuild_version} " + } + if (props.verbose) println "** Running DBB Toolkit Version ${dbbToolkitVersion} ${dbbToolkitBuildDate} " + // verify required dbb toolkit buildUtils.assertDbbBuildToolkitVersion(props.dbbToolkitVersion, props.requiredDBBToolkitVersion) @@ -329,6 +348,8 @@ def populateBuildProperties(def opts) { // need to support IDz user build parameters if (opts.srcDir) props.workspace = opts.srcDir if (opts.wrkDir) props.outDir = opts.wrkDir + + // assert workspace buildUtils.assertBuildProperties('workspace,outDir') // load build.properties @@ -641,7 +662,8 @@ def createBuildList() { } // Perform analysis and build report of external impacts - if (props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){ + // Prereq: Metadatastore Connection + if (metadataStore && props.reportExternalImpacts && props.reportExternalImpacts.toBoolean()){ if (buildSet && changedFiles) { println "** Perform analysis and reporting of external impacted files for the build list including changed files." reportingUtils.reportExternalImpacts(buildSet.plus(changedFiles)) @@ -653,7 +675,8 @@ def createBuildList() { } // Document and validate concurrent changes - if (props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){ + // Prereq: Workspace containing git repos. Skipped for --userBuild build type + if (!props.userBuild && props.reportConcurrentChanges && props.reportConcurrentChanges.toBoolean()){ println "** Calculate and document concurrent changes." reportingUtils.calculateConcurrentChanges(buildSet) } @@ -670,7 +693,6 @@ def createBuildList() { return buildList } - def finalizeBuildProcess(Map args) { println "***************** Finalization of the build process *****************" @@ -722,7 +744,10 @@ def finalizeBuildProcess(Map args) { buildResult.setProperty("filesProcessed", String.valueOf(args.count)) buildResult.setState(buildResult.COMPLETE) - + // add zAppBuild and DBB toolkit version info + if (props.zappbuild_version) buildResult.setProperty("zAppBuildVersion", props.zappbuild_version) + buildResult.setProperty("DBBToolkitVersion" , "${props.dbbToolkitVersion} ${props.dbbToolkitBuildDate}") + // store build result properties in BuildReport.json PropertiesRecord buildReportRecord = new PropertiesRecord("DBB.BuildResultProperties") def buildResultProps = buildResult.getPropertyNames() @@ -769,6 +794,10 @@ def finalizeBuildProcess(Map args) { if (props.preview) println("** Build ran in preview mode.") println("** Total files processed : ${args.count}") println("** Total build time : $duration\n") + + // if error occurred signal process error + if (props.error) + System.exit(1) } diff --git a/languages/Assembler.groovy b/languages/Assembler.groovy index 0a22f10b6..0f712532e 100644 --- a/languages/Assembler.groovy +++ b/languages/Assembler.groovy @@ -458,6 +458,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/Cobol.groovy b/languages/Cobol.groovy index 1fdf48ade..345b1526a 100644 --- a/languages/Cobol.groovy +++ b/languages/Cobol.groovy @@ -197,7 +197,7 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe compile.dd(new DDStatement().name("SYSPRINT").options(props.cobol_printTempOptions)) compile.dd(new DDStatement().name("SYSMDECK").options(props.cobol_tempOptions)) - (1..17).toList().each { num -> + (1..15).toList().each { num -> compile.dd(new DDStatement().name("SYSUT$num").options(props.cobol_tempOptions)) } @@ -384,6 +384,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/LinkEdit.groovy b/languages/LinkEdit.groovy index 8b7c4b060..8c69ca2f8 100644 --- a/languages/LinkEdit.groovy +++ b/languages/LinkEdit.groovy @@ -109,13 +109,16 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb linkedit.dd(new DDStatement().dsn(syslibDataset).options("shr")) } linkedit.dd(new DDStatement().dsn(props.SCEELKED).options("shr")) - + if (props.debug && props.SEQAMOD) linkedit.dd(new DDStatement().dsn(props.SEQAMOD).options("shr")) - - if (props.SDFHLOAD) + + if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) - + + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (props.SDSNLOAD) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/PLI.groovy b/languages/PLI.groovy index 34a90cf90..9264abde6 100644 --- a/languages/PLI.groovy +++ b/languages/PLI.groovy @@ -129,6 +129,7 @@ def createPLIParms(String buildFile, LogicalFile logicalFile) { def parms = props.getFileProperty('pli_compileParms', buildFile) ?: "" def cics = props.getFileProperty('pli_compileCICSParms', buildFile) ?: "" def sql = props.getFileProperty('pli_compileSQLParms', buildFile) ?: "" + def ims = props.getFileProperty('pli_compileIMSParms', buildFile) ?: "" def errPrefixOptions = props.getFileProperty('pli_compileErrorPrefixParms', buildFile) ?: "" def compileDebugParms = props.getFileProperty('pli_compileDebugParms', buildFile) @@ -142,6 +143,9 @@ def createPLIParms(String buildFile, LogicalFile logicalFile) { if (props.errPrefix) parms = "$parms,$errPrefixOptions" + if (buildUtils.isIMS(logicalFile)) + parms = "$parms,$ims" + // add debug options if (props.debug) { parms = "$parms,$compileDebugParms" @@ -355,6 +359,9 @@ def createLinkEditCommand(String buildFile, LogicalFile logicalFile, String memb if (buildUtils.isCICS(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr")) + if (buildUtils.isIMS(logicalFile)) + linkedit.dd(new DDStatement().dsn(props.SDFSRESL).options("shr")) + if (buildUtils.isSQL(logicalFile)) linkedit.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr")) diff --git a/languages/Transfer.groovy b/languages/Transfer.groovy index e3c914460..0e9ce3000 100644 --- a/languages/Transfer.groovy +++ b/languages/Transfer.groovy @@ -122,10 +122,8 @@ buildList.each { buildFile -> buildUtils.updateBuildResult(errorMsg:errorMsg) } } catch (BuildException e) { // Catch potential exceptions like file truncation - String errorMsg = "*! The CopyToPDS failed with an exception ${e.getMessage()}." - println(errorMsg) - props.error = "true" - buildUtils.updateBuildResult(errorMsg:errorMsg) + String errorMsg = "*! (Transfer.groovy) CopyToPDS of file ${buildFile} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) } } else { String errorMsg = "*! Target dataset for $buildFile could not be obtained from file properties. " diff --git a/languages/zCEE2.groovy b/languages/zCEE2.groovy new file mode 100644 index 000000000..c77a21e76 --- /dev/null +++ b/languages/zCEE2.groovy @@ -0,0 +1,219 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import com.ibm.dbb.dependency.* +import com.ibm.dbb.build.* +import groovy.transform.* +import com.ibm.dbb.build.report.* +import com.ibm.dbb.build.report.records.* +import java.nio.file.*; + + +// define script properties +@Field BuildProperties props = BuildProperties.getInstance() +@Field def buildUtils= loadScript(new File("${props.zAppBuildDir}/utilities/BuildUtilities.groovy")) + +// verify required build properties +buildUtils.assertBuildProperties(props.zcee2_requiredBuildProperties) + +// create updated build map, removing duplicates in case of PROJECT input Type +HashMap updatedBuildMap = new HashMap() + +println("** Streamlining the build list to remove duplicates") +argMap.buildList.each { buildFile -> + PropertyMappings inputTypeMappings = new PropertyMappings("zcee2_inputType") + inputType = inputTypeMappings.getValue(buildFile) + + if (inputType) { + if (inputType == "PROJECT") { + File changedBuildFile = new File(buildFile); + File projectDir = changedBuildFile.getParentFile() + boolean projectDirFound = false + while (projectDir != null && !projectDirFound) { + File projectFile = new File(projectDir.getPath() + '/.project') + if (projectFile.exists()) { + projectDirFound = true + } else { + projectDir = projectDir.getParentFile() + } + } + if (projectDirFound) { + updatedBuildMap.put(projectDir.getPath(), "PROJECT") + } else { + if (props.verbose) println("!* No project directory found for file '${buildFile}'. Skipping...") + } + } else { + updatedBuildMap.put(buildFile, inputType); + } + } else { + println("!* No Input Type mapping for file ${buildFile}, skipping it...") + } +} + +println("** Building ${updatedBuildMap.size()} API ${updatedBuildMap.size() == 1 ? 'definition' : 'definitions'} mapped to ${this.class.getName()}.groovy script") +// sort the build list based on build file rank if provided +HashMap sortedMap = buildUtils.sortBuildMap(updatedBuildMap, 'zcee2_fileBuildRank') + +int currentBuildFileNumber = 1 + +// iterate through build list +sortedMap.each { buildFile, inputType -> + println "*** (${currentBuildFileNumber++}/${sortedMap.size()}) Building ${inputType == "PROJECT" ? 'project' : 'properties file'} $buildFile" + + String parameters = "" + String outputDir = "" + String outputFile = "" + if (inputType == "PROJECT") { + outputDir = "${props.buildOutDir}/zCEE2/$buildFile" + parameters = "-od $outputDir -pd $buildFile" + } else { + File changedBuildFile = new File(buildFile); + String outputFileName = changedBuildFile.getName().split("\\.")[0] + "." + inputType.toLowerCase() + File projectDir = changedBuildFile.getParentFile() + outputFile = "${props.buildOutDir}/zCEE2/${projectDir.getPath()}/${outputFileName}" + outputDir = "${props.buildOutDir}/zCEE2/${projectDir.getPath()}" + parameters = "-f $outputFile -p $buildFile" + } + File outputDirectory = new File(outputDir) + outputDirectory.mkdirs() + + + Properties ARAproperties = new Properties() + File dataStructuresLocation + File apiInfoFileLocation + File logFileDirectory + if (inputType == "ARA") { + File ARApropertiesFile = new File(buildFile) + ARApropertiesFile.withInputStream { + ARAproperties.load(it) + } + println("*** dataStructuresLocation: ${ARAproperties.dataStructuresLocation}") + println("*** apiInfoFileLocation: ${ARAproperties.apiInfoFileLocation}") + println("*** logFileDirectory: ${ARAproperties.logFileDirectory}") + dataStructuresLocation = new File(ARAproperties.dataStructuresLocation) + dataStructuresLocation.mkdirs() + apiInfoFileLocation = new File(ARAproperties.apiInfoFileLocation) + apiInfoFileLocation.mkdirs() + logFileDirectory = new File(ARAproperties.logFileDirectory) + logFileDirectory.mkdirs() + } + + + // log file - Changing slashes with dots to avoid conflicts + String logFilePath = buildFile.replace("/", ".") + File logFile = new File("${props.buildOutDir}/${logFilePath}.zCEE2.log") + if (logFile.exists()) + logFile.delete() + + String zconbtPath = props.getFileProperty('zcee2_zconbtPath', buildFile) + + File zconbt = new File(zconbtPath) + if (!zconbt.exists()) { + def errorMsg = "*! zconbt wasn't find at location '$zconbtPath'" + println(errorMsg) + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + } else { + String[] command; + + command = [zconbtPath, parameters] + String commandString = command.join(" ") + if (props.verbose) + println("** Executing command '${commandString}'...") + + StringBuffer shellOutput = new StringBuffer() + StringBuffer shellError = new StringBuffer() + + String JAVA_HOME = props.getFileProperty('zcee2_JAVA_HOME', buildFile) + + ProcessBuilder cmd = new ProcessBuilder(zconbtPath, parameters); + Map env = cmd.environment(); + env.put("JAVA_HOME", JAVA_HOME); + env.put("PATH", JAVA_HOME + "/bin" + ";" + env.get("PATH")) + Process process = cmd.start() + process.consumeProcessOutput(shellOutput, shellError) + process.waitFor() + if (props.verbose) + println("** Exit value for the zconbt process: ${process.exitValue()}"); + + // write outputs to log file + String enc = props.logEncoding ?: 'IBM-1047' + logFile.withWriter(enc) { writer -> + writer.append(shellOutput) + writer.append(shellError) + } + + if (process.exitValue() != 0) { + def errorMsg = "*! Error during the zconbt process" + println(errorMsg) + if (props.verbose) + println("*! zconbt error message:\n${shellError}") + props.error = "true" + buildUtils.updateBuildResult(errorMsg:errorMsg) + } else { + if (props.verbose) + println("** zconbt output:\n${shellOutput}") + + ArrayList outputProperty = [] + Path outputDirectoryPath = Paths.get(props.buildOutDir) + if (inputType == "PROJECT") { + String[] outputFiles = outputDirectory.list() + for (int i=0; i WS-MAX-FIELD + MOVE WS-MAX-FIELD TO WS-IDX + ELSE + MOVE WS-IDX TO WS-MAX-FIELD + END-IF. + + MOVE ZERO TO WS-END-SPACE. + MOVE SPACES TO EPSPARM-RETURN-ERROR. + MOVE ZERO TO EPSPARM-BINARY-NUMBER + EPSPARM-NUMBER + EPSPARM-DECIMAL. + + * FIND TRAILING SPACES + PERFORM UNTIL WS-IDX = 0 + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACES + ADD 1 TO WS-TRAILING-SPACES + SUBTRACT 1 FROM WS-IDX + ELSE + MOVE WS-IDX TO WS-END-SPACE + MOVE 0 TO WS-IDX + END-IF + END-PERFORM. + + * FIND LEADING SPACES + MOVE 1 TO WS-LEADING-SPACES. + + IF WS-END-SPACE NOT = 0 + MOVE 1 TO WS-IDX + PERFORM UNTIL WS-IDX >= WS-END-SPACE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACES + ADD 1 TO WS-LEADING-SPACES + ADD 1 TO WS-IDX + ELSE + COMPUTE WS-IDX = WS-END-SPACE + 1 + END-IF + END-PERFORM + ELSE + MOVE STATIC-ERROR-TABLE(1) TO EPSPARM-RETURN-ERROR + END-IF. + + MOVE WS-LEADING-SPACES TO WS-IDX. + MOVE 1 TO WS-DEC-IDX. + MOVE 0 TO WS-DECIMAL-SPACE. + + * FIND DECIMAL POINT + PERFORM A002-COMPUTE-DECIMAL + UNTIL WS-IDX > WS-END-SPACE + . + + IF WS-DECIMAL-SPACE > 0 + COMPUTE WS-END-SPACE = WS-DECIMAL-SPACE - 1 + END-IF. + + * VALIDATE NO INTERNAL BLANKS + MOVE WS-END-SPACE TO WS-IDX. + MOVE LENGTH OF EPSPARM-NUMBER TO WS-NUM-IDX. + * SUBTRACT 1 FROM WS-NUM-IDX. + + PERFORM A001-COMPUTE-INTEGER + UNTIL WS-IDX < WS-LEADING-SPACES + . + + IF EPSPARM-RETURN-ERROR = SPACES + COMPUTE EPSPARM-BINARY-NUMBER = EPSPARM-NUMBER + + EPSPARM-DECIMAL + END-IF. + GOBACK + . + + A001-COMPUTE-INTEGER. + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = ',' + SUBTRACT 1 FROM WS-IDX + ELSE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = SPACE + OR EPSPARM-VALIDATE-DATA(WS-IDX:1) IS NOT NUMERIC + MOVE STATIC-ERROR-TABLE(2) TO EPSPARM-RETURN-ERROR + MOVE 0 TO WS-IDX + ELSE + MOVE EPSPARM-VALIDATE-DATA(WS-IDX:1) TO + EPSPARM-NUMBER(WS-NUM-IDX:1) + SUBTRACT 1 FROM WS-IDX + WS-NUM-IDX + END-IF + END-IF + . + + A002-COMPUTE-DECIMAL. + IF WS-DECIMAL-SPACE = 0 + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = '.' + MOVE WS-IDX TO WS-DECIMAL-SPACE + END-IF + ELSE + IF EPSPARM-VALIDATE-DATA(WS-IDX:1) = '.' + MOVE STATIC-ERROR-TABLE(3) TO EPSPARM-RETURN-ERROR + MOVE WS-END-SPACE TO WS-IDX + MOVE 1 TO WS-DEC-IDX + ELSE + MOVE EPSPARM-VALIDATE-DATA(WS-IDX:1) TO + EPSPARM-DECIMAL(WS-DEC-IDX:1) + ADD 1 TO WS-DEC-IDX + END-IF + END-IF + ADD 1 TO WS-IDX + . diff --git a/test/applications/MortgageApplication/test.properties b/test/applications/MortgageApplication/test.properties index be0c4c14b..caafda573 100644 --- a/test/applications/MortgageApplication/test.properties +++ b/test/applications/MortgageApplication/test.properties @@ -8,6 +8,7 @@ test_testOrder=resetBuild.groovy,\ mergeBuild.groovy,\ fullBuild.groovy,\ fullBuild_languageConfigurations.groovy,\ +fullBuild_fileTruncation.groovy,\ impactBuild.groovy,\ impactBuild_preview.groovy,\ impactBuild_properties.groovy,\ @@ -42,6 +43,18 @@ fullBuild_expectedFilesBuilt = epsnbrvl.cbl,epscsmrt.cbl,epsmort.bms,epsmlist.ln # list of source datasets (LLQ) that should be deleted during fullBuild.groovy cleanUp fullBuild_datasetsToCleanUp = BMS,COBOL,LINK +############################# +# fullBuild_fileTruncation.groovy properties +############################# +# +# list of changed source files to create test scenario with truncation error +fullBuild_truncation_changedFiles = cobol/epsnbrvl.cbl + +# list of changed source files to create test scenario with truncation error +fullBuild_truncation_errorMsg = CopyToPDS of buildFile MortgageApplication/cobol/epsnbrvl.cbl failed with an exception :: cobol/epsnbrvl.cbl + +# list of source datasets (LLQ) that should be deleted during cleanUp +fullBuild_datasetsToCleanUp = BMS,COBOL,OBJ,LINK ############################### # impactBuild.groovy properties diff --git a/test/testScripts/fullBuild_fileTruncation.groovy b/test/testScripts/fullBuild_fileTruncation.groovy new file mode 100644 index 000000000..89aecd4d2 --- /dev/null +++ b/test/testScripts/fullBuild_fileTruncation.groovy @@ -0,0 +1,113 @@ +@groovy.transform.BaseScript com.ibm.dbb.groovy.ScriptLoader baseScript +import groovy.transform.* +import com.ibm.dbb.* +import com.ibm.dbb.build.* + +@Field BuildProperties props = BuildProperties.getInstance() +@Field def testUtils = loadScript(new File("../utils/testUtilities.groovy")) + +println "\n**************************************************************" +println "** Executing test script ${this.class.getName()}.groovy" +println "**************************************************************" + +// Get the DBB_HOME location +def dbbHome = EnvVars.getHome() +if (props.verbose) println "** DBB_HOME = ${dbbHome}" + +// Create full build command +def fullBuildCommand = [] +fullBuildCommand << "${dbbHome}/bin/groovyz" +fullBuildCommand << "${props.zAppBuildDir}/build.groovy" +fullBuildCommand << "--workspace ${props.workspace}" +fullBuildCommand << "--application ${props.app}" +fullBuildCommand << (props.outDir ? "--outDir ${props.outDir}" : "--outDir ${props.zAppBuildDir}/out") +fullBuildCommand << "--hlq ${props.hlq}" +fullBuildCommand << "--logEncoding UTF-8" +fullBuildCommand << (props.url ? "--url ${props.url}" : "") +fullBuildCommand << (props.id ? "--id ${props.id}" : "") +fullBuildCommand << (props.pw ? "--pw ${props.pw}" : "") +fullBuildCommand << (props.pwFile ? "--pwFile ${props.pwFile}" : "") +fullBuildCommand << (props.verbose ? "--verbose" : "") +fullBuildCommand << (props.propFiles ? "--propFiles ${props.propFiles}" : "") +fullBuildCommand << "--fullBuild" + +@Field def assertionList = [] + +def changedFiles = props.fullBuild_truncation_changedFiles.split(',') +PropertyMappings fullBuild_truncation_errorMsgs = new PropertyMappings('fullBuild_truncation_errorMsg') + +try { + + // Create full build command to set baseline + testUtils.runBaselineBuild() + + // test setup + changedFiles.each { changedFile -> + + println "\n** Running build test for changed file $changedFile" + + // update changed file in Git repo test branch + testUtils.copyAndCommit(changedFile) + + // run build + println "** Executing ${fullBuildCommand.join(" ")}" + outputStream = new StringBuffer() + process = ['bash', '-c', fullBuildCommand.join(" ")].execute() + process.waitForProcessOutput(outputStream, System.err) + + // validate build results + validateBuild(changedFile, fullBuild_truncation_errorMsgs, outputStream) + } +} +catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' +} +finally { + // report failures + if (assertionList.size()>0) { + println "\n***" + println "**START OF FULL BUILD TRUNCATION ERROR TEST RESULTS**\n" + println "*FAILED FULL BUILD TRUNCATION ERROR TEST RESULTS*\n" + assertionList + println "\n**END OF FULL BUILD TRUNCATION ERROR TEST RESULTS**" + println "***" + } + + // reset test branch + testUtils.resetTestBranch() + + // cleanup datasets + testUtils.cleanUpDatasets(props.fullBuild_datasetsToCleanUp) +} + + +//************************************************************* +// Method Definitions +//************************************************************* + +def validateBuild(String changedFile, PropertyMappings fullBuild_truncation_errorMsgs, StringBuffer outputStream) { + + println "** Validating full build truncation results" + def expectedlogMsg = fullBuild_truncation_errorMsgs.getValue(changedFile) + + try{ + // Validate that error state build is written to log + assert outputStream.contains("Build State : ERROR") : "*! FULL BUILD TRUNCATION ERROR FOR $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + // Validate that truncation message is written to log + assert outputStream.contains(expectedlogMsg) : "*! FULL BUILD TRUNCATION ERROR for $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + // Validate that Finalization process is executed + assert outputStream.contains("********* Finalization of the build process *****") : "*! FULL BUILD TRUNCATION ERROR FOR $changedFile\nOUTPUT STREAM:\n$outputStream\n" + + println "**" + println "** FULL BUILD TRUNCATION ERROR TEST : PASSED FOR $changedFile **" + println "**" + } + catch(AssertionError e) { + def result = e.getMessage() + assertionList << result; + props.testsSucceeded = 'false' + } +} diff --git a/utilities/BuildUtilities.groovy b/utilities/BuildUtilities.groovy index 46e2885c1..66f0b883d 100644 --- a/utilities/BuildUtilities.groovy +++ b/utilities/BuildUtilities.groovy @@ -42,10 +42,10 @@ def assertBuildProperties(String requiredProps) { */ def createFullBuildList() { Set buildSet = new HashSet() - + // PropertyMappings PropertyMappings githashBuildableFilesMap = new PropertyMappings("githashBuildableFilesMap") - + // create the list of build directories List srcDirs = [] if (props.applicationSrcDirs) @@ -55,13 +55,13 @@ def createFullBuildList() { dir = getAbsolutePath(dir) Set fileSet =getFileSet(dir, true, '**/*.*', props.excludeFileList) buildSet.addAll(fileSet) - + // capture abbreviated gitHash for all buildable files String abbrevHash = gitUtils.getCurrentGitHash(dir, true) buildSet.forEach { buildableFile -> githashBuildableFilesMap.addFilePattern(abbrevHash, buildableFile) } - + } return buildSet @@ -100,19 +100,24 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap // only copy the build file once if (!copiedFileCache.contains(buildFile)) { copiedFileCache.add(buildFile) - new CopyToPDS().file(new File(getAbsolutePath(buildFile))) - .dataset(srcPDS) - .member(CopyToPDS.createMemberName(buildFile)) - .execute() + try { + new CopyToPDS().file(new File(getAbsolutePath(buildFile))) + .dataset(srcPDS) + .member(CopyToPDS.createMemberName(buildFile)) + .execute() + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of buildFile ${buildFile} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) + } } - + if (dependencyDatasetMapping && props.userBuildDependencyFile && props.userBuild) { if (props.verbose) println "*** User Build Dependency File Detected. Skipping DBB Dependency Resolution." // userBuildDependencyFile present (passed from the IDE) // Skip dependency resolution, extract dependencies from userBuildDependencyFile, and copy directly dataset // Load property mapping containing the map of targetPDS and dependencyfile PropertyMappings dependenciesDatasetMapping = new PropertyMappings(dependencyDatasetMapping) - + // parse JSON and validate fields of userBuildDependencyFile def depFileData = validateDependencyFile(buildFile, props.userBuildDependencyFile) @@ -145,26 +150,31 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null // get index of last '.' in file path to extract the file extension def extIndex = dependencyLoc.lastIndexOf('.') - if( zunitFileExtension && !zunitFileExtension.isEmpty() && (dependencyLoc.substring(extIndex).contains(zunitFileExtension))){ - new CopyToPDS().file(new File(dependencyLoc)) - .copyMode(CopyMode.BINARY) - .dataset(dependencyPDS) - .member(memberName) - .execute() - } - else - { - new CopyToPDS().file(new File(dependencyLoc)) - .dataset(dependencyPDS) - .member(memberName) - .execute() + try { + if( zunitFileExtension && !zunitFileExtension.isEmpty() && (dependencyLoc.substring(extIndex).contains(zunitFileExtension))){ + new CopyToPDS().file(new File(dependencyLoc)) + .copyMode(CopyMode.BINARY) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + else + { + new CopyToPDS().file(new File(dependencyLoc)) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of dependency ${dependencyLoc} failed with an exception ${e.getMessage()}." + throw new BuildException(errorMsg) } } } } else if (dependencyDatasetMapping && dependencyResolver) { // resolve the logical dependencies to physical files to copy to data sets - + List physicalDependencies = resolveDependencies(dependencyResolver, buildFile) if (props.verbose) println "*** Physical dependencies for $buildFile:" @@ -177,18 +187,18 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap printPhysicalDependencies(physicalDependencies) } } - + physicalDependencies.each { physicalDependency -> // Write Physical Dependency details to log on verbose, not on formatConsoleOutput if (props.verbose && !(props.formatConsoleOutput && props.formatConsoleOutput.toBoolean())) println physicalDependency - + if (physicalDependency.isResolved()) { // obtain target dataset based on Mappings // Order : // 1. langprefix_dependenciesAlternativeLibraryNameMapping based on the library setting recognized by DBB (COBOL and PLI) - // 2. langprefix_dependenciesDatasetMapping as a manual overwrite to determine an alternative library used in the default dd concatentation - String dependencyPDS + // 2. langprefix_dependenciesDatasetMapping as a manual overwrite to determine an alternative library used in the default dd concatentation + String dependencyPDS if (!physicalDependency.getLibrary().equals("SYSLIB") && dependenciesAlternativeLibraryNameMapping) { dependencyPDS = props.getProperty(parseJSONStringToMap(dependenciesAlternativeLibraryNameMapping).get(physicalDependency.getLibrary())) } @@ -207,19 +217,23 @@ def copySourceFiles(String buildFile, String srcPDS, String dependencyDatasetMap String memberName = CopyToPDS.createMemberName(physicalDependency.getFile()) //retrieve zUnitFileExtension plbck zunitFileExtension = (props.zunit_playbackFileExtension) ? props.zunit_playbackFileExtension : null - - if( zunitFileExtension && !zunitFileExtension.isEmpty() && ((physicalDependency.getFile().substring(physicalDependency.getFile().indexOf("."))).contains(zunitFileExtension))){ - new CopyToPDS().file(new File(physicalDependencyLoc)) - .copyMode(CopyMode.BINARY) - .dataset(dependencyPDS) - .member(memberName) - .execute() - } else - { - new CopyToPDS().file(new File(physicalDependencyLoc)) - .dataset(dependencyPDS) - .member(memberName) - .execute() + try { + if( zunitFileExtension && !zunitFileExtension.isEmpty() && ((physicalDependency.getFile().substring(physicalDependency.getFile().indexOf("."))).contains(zunitFileExtension))){ + new CopyToPDS().file(new File(physicalDependencyLoc)) + .copyMode(CopyMode.BINARY) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } else + { + new CopyToPDS().file(new File(physicalDependencyLoc)) + .dataset(dependencyPDS) + .member(memberName) + .execute() + } + } catch (BuildException e) { // Catch potential exceptions like file truncation + String errorMsg = "*! (BuildUtilities.copySourceFiles) CopyToPDS of dependency ${physicalDependencyLoc} failed with an exception \n ${e.getMessage()}." + throw new BuildException(errorMsg) } } } else { @@ -321,9 +335,9 @@ def updateBuildResult(Map args) { */ def createLogicalFile(SearchPathDependencyResolver spDependencyResolver, String buildFile) { - + LogicalFile logicalFile - + if (props.resolveSubsystems && props.resolveSubsystems.toBoolean()) { // include resolved dependencies to define file flags of logicalFile logicalFile = spDependencyResolver.resolveSubsystems(buildFile,props.workspace) @@ -410,26 +424,38 @@ def isMQ(LogicalFile logicalFile) { return isMQ } +/* + * isIMS - tests to see if the program is a DL/I program. If the logical file is false, then + * check to see if there is a file property. + */ +def isIMS(LogicalFile logicalFile) { + isIMS = false + String imsFlag = props.getFileProperty('isIMS', logicalFile.getFile()) + if (imsFlag) + isIMS = imsFlag.toBoolean() + return isIMS +} + /* * getMqStubInstruction - * returns include defintion for mq sub program for link edit */ def getMqStubInstruction(LogicalFile logicalFile) { String mqStubInstruction - + if (isMQ(logicalFile)) { // https://www.ibm.com/docs/en/ibm-mq/9.3?topic=files-mq-zos-stub-programs if (isCICS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQCSTUB)\n" - } else if (isDLI(logicalFile)) { + } else if (isDLI(logicalFile) || isIMS(logicalFile)) { mqStubInstruction = " INCLUDE SYSLIB(CSQQSTUB)\n" } else { mqStubInstruction = " INCLUDE SYSLIB(CSQBSTUB)\n" } } else { - println("*! (BuildUtilities.getMqStubInstruction) MQ file attribute for ${logicalFile.getFile()} is false.") + println("*! (BuildUtilities.getMqStubInstruction) MQ file attribute for ${logicalFile.getFile()} is false.") } - + return mqStubInstruction } @@ -542,7 +568,8 @@ def retrieveLastBuildResult(){ } /* - * returns the deployType for a logicalFile depending on the isCICS, isDLI setting + * returns the deployType for a logicalFile depending on the + * isCICS, isIMS and isDLI setting */ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFile){ // getDefault @@ -555,6 +582,9 @@ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFil if(isCICS(logicalFile)){ // if CICS String cicsDeployType = props.getFileProperty("${langQualifier}_deployTypeCICS", buildFile) if (cicsDeployType != null) deployType = cicsDeployType + } else if (isIMS(logicalFile)){ + String imsDeployType = props.getFileProperty("${langQualifier}_deployTypeIMS", buildFile) + if (imsDeployType != null) deployType = imsDeployType } else if (isDLI(logicalFile)){ String dliDeployType = props.getFileProperty("${langQualifier}_deployTypeDLI", buildFile) if (dliDeployType != null) deployType = dliDeployType @@ -570,10 +600,10 @@ def getDeployType(String langQualifier, String buildFile, LogicalFile logicalFil * Creates a Generic PropertyRecord with the provided db2 information in bind.properties */ def generateDb2InfoRecord(String buildFile){ - + // New Generic Property Record PropertiesRecord db2BindInfo = new PropertiesRecord("db2BindInfo:${buildFile}") - + // Link to buildFile db2BindInfo.addProperty("file", buildFile) @@ -586,8 +616,8 @@ def generateDb2InfoRecord(String buildFile){ if (bindPropertyValue != null ) db2BindInfo.addProperty("${db2Prop}",bindPropertyValue) } } - - return db2BindInfo + + return db2BindInfo } /* @@ -596,12 +626,20 @@ def generateDb2InfoRecord(String buildFile){ */ def validateDependencyFile(String buildFile, String depFilePath) { String[] allowedEncodings = ["UTF-8", "IBM-1047"] - String[] reqDepFileProps = ["fileName", "isCICS", "isSQL", "isDLI", "isMQ", "dependencies", "schemaVersion"] + String[] reqDepFileProps = [ + "fileName", + "isCICS", + "isSQL", + "isDLI", + "isMQ", + "dependencies", + "schemaVersion" + ] depFilePath = getAbsolutePath(depFilePath) // Load dependency file and verify existance File depFile = new File(depFilePath) assert depFile.exists() : "*! Dependency file not found: ${depFile.getAbsolutePath()}" - + // Parse the JSON file String encoding = retrieveHFSFileEncoding(depFile) // Determine the encoding from filetag JsonSlurper slurper = new JsonSlurper().setType(JsonParserType.INDEX_OVERLAY) // Use INDEX_OVERLAY, fastest parser @@ -616,7 +654,7 @@ def validateDependencyFile(String buildFile, String depFilePath) { depFileData = slurper.parse(depFile) // Assume default encoding for system } if (props.verbose) println new JsonBuilder(depFileData).toPrettyString() // Pretty print if verbose - + // Validate JSON structure reqDepFileProps.each { depFileProp -> assert depFileData."${depFileProp}" != null : "*! Missing required dependency file field '$depFileProp'" @@ -650,8 +688,8 @@ def assertDbbBuildToolkitVersion(String currentVersion, String requiredVersion){ assert (label as int) >= ((requiredVersionList[i]) as int) : "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." if (label > requiredVersionList[i]) foundValidVersion = true } - - } + + } } catch(AssertionError e) { println "Current DBB Toolkit Version $currentVersion does not meet the minimum required version $requiredVersion. EXIT." @@ -666,12 +704,12 @@ def assertDbbBuildToolkitVersion(String currentVersion, String requiredVersion){ */ def retrieveHFSFileEncoding(File file) { FileAttribute.Stat stat = FileAttribute.getStat(file.getAbsolutePath()) - FileAttribute.Tag tag = stat.getTag() + FileAttribute.Tag tag = stat.getTag() int i = 0 if (tag != null) { - char x = tag.getCodeCharacterSetID() - i = (int) x + char x = tag.getCodeCharacterSetID() + i = (int) x } switch(i) { @@ -679,7 +717,7 @@ def retrieveHFSFileEncoding(File file) { case 1208: return "UTF-8" default: return "IBM-${i}" } - + } /* @@ -689,7 +727,7 @@ def retrieveHFSFileEncoding(File file) { // def printResolutionRules(List rules) { // println("*** Configured resulution rules:") - + // // Print header of table // println(" " + "Library".padRight(10) + "Category".padRight(12) + "SourceDir/File".padRight(50) + "Directory".padRight(36) + "Collection".padRight(24) + "Archive".padRight(20)) // println(" " + " ".padLeft(10,"-") + " ".padLeft(12,"-") + " ".padLeft(50,"-") + " ".padLeft(36,"-") + " ".padLeft(24,"-") + " ".padLeft(20,"-")) @@ -812,38 +850,38 @@ def matches(String file, List pathMatchers) { */ def generateIdentifyStatement(String buildFile, String dsProperty) { - def String identifyStmt + def String identifyStmt int maxRecordLength = dsProperty.toLowerCase().contains("library") ? 80 : 40 - - if((props.mergeBuild || props.impactBuild || props.fullBuild) && MetadataStoreFactory.getMetadataStore() != null) { - - String member = CopyToPDS.createMemberName(buildFile) - String shortGitHash = getShortGitHash(buildFile) - - if (shortGitHash != null) { - - String identifyString = props.application + "/" + shortGitHash - // IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc') - identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" - if (identifyString.length() > maxRecordLength) { - String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt" - println(errorMsg) - props.error = "true" - updateBuildResult(errorMsg:errorMsg) - return null - } else { - return identifyStmt - } + + if((props.mergeBuild || props.impactBuild || props.fullBuild) && MetadataStoreFactory.getMetadataStore() != null) { + + String member = CopyToPDS.createMemberName(buildFile) + String shortGitHash = getShortGitHash(buildFile) + + if (shortGitHash != null) { + + String identifyString = props.application + "/" + shortGitHash + // IDENTIFY EPSCSMRT('MortgageApplication/abcabcabc') + identifyStmt = " " + "IDENTIFY ${member}(\'$identifyString\')" + if (identifyString.length() > maxRecordLength) { + String errorMsg = "*!* BuildUtilities.generateIdentifyStatement() - Identify string exceeds $maxRecordLength chars: identifyStmt=$identifyStmt" + println(errorMsg) + props.error = "true" + updateBuildResult(errorMsg:errorMsg) + return null } else { - println("*!* BuildUtilities.generateIdentifyStatement() - Could not obtain abbreviated git hash for $buildFile") - return null - } + return identifyStmt + } + } else { + println("*!* BuildUtilities.generateIdentifyStatement() - Could not obtain abbreviated git hash for $buildFile") + return null + } - } else { - return null - } - } + } else { + return null + } +} /** * method to print the logicalFile attributes (CICS, SQL, DLI, MQ) of a scanned file @@ -860,16 +898,16 @@ def generateIdentifyStatement(String buildFile, String dsProperty) { * This is implementing * https://github.com/IBM/dbb-zappbuild/issues/339 * -*/ + */ def printLogicalFileAttributes(LogicalFile logicalFile) { String cicsFlag = (logicalFile.isCICS() == isCICS(logicalFile)) ? "${logicalFile.isCICS()}" : "${isCICS(logicalFile)}*" String sqlFlag = (logicalFile.isSQL() == isSQL(logicalFile)) ? "${logicalFile.isSQL()}" : "${isSQL(logicalFile)}*" String dliFlag = (logicalFile.isDLI() == isDLI(logicalFile)) ? "${logicalFile.isDLI()}" : "${isDLI(logicalFile)}*" String mqFlag = (logicalFile.isMQ() == isMQ(logicalFile)) ? "${logicalFile.isMQ()}" : "${isMQ(logicalFile)}*" - + println "Program attributes: CICS=$cicsFlag, SQL=$sqlFlag, DLI=$dliFlag, MQ=$mqFlag" - + } /** @@ -902,3 +940,4 @@ def isGeneratedzUnitTestCaseProgram(String buildFile) { return false } + diff --git a/utilities/ReportingUtilities.groovy b/utilities/ReportingUtilities.groovy index 9aedde5f9..08b972a7a 100644 --- a/utilities/ReportingUtilities.groovy +++ b/utilities/ReportingUtilities.groovy @@ -194,13 +194,15 @@ def calculateConcurrentChanges(Set buildSet) { // initialize patterns List gitRefMatcherPatterns = createMatcherPatterns(props.reportConcurrentChangesGitBranchReferencePatterns) - + // obtain all current remote branches // TODO: Handle / Exclude branches from other repositories Set remoteBranches = new HashSet() props.applicationSrcDirs.split(",").each { dir -> dir = buildUtils.getAbsolutePath(dir) - remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) + if (gitUtils.isGitDir(dir)) { + remoteBranches.addAll(gitUtils.getRemoteGitBranches(dir)) + } } // Run analysis for each remoteBranch, which matches the configured criteria diff --git a/version.properties b/version.properties index 48f92f7d9..b53aacdd5 100644 --- a/version.properties +++ b/version.properties @@ -1 +1,13 @@ -version=3.5.0 \ No newline at end of file +# internal zAppbuild version number that is bumped up for each new release +# delivered via the public github repository +# https://github.com/IBM/dbb-zappbuild. +# Helps to understand from which version a copy was taken. +zappbuild_baseVersion=3.6.0 + +# use this property to add you own version suffix to indicate +# contributed enhancements and your own revisions +zappbuild_customVersion=BASE + +# zappbuild version string that is +# combining the base and custom version properties +zappbuild_version=${zappbuild_baseVersion}-${zappbuild_customVersion} \ No newline at end of file