[FEATURE] Add JSDoc build functionalities

.eslintignore

@@ -1,5 +1,8 @@
 # /node_modules/* and /bower_components/* ignored by default
 
+# Exclude shared resources that can't (yet) follow our conventions
+lib/processors/jsdoc/lib
+
 # Exclude coverage folder
 coverage/
 

index.js

@@ -13,6 +13,9 @@ module.exports = {
 		flexChangesBundler: require("./lib/processors/bundlers/flexChangesBundler"),
 		manifestBundler: require("./lib/processors/bundlers/manifestBundler"),
 		moduleBundler: require("./lib/processors/bundlers/moduleBundler"),
+		apiIndexGenerator: require("./lib/processors/jsdoc/apiIndexGenerator"),
+		jsdocGenerator: require("./lib/processors/jsdoc/jsdocGenerator"),
+		sdkTransformer: require("./lib/processors/jsdoc/sdkTransformer"),
 		bootstrapHtmlTransformer: require("./lib/processors/bootstrapHtmlTransformer"),
 		debugFileCreator: require("./lib/processors/debugFileCreator"),
 		resourceCopier: require("./lib/processors/resourceCopier"),
@@ -35,6 +38,9 @@ module.exports = {
 		generateBundle: require("./lib/tasks/bundlers/generateBundle"),
 		buildThemes: require("./lib/tasks/buildThemes"),
 		createDebugFiles: require("./lib/tasks/createDebugFiles"),
+		executeJsdocSdkTransformation: require("./lib/tasks/jsdoc/executeJsdocSdkTransformation"),
+		generateApiIndex: require("./lib/tasks/jsdoc/generateApiIndex"),
+		generateJsdoc: require("./lib/tasks/jsdoc/generateJsdoc"),
 		generateVersionInfo: require("./lib/tasks/generateVersionInfo"),
 		replaceCopyright: require("./lib/tasks/replaceCopyright"),
 		replaceVersion: require("./lib/tasks/replaceVersion"),

lib/builder/builder.js

@@ -1,5 +1,6 @@
 const log = require("@ui5/logger").getGroupLogger("builder:builder");
 const resourceFactory = require("@ui5/fs").resourceFactory;
+const MemAdapter = require("@ui5/fs").adapters.Memory;
 const typeRepository = require("../types/typeRepository");
 const taskRepository = require("../tasks/taskRepository");
 
@@ -36,20 +37,24 @@ function getElapsedTime(startTime) {
  * @param {Object} parameters
  * @param {boolean} parameters.dev Sets development mode, which only runs essential tasks
  * @param {boolean} parameters.selfContained True if a the build should be self-contained or false for prelead build bundles
+ * @param {boolean} parameters.jsdoc True if a JSDoc build should be executed
  * @param {Array} parameters.includedTasks Task list to be included from build
  * @param {Array} parameters.excludedTasks Task list to be excluded from build
  * @returns {Array} Return a task list for the builder
  */
-function composeTaskList({dev, selfContained, includedTasks, excludedTasks}) {
+function composeTaskList({dev, selfContained, jsdoc, includedTasks, excludedTasks}) {
 	let selectedTasks = Object.keys(definedTasks).reduce((list, key) => {
 		list[key] = true;
 		return list;
 	}, {});
 
-	// Exclude tasks: manifestBundler
+	// Exclude non default tasks
 	selectedTasks.generateManifestBundle = false;
 	selectedTasks.generateStandaloneAppBundle = false;
 	selectedTasks.transformBootstrapHtml = false;
+	selectedTasks.generateJsdoc = false;
+	selectedTasks.executeJsdocSdkTransformation = false;
+	selectedTasks.generateApiIndex = false;
 
 	if (selfContained) {
 		// No preloads, bundle only
@@ -59,6 +64,27 @@ function composeTaskList({dev, selfContained, includedTasks, excludedTasks}) {
 		selectedTasks.generateLibraryPreload = false;
 	}
 
+	if (jsdoc) {
+		// Include JSDoc tasks
+		selectedTasks.generateJsdoc = true;
+		selectedTasks.executeJsdocSdkTransformation = true;
+		selectedTasks.generateApiIndex = true;
+
+		// Include theme build as required for SDK
+		selectedTasks.buildThemes = true;
+
+		// Exclude all tasks not relevant to JSDoc generation
+		selectedTasks.replaceCopyright = false;
+		selectedTasks.replaceVersion = false;
+		selectedTasks.generateComponentPreload = false;
+		selectedTasks.generateLibraryPreload = false;
+		selectedTasks.generateLibraryManifest = false;
+		selectedTasks.createDebugFiles = false;
+		selectedTasks.uglify = false;
+		selectedTasks.generateFlexChangesBundle = false;
+		selectedTasks.generateManifestBundle = false;
+	}
+
 	// Only run essential tasks in development mode, it is not desired to run time consuming tasks during development.
 	if (dev) {
 		// Overwrite all other tasks with noop promise
@@ -123,22 +149,23 @@ module.exports = {
 	 * @param {boolean} [parameters.buildDependencies=false] Decides whether project dependencies are built as well
 	 * @param {boolean} [parameters.dev=false] Decides whether a development build should be activated (skips non-essential and time-intensive tasks)
 	 * @param {boolean} [parameters.selfContained=false] Flag to activate self contained build
+	 * @param {boolean} [parameters.jsdoc=false] Flag to activate JSDoc build
 	 * @param {Array} [parameters.includedTasks=[]] List of tasks to be included
 	 * @param {Array} [parameters.excludedTasks=[]] List of tasks to be excluded. If the wildcard '*' is provided, only the included tasks will be executed.
 	 * @param {Array} [parameters.devExcludeProject=[]] List of projects to be excluded from development build
 	 * @returns {Promise} Promise resolving to <code>undefined</code> once build has finished
 	 */
 	build({
 		tree, destPath,
-		buildDependencies = false, dev = false, selfContained = false,
+		buildDependencies = false, dev = false, selfContained = false, jsdoc = false,
 		includedTasks = [], excludedTasks = [], devExcludeProject = []
 	}) {
 		const startTime = process.hrtime();
 		log.info(`Building project ${tree.metadata.name}` + (buildDependencies ? "" : " not") +
 			" including dependencies..." + (dev ? " [dev mode]" : ""));
 		log.verbose(`Building to ${destPath}...`);
 
-		const selectedTasks = composeTaskList({dev, selfContained, includedTasks, excludedTasks});
+		const selectedTasks = composeTaskList({dev, selfContained, jsdoc, includedTasks, excludedTasks});
 
 		const fsTarget = resourceFactory.createAdapter({
 			fsBasePath: destPath,
@@ -147,6 +174,7 @@ module.exports = {
 
 
 		const projects = {}; // Unique project index to prevent building the same project multiple times
+		const projectWriters = {}; // Collection of memory adapters of already built libraries
 
 		const projectCountMarker = {};
 		function projectCount(project, count = 0) {
@@ -187,17 +215,29 @@ module.exports = {
 
 				const projectType = typeRepository.getType(project.type);
 				const resourceCollections = resourceFactory.createCollectionsForTree(project, {
-					useNamespaces: true
+					useNamespaces: true,
+					virtualReaders: projectWriters
+				});
+
+				const writer = new MemAdapter({
+					virBasePath: "/"
 				});
+				// Store project writer as virtual reader for parent projects
+				//	so they can access the build results of this project
+				projectWriters[project.metadata.name] = writer;
 
+				// TODO: Add getter for writer of DuplexColection
 				const workspace = resourceFactory.createWorkspace({
+					virBasePath: "/",
+					writer,
 					reader: resourceCollections.source,
 					name: project.metadata.name
 				});
 
 				if (dev && devExcludeProject.indexOf(project.metadata.name) !== -1) {
 					projectTasks = composeTaskList({dev: false, selfContained, includedTasks, excludedTasks});
 				}
+
 				return projectType.build({
 					resourceCollections: {
 						workspace,

lib/processors/jsdoc/apiIndexGenerator.js

@@ -0,0 +1,51 @@
+const resourceFactory = require("@ui5/fs").resourceFactory;
+const createIndex = require("./lib/create-api-index");
+
+/**
+ * Compiles API index resources from all <code>api.json</code> resources available in the given test resources directory
+ * as created by the [sdkTransformer]{@link module:@ui5/builder.processors.sdkTransformer} processor.
+ * The resulting index resources (e.g. <code>api-index.json</code>,  <code>api-index-deprecated.json</code>,
+ * <code>api-index-experimental.json</code> and <code>api-index-since.json</code>) are mainly to be used in the SDK.
+ *
+ * @public
+ * @alias module:@ui5/builder.processors.apiIndexGenerator
+ * @param {Object} parameters Parameters
+ * @param {string} parameters.versionInfoPath Path to <code>sap-ui-version.json</code> resource
+ * @param {string} parameters.testResourcesRootPath Path to <code>/test-resources</code> root directory in the
+ *														given fs
+ * @param {string} parameters.targetApiIndexPath Path to create the generated API index JSON resource for
+ * @param {string} parameters.targetApiIndexDeprecatedPath Path to create the generated API index "deprecated" JSON
+ *															resource for
+ * @param {string} parameters.targetApiIndexExperimentalPath Path to create the generated API index "experimental" JSON
+ *																resource for
+ * @param {string} parameters.targetApiIndexSincePath Path to create the generated API index "since" JSON resource for
+ * @param {fs|module:@ui5/fs.fsInterface} parameters.fs Node fs or
+ * 				custom [fs interface]{@link module:resources/module:@ui5/fs.fsInterface} to use
+ * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with created resources <code>api-index.json</code>,
+ * <code>api-index-deprecated.json</code>, <code>api-index-experimental.json</code> and
+ * <code>api-index-since.json</code> (names depend on the supplied paths)
+ */
+const apiIndexGenerator = async function({
+	versionInfoPath, testResourcesRootPath, targetApiIndexPath, targetApiIndexDeprecatedPath,
+	targetApiIndexExperimentalPath, targetApiIndexSincePath, fs
+} = {}) {
+	if (!versionInfoPath || !testResourcesRootPath || !targetApiIndexPath || !targetApiIndexDeprecatedPath ||
+			!targetApiIndexExperimentalPath || !targetApiIndexSincePath || !fs) {
+		throw new Error("[apiIndexGenerator]: One or more mandatory parameters not provided");
+	}
+
+	const resourceMap = await createIndex(versionInfoPath, testResourcesRootPath, targetApiIndexPath,
+		targetApiIndexDeprecatedPath, targetApiIndexExperimentalPath, targetApiIndexSincePath, {
+			fs,
+			returnOutputFiles: true
+		});
+
+	return Object.keys(resourceMap).map((resPath) => {
+		return resourceFactory.createResource({
+			path: resPath,
+			string: resourceMap[resPath]
+		});
+	});
+};
+
+module.exports = apiIndexGenerator;

lib/processors/jsdoc/jsdocGenerator.js

@@ -0,0 +1,164 @@
+const spawn = require("child_process").spawn;
+const fs = require("graceful-fs");
+const path = require("path");
+const {promisify} = require("util");
+const writeFile = promisify(fs.writeFile);
+const {resourceFactory} = require("@ui5/fs");
+
+/**
+ * JSDoc generator
+ *
+ * @public
+ * @alias module:@ui5/builder.processors.jsdocGenerator
+ * @param {Object} parameters Parameters
+ * @param {string} parameters.sourcePath Path of the source files to be processed
+ * @param {string} parameters.targetPath Path to write any output files
+ * @param {string} parameters.tmpPath Path to write temporary and debug files
+ * @param {Object} parameters.options Options
+ * @param {string} parameters.options.projectName Project name
+ * @param {string} parameters.options.namespace Namespace to build (e.g. <code>some/project/name</code>)
+ * @param {string} parameters.options.version Project version
+ * @param {Array} [parameters.options.variants=["apijson"]] JSDoc variants to be built
+ * @returns {Promise<module:@ui5/fs.Resource[]>} Promise resolving with newly created resources
+ */
+const jsdocGenerator = async function({sourcePath, targetPath, tmpPath, options} = {}) {
+	if (!sourcePath || !targetPath || !tmpPath || !options.projectName || !options.namespace || !options.version) {
+		throw new Error("[jsdocGenerator]: One or more mandatory parameters not provided");
+	}
+
+	if (!options.variants || options.variants.length === 0) {
+		options.variants = ["apijson"];
+	}
+
+	const config = await jsdocGenerator._generateJsdocConfig({
+		targetPath,
+		tmpPath,
+		namespace: options.namespace,
+		projectName: options.projectName,
+		version: options.version,
+		variants: options.variants
+	});
+
+	const configPath = await jsdocGenerator._writeJsdocConfig(tmpPath, config);
+
+	await jsdocGenerator._buildJsdoc({
+		sourcePath,
+		configPath
+	});
+
+	const fsTarget = resourceFactory.createAdapter({
+		fsBasePath: targetPath,
+		virBasePath: "/"
+	});
+
+	// create resources from the output files
+	return Promise.all([
+		fsTarget.byPath(`/test-resources/${options.namespace}/designtime/api.json`)
+		// fsTarget.byPath(`/libraries/${options.projectName}.js`)
+	]).then((res) => res.filter(($)=>$));
+};
+
+
+/**
+ * Generate jsdoc-config.json content
+ *
+ * @private
+ * @param {Object} parameters Parameters
+ * @param {string} parameters.targetPath Path to write any output files
+ * @param {string} parameters.tmpPath Path to write temporary and debug files
+ * @param {string} parameters.projectName Project name
+ * @param {string} parameters.version Project version
+ * @param {Array} parameters.variants JSDoc variants to be built
+ * @returns {string} jsdoc-config.json content string
+ */
+async function generateJsdocConfig({targetPath, tmpPath, namespace, projectName, version, variants}) {
+	// Backlash needs to be escaped as double-backslash
+	// This is not only relevant for win32 paths but also for
+	//	Unix directory names that contain a backslash in their name
+	const backslashRegex = /\\/g;
+
+	// Resolve path to this script to get the path to the JSDoc extensions folder
+	const jsdocPath = path.normalize(__dirname);
+	const pluginPath = path.join(jsdocPath, "lib", "ui5", "plugin.js").replace(backslashRegex, "\\\\");
+	const templatePath = path.join(jsdocPath, "lib", "ui5", "template").replace(backslashRegex, "\\\\");
+	const destinationPath = path.normalize(tmpPath).replace(backslashRegex, "\\\\");
+	const jsapiFilePath = path.join(targetPath, "libraries", projectName + ".js").replace(backslashRegex, "\\\\");
+	const apiJsonFolderPath = path.join(tmpPath, "dependency-apis").replace(backslashRegex, "\\\\");
+	const apiJsonFilePath =
+		path.join(targetPath, "test-resources", path.normalize(namespace), "designtime", "api.json")
+			.replace(backslashRegex, "\\\\");
+
+	const config = `{
+		"plugins": ["${pluginPath}"],
+		"opts": {
+			"recurse": true,
+			"lenient": true,
+			"template": "${templatePath}",
+			"ui5": {
+				"saveSymbols": true
+			},
+			"destination": "${destinationPath}"
+		},
+		"templates": {
+			"ui5": {
+				"variants": ${JSON.stringify(variants)},
+				"version": "${version}",
+				"jsapiFile": "${jsapiFilePath}",
+				"apiJsonFolder": "${apiJsonFolderPath}",
+				"apiJsonFile": "${apiJsonFilePath}"
+			}
+		}
+	}`;
+	return config;
+}
+
+/**
+ * Write jsdoc-config.json to file system
+ *
+ * @private
+ * @param {string} targetDirPath Directory Path to write the jsdoc-config.json file to
+ * @param {string} config jsdoc-config.json content
+ * @returns {string} Full path to the written jsdoc-config.json file
+ */
+async function writeJsdocConfig(targetDirPath, config) {
+	const configPath = path.join(targetDirPath, "jsdoc-config.json");
+	await writeFile(configPath, config);
+	return configPath;
+}
+
+
+/**
+ * Execute JSDoc build by spawning JSDoc as an external process
+ *
+ * @private
+ * @param {Object} parameters Parameters
+ * @param {string} parameters.sourcePath Project resources (input for JSDoc generation)
+ * @param {string} parameters.configPath Full path to jsdoc-config.json file
+ * @returns {Promise<undefined>}
+ */
+async function buildJsdoc({sourcePath, configPath}) {
+	const args = [
+		require.resolve("jsdoc/jsdoc"),
+		"-c",
+		configPath,
+		"--verbose",
+		sourcePath
+	];
+	return new Promise((resolve, reject) => {
+		const child = spawn("node", args, {
+			stdio: ["ignore", "ignore", process.stderr]
+		});
+		child.on("close", function(code) {
+			if (code === 0 || code === 1) {
+				resolve();
+			} else {
+				reject(new Error(`JSDoc child process closed with code ${code}`));
+			}
+		});
+	});
+}
+
+module.exports = jsdocGenerator;
+module.exports._generateJsdocConfig = generateJsdocConfig;
+module.exports._writeJsdocConfig = writeJsdocConfig;
+module.exports._buildJsdoc = buildJsdoc;