feat: serialize AST node back to wikitext string (#8258)

* refactor: extract a new $tw.wiki.getParser

* feat: allow $tw.utils.getParseTreeText to render other rules' text

* feat: two example getText handler

* Revert "feat: allow $tw.utils.getParseTreeText to render other rules' text"

This reverts commit 8a12498fa9.

* refactor: keep original getParseTreeText not touched

* refactor: use serialize in rules

* refactor: $tw.utils.extend({},options) -> options || {}

* Update codeinline.js

* Create test-wikitext-serialize.js

* DEBUG: only run my tests for development, remove before PR merge

* lint: if

* feat: add rule: 'parseBlock' metadata

* feat: handle tailing \n that may be missing

* feat: allow recursive

* feat: generate more rule and tests

* feat: generate more rule and tests

* fix: remove pragma:true, otherwise following text will become children of it

* fix: condition manually

Deekseek is silly

* fix: some test

* fix: some test

* feat: $tw.utils.serializeAttribute

* fix: use "" for string param

* feat: list

* refactor: ' -> "

* fix: parsemode don't have node

* fix: render invisible comment and parsemode as data element

* feat: add void: true, in ast node to prevent render

* feat: use void widget, so methods always return a widget

* feat: ast to use new widget type void

* test: add rule: 'parseBlock' and isRuleEnd: true

* lint: quote

* Update widget.js

* fix: void node need to handle its children

* Update test-wikitext-parser.js

* lint: quote

* Update void.js

* Update test-wikitext-parser.js

* fix: macrodef with comment (void node) not working

* lint: ' -> "

* feat: add to styleblock

* feat: styleblock

* feat: styleinline

* Update table.js

* lint: useless comments

* feat: transcludeblock

* refactor: reuse block on inline when possible

* feat: use void node to carry important info for typedblock

* feat: run all tests

* lint: useless ai generated comments

* Update conditional.js to not include space

* Update test-wikitext-serialize.js

* Update conditional.js

* refactor: move tiddlers to /data

* refactor: no need for new $tw.Wiki()

* lint: double quote

* refactor: lowercase the parseblock rule name

* fix: Wiki parser initialize blockRuleClasses only when first new an instance

* feat: restore inline macro def

* fix: macro in widget param

* fix: positional attribute in macro call

* fix: table space and horizrule block new line

* feat: make sure block rule all have \n\n for visiblity

* lint: function param

* fix: empty list item

* feat: add \n\n based on isBlock, if could also be inline

* fix: conditional without elseif

* refactor: use isBlock in macrodef to know inline or block

* fix: link may not have attribute and children

* DEBUG: render result and diff below body only on browser

DEBUG: render result below body only on browser

DEBUG: render result below body

DEBUG: fix build

DEBUG: show render result as ViewTemplate

* fix: remove pad space in />

* test: remove pad space in />

* Revert DEBUG: render result and diff below body only on browser

* refactor: fold commentText variable

* refactor: fold long comment

* fix: double quotes for parameter values

* Update void.js

* refactor: move all exports.serialize = function(tree,serialize) { to plugin

* fix: expost listTypes from core, and require it in plugin

* refactor: move serializeWikitextParseTree to plugin and init it

* refactor: move serializeAttribute util also to the plugin

* fix: Delete unused file

* Update macrodef.js

* Update test-wikitext-parser.js

* lint: fix

* Update plugins/tiddlywiki/wikitext-serialize/rules/filteredtranscludeblock.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update core/modules/widgets/void.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update plugins/tiddlywiki/wikitext-serialize/rules/list.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update plugins/tiddlywiki/wikitext-serialize/rules/list.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update plugins/tiddlywiki/wikitext-serialize/rules/styleblock.js

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Remove unused methods from VoidNodeWidget

Deleted render, execute, and refresh methods from VoidNodeWidget as they are no longer needed. The widget now only inherits from the base Widget class and exports the constructor.

* docs: about regex in styleinline.js

* Update parsetree.js

* Update core/modules/widgets/void.js

Co-authored-by: Jeremy Ruston <jeremy@jermolene.com>

* feat: Ensure at least one space after the style/class

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Jeremy Ruston <jeremy@jermolene.com>

core/modules/parsers/wikiparser/rules/commentblock.js

@@ -22,7 +22,7 @@ Note that the syntax for comments is simplified to an opening "<!--" sequence an
 "use strict";
 
 exports.name = "commentblock";
-exports.types = {block:true, pragma:true};
+exports.types = {block: true, pragma: true};
 
 exports.init = function(parser) {
 	this.parser = parser;
@@ -43,9 +43,18 @@ exports.findNextMatch = function(startPos) {
 	return undefined;
 };
 
+
 exports.parse = function() {
 	// Move past the match
 	this.parser.pos = this.endMatchRegExp.lastIndex;
-	// Don't return any elements
-	return [];
+	// Return a node representing the comment that is not rendered
+	var commentStart = this.match.index;
+	var commentEnd = this.endMatch.index + this.endMatch[0].length;
+	return [{
+			type: "void",
+			children: [],
+			text: this.parser.source.slice(commentStart, commentEnd),
+			start: commentStart,
+			end: commentEnd
+	}];
 };

core/modules/parsers/wikiparser/rules/commentinline.js

@@ -40,6 +40,13 @@ exports.findNextMatch = function(startPos) {
 exports.parse = function() {
 	// Move past the match
 	this.parser.pos = this.endMatchRegExp.lastIndex;
-	// Don't return any elements
-	return [];
+	// Return a node representing the inline comment
+	var commentStart = this.match.index;
+	var commentEnd = this.endMatch.index + this.endMatch[0].length;
+	return [{
+			type: "void",
+			text: this.parser.source.slice(commentStart, commentEnd),
+			start: commentStart,
+			end: commentEnd
+	}];
 };

core/modules/parsers/wikiparser/rules/extlink.js

@@ -28,11 +28,11 @@ exports.init = function(parser) {
 
 exports.parse = function() {
 	// Move past the match
-    var start = this.parser.pos;
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Create the link unless it is suppressed
 	if(this.match[0].substr(0,1) === "~") {
-		return [{type: "text", text: this.match[0].substr(1)}];
+		return [{type: "text", text: this.match[0].substr(1), start: start, end: this.parser.pos}];
 	} else {
 		return [{
 			type: "element",

core/modules/parsers/wikiparser/rules/fnprocdef.js

@@ -6,15 +6,15 @@ module-type: wikirule
 Wiki pragma rule for function, procedure and widget definitions
 
 ```
-\function name(param:defaultvalue,param2:defaultvalue)
+\function name(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 
-\procedure name(param:defaultvalue,param2:defaultvalue)
+\procedure name(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 
-\widget $mywidget(param:defaultvalue,param2:defaultvalue)
+\widget $mywidget(param:"defaultvalue", param2:"defaultvalue")
 definition text
 \end
 ```

core/modules/parsers/wikiparser/rules/hardlinebreaks.js

@@ -50,6 +50,8 @@ exports.parse = function() {
 			}
 		}
 	} while(match && !match[1]);
-	// Return the nodes
+	// Mark first and last node, and return the nodes
+	if(tree[0]) tree[0].isRuleStart = true;
+	if(tree[tree.length-1]) tree[tree.length-1].isRuleEnd = true;
 	return tree;
 };

core/modules/parsers/wikiparser/rules/html.js

@@ -41,7 +41,7 @@ Parse the most recent match
 exports.parse = function() {
 	// Retrieve the most recent match so that recursive calls don't overwrite it
 	var tag = this.nextTag;
-	if (!tag.isSelfClosing) {
+	if(!tag.isSelfClosing) {
 		tag.openTagStart = tag.start;
 		tag.openTagEnd = tag.end;
 	}
@@ -63,22 +63,22 @@ exports.parse = function() {
 		}
 		tag.end = this.parser.pos;
 		tag.closeTagEnd = tag.end;
-		if (tag.closeTagEnd === tag.openTagEnd || this.parser.source[tag.closeTagEnd - 1] !== '>') {
+		if(tag.closeTagEnd === tag.openTagEnd || this.parser.source[tag.closeTagEnd - 1] !== ">") {
 			tag.closeTagStart = tag.end;
 		} else {
 			tag.closeTagStart = tag.closeTagEnd - 2;
 			var closeTagMinPos = tag.children.length > 0 ? tag.children[tag.children.length-1].end : tag.openTagEnd;
-			if (!Number.isSafeInteger(closeTagMinPos)) closeTagMinPos = tag.openTagEnd;
-			while (tag.closeTagStart >= closeTagMinPos) {
+			if(!Number.isSafeInteger(closeTagMinPos)) closeTagMinPos = tag.openTagEnd;
+			while(tag.closeTagStart >= closeTagMinPos) {
 				var char = this.parser.source[tag.closeTagStart];
-				if (char === '>') {
+				if(char === ">") {
 					tag.closeTagStart = -1;
 					break;
 				}
-				if (char === '<') break;
+				if(char === "<") break;
 				tag.closeTagStart -= 1;
 			}
-			if (tag.closeTagStart < closeTagMinPos) {
+			if(tag.closeTagStart < closeTagMinPos) {
 				tag.closeTagStart = tag.end;
 			}
 		}

core/modules/parsers/wikiparser/rules/list.js

@@ -59,6 +59,7 @@ var listTypes = {
 	":": {listTag: "dl", itemTag: "dd"},
 	">": {listTag: "blockquote", itemTag: "div"}
 };
+exports.listTypes = listTypes;
 
 /*
 Parse the most recent match

core/modules/parsers/wikiparser/rules/macrocallblock.js

@@ -29,7 +29,7 @@ exports.findNextMatch = function(startPos) {
 			var c = this.parser.source.charAt(nextCall.end);
 			// Ensure EOL after parsed macro
 			// If we didn't need to support IE, we'd just use /(?:\r?\n|$)/ym
-			if ((c === "") || (c === "\n") || ((c === "\r") && this.parser.source.charAt(nextCall.end+1) === "\n")) {
+			if((c === "") || (c === "\n") || ((c === "\r") && this.parser.source.charAt(nextCall.end+1) === "\n")) {
 				this.nextCall = nextCall;
 				return nextStart;
 			}

core/modules/parsers/wikiparser/rules/macrocallinline.js

@@ -42,3 +42,5 @@ exports.parse = function() {
 	this.parser.pos = call.end;
 	return [call];
 };
+
+

core/modules/parsers/wikiparser/rules/macrodef.js

@@ -52,9 +52,10 @@ exports.parse = function() {
 		}
 	}
 	// Is the remainder of the \define line blank after the parameter close paren?
-	var reEnd;
+	var reEnd,isBlock = true;
 	if(this.match[3]) {
 		// If so, it is a multiline definition and the end of the body is marked with \end
+		isBlock = false;
 		reEnd = new RegExp("((?:^|\\r?\\n)[^\\S\\n\\r]*\\\\end[^\\S\\n\\r]*(?:" + $tw.utils.escapeRegExp(this.match[1]) + ")?\\s*?(?:$|\\r?\\n))","mg");
 	} else {
 		// Otherwise, the end of the definition is marked by the end of the line
@@ -79,7 +80,8 @@ exports.parse = function() {
 		attributes: {},
 		children: [],
 		params: params,
-		isMacroDefinition: true
+		isMacroDefinition: true,
+		isBlock: isBlock && !!endMatch
 	}];
 	$tw.utils.addAttributeToParseTreeNode(parseTreeNodes[0],"name",this.match[1]);
 	$tw.utils.addAttributeToParseTreeNode(parseTreeNodes[0],"value",text);

core/modules/parsers/wikiparser/rules/parsermode.js

@@ -31,6 +31,7 @@ Parse the most recent match
 */
 exports.parse = function() {
 	// Move past the pragma invocation
+	var start = this.parser.pos;
 	this.parser.pos = this.matchRegExp.lastIndex;
 	// Parse whitespace delimited tokens terminated by a line break
 	var reMatch = /[^\S\n]*(\S+)|(\r?\n)/mg,
@@ -58,6 +59,11 @@ exports.parse = function() {
 			this.parser.parseAsInline = true;
 		}
 	}
-	// No parse tree nodes to return
-	return [];
+	return [{
+		type: "void",
+		children: [],
+		parseAsInline: this.parser.parseAsInline,
+		start: start,
+		end: this.parser.pos
+	}];
 };

core/modules/parsers/wikiparser/rules/prettyextlink.js

@@ -113,3 +113,5 @@ exports.parseLink = function(source,pos) {
 	node.end = closePos + 2;
 	return node;
 };
+
+

core/modules/parsers/wikiparser/rules/prettylink.js

@@ -32,7 +32,7 @@ exports.parse = function() {
 	var text = this.match[1],
 		link = this.match[2] || text,
 		textEndPos = this.parser.source.indexOf("|", start);
-	if (textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
+	if(textEndPos < 0 || textEndPos > this.matchRegExp.lastIndex) {
 		textEndPos = this.matchRegExp.lastIndex - 2;
 	}
 	var linkStart = this.match[2] ? (start + this.match[1].length + 1) : start;

core/modules/parsers/wikiparser/rules/rules.js

@@ -54,6 +54,13 @@ exports.parse = function() {
 	if(tokens.length > 0) {
 		this.parser.amendRules(tokens[0],tokens.slice(1));
 	}
-	// No parse tree nodes to return
-	return [];
+	// No widget to render, return void node.
+	return [{
+		type: "void",
+		attributes: {
+			action: {type: "string", value: tokens[0]},
+			rules: {type: "string", value: tokens.slice(1).join(" ")}
+		},
+		children: []
+	}];
 };

core/modules/parsers/wikiparser/rules/styleblock.js

@@ -64,5 +64,8 @@ exports.parse = function() {
 			$tw.utils.addAttributeToParseTreeNode(tree[t],"style",styles.join(""));
 		}
 	}
-	return tree;
+	return [{
+		type: "void",
+		children: tree
+	}]
 };

core/modules/parsers/wikiparser/rules/styleinline.js

@@ -21,7 +21,7 @@ exports.types = {inline: true};
 
 exports.init = function(parser) {
 	this.parser = parser;
-	// Regexp to match
+	// Regexp to match /@@(styles)?\s*(\.class\s+)?/
 	this.matchRegExp = /@@((?:[^\.\r\n\s:]+:[^\r\n;]+;)+)?(\.(?:[^\r\n\s]+)\s+)?/mg;
 };
 

core/modules/parsers/wikiparser/rules/typedblock.js

@@ -60,22 +60,37 @@ exports.parse = function() {
 	var parser = this.parser.wiki.parseText(parseType,text,{defaultType: "text/plain"});
 	// If there's no render type, just return the parse tree
 	if(!renderType) {
-		return parser.tree;
+		return  [{
+			type: "void",
+			children: $tw.utils.isArray(parser.tree) ? parser.tree : [parser.tree],
+			parseType: parseType,
+			renderType: renderType,
+			text: text,
+			start: start,
+			end: this.parser.pos
+		}];
 	} else {
 		// Otherwise, render to the rendertype and return in a <PRE> tag
 		var widgetNode = this.parser.wiki.makeWidget(parser),
 			container = $tw.fakeDocument.createElement("div");
 		widgetNode.render(container,null);
-		text = renderType === "text/html" ? container.innerHTML : container.textContent;
+		var renderResult = renderType === "text/html" ? container.innerHTML : container.textContent;
+		// Use void node to carry important info for typedblock
 		return [{
-			type: "element",
-			tag: "pre",
+			type: "void",
 			children: [{
-				type: "text",
-				text: text,
-				start: start,
-				end: this.parser.pos
-			}]
+				type: "element",
+				tag: "pre",
+				children: [{
+					type: "text",
+					text: renderResult,
+				}]
+			}],
+			parseType: parseType,
+			renderType: renderType,
+			text: text,
+			start: start,
+			end: this.parser.pos
 		}];
 	}
 };

core/modules/parsers/wikiparser/wikiparser.js

@@ -215,8 +215,8 @@ WikiParser.prototype.parsePragmas = function() {
 		var subTree = nextMatch.rule.parse();
 		if(subTree.length > 0) {
 			// Set the start and end positions of the pragma rule if
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 			$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 			// Quick hack; we only cope with a single parse tree node being returned, which is true at the moment
 			currentTreeBranch.push.apply(currentTreeBranch,subTree);
@@ -245,9 +245,9 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 		var start = this.pos;
 		var subTree = nextMatch.rule.parse();
 		// Set the start and end positions of the first and last blocks if they're not already set
-		if (subTree.length > 0) {
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+		if(subTree.length > 0) {
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 		}
 		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 		return subTree;
@@ -256,7 +256,7 @@ WikiParser.prototype.parseBlock = function(terminatorRegExpString) {
 	var start = this.pos;
 	var children = this.parseInlineRun(terminatorRegExp);
 	var end = this.pos;
-	return [{type: "element", tag: "p", children: children, start: start, end: end }];
+	return [{type: "element", tag: "p", children: children, start: start, end: end, rule: "parseblock" }];
 };
 
 /*
@@ -350,10 +350,10 @@ WikiParser.prototype.parseInlineRunUnterminated = function(options) {
 		var start = this.pos;
 		var subTree = nextMatch.rule.parse();
 		// Set the start and end positions of the first and last child if they're not already set
-		if (subTree.length > 0) {
+		if(subTree.length > 0) {
 			// Set the start and end positions of the first and last child if they're not already set
-			if (subTree[0].start === undefined) subTree[0].start = start;
-			if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree[0].start === undefined) subTree[0].start = start;
+			if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 		}
 		$tw.utils.each(subTree, function (node) { node.rule = nextMatch.rule.name; });
 		tree.push.apply(tree,subTree);
@@ -410,9 +410,9 @@ WikiParser.prototype.parseInlineRunTerminatedExtended = function(terminatorRegEx
 			var start = this.pos;
 			var subTree = inlineRuleMatch.rule.parse();
 			// Set the start and end positions of the first and last child if they're not already set
-			if (subTree.length > 0) {
-				if (subTree[0].start === undefined) subTree[0].start = start;
-				if (subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
+			if(subTree.length > 0) {
+				if(subTree[0].start === undefined) subTree[0].start = start;
+				if(subTree[subTree.length - 1].end === undefined) subTree[subTree.length - 1].end = this.pos;
 			}
 			$tw.utils.each(subTree, function (node) { node.rule = inlineRuleMatch.rule.name; });
 			tree.push.apply(tree,subTree);

core/modules/utils/parsetree.js

@@ -119,3 +119,19 @@ exports.getParseTreeText = function getParseTreeText(tree) {
 	}
 	return output.join("");
 };
+
+exports.getParser = function(type,options) {
+	options = options || {};
+	// Select a parser
+	var Parser = $tw.Wiki.parsers[type];
+	if(!Parser && $tw.utils.getFileExtensionInfo(type)) {
+		Parser = $tw.Wiki.parsers[$tw.utils.getFileExtensionInfo(type).type];
+	}
+	if(!Parser) {
+		Parser = $tw.Wiki.parsers[options.defaultType || "text/vnd.tiddlywiki"];
+	}
+	if(!Parser) {
+		return null;
+	}
+	return Parser;
+};

core/modules/widgets/importvariables.js

@@ -49,7 +49,8 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 		var parser = widgetPointer.wiki.parseTiddler(title,{parseAsInline:true, configTrimWhiteSpace:false});
 		if(parser) {
 			var parseTreeNode = parser.tree[0];
-			while(parseTreeNode && ["setvariable","set","parameters"].indexOf(parseTreeNode.type) !== -1) {
+			// process AST nodes generated by pragma rules.
+			while(parseTreeNode && ["setvariable","set","parameters","void"].indexOf(parseTreeNode.type) !== -1) {
 				var node = {
 					type: "set",
 					attributes: parseTreeNode.attributes,
@@ -82,7 +83,7 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 						// this widget. If it needs to refresh,
 						// it'll do so along with the the whole
 						// importvariable tree.
-						if (widgetPointer != this) {
+						if(widgetPointer != this) {
 							widgetPointer.makeChildWidgets = function(){};
 						}
 						widgetPointer = widgetPointer.children[0];
@@ -93,7 +94,7 @@ ImportVariablesWidget.prototype.execute = function(tiddlerList) {
 		} 
 	});
 
-	if (widgetPointer != this) {
+	if(widgetPointer != this) {
 		widgetPointer.parseTreeNode.children = this.parseTreeNode.children;
 	} else {
 		widgetPointer.makeChildWidgets();

core/modules/widgets/void.js

@@ -0,0 +1,23 @@
+/*\
+title: $:/core/modules/widgets/void.js
+type: application/javascript
+module-type: widget
+
+Void widget that corresponds to pragma and comment AST nodes, etc. It does not render itself but renders all its children.
+
+\*/
+
+"use strict";
+
+var Widget = require("$:/core/modules/widgets/widget.js").widget;
+
+var VoidNodeWidget = function(parseTreeNode,options) {
+	this.initialise(parseTreeNode,options);
+};
+
+/*
+Inherit from the base widget class
+*/
+VoidNodeWidget.prototype = new Widget();
+
+exports.void = VoidNodeWidget;

core/modules/wiki.js

@@ -1059,17 +1059,7 @@ Options include:
 exports.parseText = function(type,text,options) {
 	text = text || "";
 	options = options || {};
-	// Select a parser
-	var Parser = $tw.Wiki.parsers[type];
-	if(!Parser && $tw.utils.getFileExtensionInfo(type)) {
-		Parser = $tw.Wiki.parsers[$tw.utils.getFileExtensionInfo(type).type];
-	}
-	if(!Parser) {
-		Parser = $tw.Wiki.parsers[options.defaultType || "text/vnd.tiddlywiki"];
-	}
-	if(!Parser) {
-		return null;
-	}
+	var Parser = $tw.utils.getParser(type,options)
 	// Return the parser instance
 	return new Parser(type,text,{
 		parseAsInline: options.parseAsInline,
@@ -1083,7 +1073,7 @@ exports.parseText = function(type,text,options) {
 Parse a tiddler according to its MIME type
 */
 exports.parseTiddler = function(title,options) {
-	options = $tw.utils.extend({},options);
+	options = options || {};
 	var cacheType = options.parseAsInline ? "inlineParseTree" : "blockParseTree",
 		tiddler = this.getTiddler(title),
 		self = this;