Add locale support for sort operator (#9400)

* Locale support for sort operator

* Add checkLanguageCode util function

* Update docs & add language code validation

* Replace multiple isDescending with reverse

* Revert "Replace multiple isDescending with reverse"

This reverts commit 793177b8bc.

* Simplify sortTiddler with Intl.Collator

* Add change notes

* Remove comment

* Update makeCompareFunction to support locale

* Update checkLanguageCode

* Add locale support for sortsub

* Add locale support for sort filter run prefix

* Revert "Add locale support for sort filter run prefix"

This reverts commit 9479a156d7.

* Remove checkLanguageCode
Since filters are able to catch errors now

* Update locale example

* Revert " Add locale support for sortsub"

This reverts commit 4a617188fc.

* Revert "Update makeCompareFunction to support locale"

This reverts commit 0ebca08036.

* Update docs

* Update change note

core/modules/filters/sort.js

@@ -14,31 +14,31 @@ Export our filter function
 */
 exports.sort = function(source,operator,options) {
 	var results = prepare_results(source);
-	options.wiki.sortTiddlers(results,operator.operand || "title",operator.prefix === "!",false,false);
+	options.wiki.sortTiddlers(results,operator.operands[0] || "title",operator.prefix === "!",false,false,undefined,operator.operands[1]);
 	return results;
 };
 
 exports.nsort = function(source,operator,options) {
 	var results = prepare_results(source);
-	options.wiki.sortTiddlers(results,operator.operand || "title",operator.prefix === "!",false,true);
+	options.wiki.sortTiddlers(results,operator.operands[0] || "title",operator.prefix === "!",false,true,undefined,operator.operands[1]);
 	return results;
 };
 
 exports.sortan = function(source, operator, options) {
 	var results = prepare_results(source);
-	options.wiki.sortTiddlers(results, operator.operand || "title", operator.prefix === "!",false,false,true);
+	options.wiki.sortTiddlers(results, operator.operands[0] || "title", operator.prefix === "!",false,false,true,operator.operands[1]);
 	return results;
 };
 
 exports.sortcs = function(source,operator,options) {
 	var results = prepare_results(source);
-	options.wiki.sortTiddlers(results,operator.operand || "title",operator.prefix === "!",true,false);
+	options.wiki.sortTiddlers(results,operator.operands[0] || "title",operator.prefix === "!",true,false,undefined,operator.operands[1]);
 	return results;
 };
 
 exports.nsortcs = function(source,operator,options) {
 	var results = prepare_results(source);
-	options.wiki.sortTiddlers(results,operator.operand || "title",operator.prefix === "!",true,true);
+	options.wiki.sortTiddlers(results,operator.operands[0] || "title",operator.prefix === "!",true,true,undefined,operator.operands[1]);
 	return results;
 };
 

core/modules/wiki.js

@@ -369,31 +369,16 @@ Sort an array of tiddler titles by a specified field
 	isDescending: true if the sort should be descending
 	isCaseSensitive: true if the sort should consider upper and lower case letters to be different
 */
-exports.sortTiddlers = function(titles,sortField,isDescending,isCaseSensitive,isNumeric,isAlphaNumeric) {
+exports.sortTiddlers = function(titles,sortField,isDescending,isCaseSensitive,isNumeric,isAlphaNumeric,locale) {
 	var self = this;
 	if(sortField === "title") {
 		if(!isNumeric && !isAlphaNumeric) {
-			if(isCaseSensitive) {
-				if(isDescending) {
-					titles.sort(function(a,b) {
-						return b.localeCompare(a);
-					});
-				} else {
-					titles.sort(function(a,b) {
-						return a.localeCompare(b);
-					});
-				}	
+			const sorter = new Intl.Collator(locale, { sensitivity: isCaseSensitive ? "variant" : "accent" });
+			if(isDescending) {
+				titles.sort((a,b) => sorter.compare(b, a));
 			} else {
-				if(isDescending) {
-					titles.sort(function(a,b) {
-						return b.toLowerCase().localeCompare(a.toLowerCase());
-					});
-				} else {
-					titles.sort(function(a,b) {
-						return a.toLowerCase().localeCompare(b.toLowerCase());
-					});
-				}	
-			}
+				titles.sort((a,b) => sorter.compare(a, b));
+			}	
 		} else {
 			titles.sort(function(a,b) {
 				var x,y;
@@ -414,14 +399,8 @@ exports.sortTiddlers = function(titles,sortField,isDescending,isCaseSensitive,is
 						}
 					}
 				}
-				if(isAlphaNumeric) {
-					return isDescending ? b.localeCompare(a,undefined,{numeric: true,sensitivity: "base"}) : a.localeCompare(b,undefined,{numeric: true,sensitivity: "base"});
-				}
-				if(!isCaseSensitive) {
-					a = a.toLowerCase();
-					b = b.toLowerCase();
-				}
-				return isDescending ? b.localeCompare(a) : a.localeCompare(b);
+				const sorter = new Intl.Collator(locale, { numeric: isAlphaNumeric, sensitivity: isAlphaNumeric ? "base" : isCaseSensitive ? "variant" :  "accent" });
+				return isDescending ? sorter.compare(b, a) : sorter.compare(a, b);
 			});
 		}
 	} else {
@@ -463,14 +442,8 @@ exports.sortTiddlers = function(titles,sortField,isDescending,isCaseSensitive,is
 			}
 			a = String(a);
 			b = String(b);
-			if(isAlphaNumeric) {
-				return isDescending ? b.localeCompare(a,undefined,{numeric: true,sensitivity: "base"}) : a.localeCompare(b,undefined,{numeric: true,sensitivity: "base"});
-			}
-			if(!isCaseSensitive) {
-				a = a.toLowerCase();
-				b = b.toLowerCase();
-			}
-			return isDescending ? b.localeCompare(a) : a.localeCompare(b);
+			const sorter = new Intl.Collator(locale, { numeric: isAlphaNumeric, sensitivity: isAlphaNumeric ? "base" : isCaseSensitive ? "variant" :  "accent" });
+			return isDescending ? sorter.compare(b, a) : sorter.compare(a, b);
 		});
 	}
 };

editions/tw5.com/tiddlers/Locale Example.tid

@@ -0,0 +1,6 @@
+created: 20251001034405510
+list: A a Ä ä Z z O o Õ õ Ö ö Ü ü Y y
+modified: 20251218023544134
+tags: 
+title: Locale Example
+type: text/vnd.tiddlywiki

editions/tw5.com/tiddlers/filters/examples/sort.tid

@@ -1,5 +1,5 @@
 created: 20150124112340000
-modified: 20150124113250000
+modified: 20251218023608468
 tags: [[sort Operator]] [[Operator Examples]]
 title: sort Operator (Examples)
 type: text/vnd.tiddlywiki
@@ -11,3 +11,10 @@ type: text/vnd.tiddlywiki
 <<.operator-example 3 "one two Three four +[sort[]]">>
 <<.operator-example 4 "[prefix[Tiddl]sort[text]]">>
 <<.operator-example 5 "[has[created]sort[created]limit[10]]" "the oldest 10 tiddlers in the wiki">>
+
+! Using a custom locale
+The following examples shows the differences when using the sort operator with default, Swedish and Estonian locale.
+
+<<.operator-example 6 "[list[Locale Example]sort[]]">>
+<<.operator-example 7 "[list[Locale Example]sort[],[sv]]">>
+<<.operator-example 8 "[list[Locale Example]sort[],[et]]">>

editions/tw5.com/tiddlers/filters/nsort.tid

@@ -1,15 +1,15 @@
+caption: nsort
 created: 20140410103123179
-modified: 20150203190051000
+modified: 20251227084602319
+op-input: a [[selection of titles|Title Selection]]
+op-neg-output: the input, likewise sorted into descending order
+op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as numbers
+op-parameter: accept same parameters as the [[sort Operator]]
+op-parameter-name: F
+op-purpose: sort the input by number field
 tags: [[Filter Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]
 title: nsort Operator
 type: text/vnd.tiddlywiki
-caption: nsort
-op-purpose: sort the input by number field
-op-input: a [[selection of titles|Title Selection]]
-op-parameter: the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
-op-parameter-name: F
-op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as numbers
-op-neg-output: the input, likewise sorted into descending order
 
 Non-numeric values are treated as having a higher value than any number, and the difference between capital and lowercase letters is ignored. Compare <<.olink nsortcs>>.
 

editions/tw5.com/tiddlers/filters/nsortcs.tid

@@ -1,10 +1,10 @@
 caption: nsortcs
 created: 20140410103123179
-modified: 20150417125717078
+modified: 20251227084615705
 op-input: a [[selection of titles|Title Selection]]
 op-neg-output: the input, likewise sorted into descending order
 op-output: the input, sorted into ascending order by field <<.place F>>, treating field values as numbers
-op-parameter: the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
+op-parameter: accept same parameters as the [[sort Operator]]
 op-parameter-name: F
 op-purpose: sort the input titles by number field, treating upper and lower case as different
 tags: [[Filter Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]

editions/tw5.com/tiddlers/filters/sort.tid

@@ -1,16 +1,22 @@
+caption: sort
 created: 20140410103123179
-modified: 20150203191228000
+modified: 20251227084644651
+op-input: a [[selection of titles|Title Selection]]
+op-neg-output: the input, likewise sorted into descending order
+op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as text
+op-parameter: the <<.op sort>> operator accepts 1 or 2 parameters, see below for details
+op-parameter-name: F
+op-purpose: sort the input by text field
 tags: [[Filter Operators]] [[Common Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]
 title: sort Operator
 type: text/vnd.tiddlywiki
-caption: sort
-op-purpose: sort the input by text field
-op-input: a [[selection of titles|Title Selection]]
-op-parameter: the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
-op-parameter-name: F
-op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as text
-op-neg-output: the input, likewise sorted into descending order
 
 The difference between capital and lowercase letters is ignored. Compare <<.olink sortcs>>.
 
+```
+[sort[<field>],[<locale>]]
+```
+* ''field'' : the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
+* ''locale'': (optional). A string with a [[BCP 47 language tag|https://developer.mozilla.org/en-US/docs/Glossary/BCP_47_language_tag]]
+
 <<.operator-examples "sort">>

editions/tw5.com/tiddlers/filters/sortan.tid

@@ -1,10 +1,10 @@
 caption: sortan
 created: 20180222071605739
-modified: 20180223012553446
+modified: 20251227084439804
 op-input: a [[selection of titles|Title Selection]]
 op-neg-output: the input, likewise sorted into descending order
 op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as alphanumerics
-op-parameter: the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
+op-parameter: accept same parameters as the [[sort Operator]]
 op-parameter-name: F
 op-purpose: sort the input by text field considering them as alphanumerics
 tags: [[Filter Operators]] [[Common Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]

editions/tw5.com/tiddlers/filters/sortcs.tid

@@ -1,10 +1,10 @@
 caption: sortcs
 created: 20140410103123179
-modified: 20150417125704503
+modified: 20251227084416522
 op-input: a [[selection of titles|Title Selection]]
 op-neg-output: the input, likewise sorted into descending order
 op-output: the input, sorted into ascending order by field <<.field F>>, treating field values as text
-op-parameter: the name of a [[field|TiddlerFields]], defaulting to <<.field title>>
+op-parameter: accept same parameters as the [[sort Operator]]
 op-parameter-name: F
 op-purpose: sort the input by text field, treating upper and lower case as different
 tags: [[Filter Operators]] [[Field Operators]] [[Order Operators]] [[Negatable Operators]]

editions/tw5.com/tiddlers/releasenotes/5.4.0/#9400 - collator.tid

@@ -0,0 +1,9 @@
+title: $:/changenotes/5.4.0/#9400/impacts/collator
+changenote: $:/changenotes/5.4.0/#9400
+created: 20251114102355243
+modified: 20251114102355243
+tags: $:/tags/ImpactNote
+description: Tiddlywiki no longer works on browsers that doesn't support [[Intl.Collator|https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Intl/Collator]]
+impact-type: compatibility-break
+
+

editions/tw5.com/tiddlers/releasenotes/5.4.0/#9400.tid

@@ -0,0 +1,10 @@
+title: $:/changenotes/5.4.0/#9400
+description: Add locale support for sort operators
+release: 5.4.0
+tags: $:/tags/ChangeNote
+change-type: feature
+change-category: filters
+github-links: https://github.com/TiddlyWiki/TiddlyWiki5/pull/9400
+github-contributors: Leilei332
+
+Add a new parameter for `sort`, `nsort`, `sortan`, `sortcs` and `nsortcs` to support sorting with a custom locale.