characters without diacritics should be recomposed

oyejorge · oyejorge · commit 219f436033be · 2022-10-12T09:35:50.000-06:00
diff --git a/lib/index.mjs b/lib/index.mjs
@@ -46,7 +46,7 @@ const latin_condensed = {
 	"f": "ꝼƒ",
 	"g": "ǥɠꞡᵹꝿɢ",
 	"h": "ħⱨⱶɥ",
-	"i": "i̇ɨı",
+	"i": "ɨı",
 	"j": "ɉȷ",
 	"k": "ƙⱪꝁꝃꝅꞣ",
 	"l": "łƚɫⱡꝉꝇꞁɭ",
@@ -60,7 +60,7 @@ const latin_condensed = {
 	"p": "ƥᵽꝑꝓꝕρ",
 	"q": "ꝗꝙɋ",
 	"r": "ɍɽꝛꞧꞃ",
-	"s": "ßȿꞩꞅʂṧṩ",
+	"s": "ßȿꞩꞅʂ",
 	"t": "ŧƭʈⱦꞇ",
 	"th": "þ",
 	"tz": "ꜩ",
@@ -107,44 +107,37 @@ export const initialize = (_code_points) => {
 export const normalize = (str,form='NFKD') => str.normalize(form);
 
 
-/**
- * Compatibility Decomposition without reordering string
- * calling str.normalize('NFKD') on \u{594}\u{595}\u{596} becomes \u{596}\u{594}\u{595}
- * @param {string} str
- */
-export const decompose = (str) =>{
-
-	if( str.match(/[\u0f71-\u0f81]/) ){
-		return toArray(str).reduce(
-			/**
-			 * @param {string} result
-			 * @param {string} char
-			 */
-			(result, char) =>{
-				return result + normalize(char)
-			},
-			''
-		);
-	}
-
-	return normalize(str);
-}
-
-
-
 
 /**
- * Remove accents
+ * Remove accents without reordering string
+ * calling str.normalize('NFKD') on \u{594}\u{595}\u{596} becomes \u{596}\u{594}\u{595}
  * via https://github.com/krisk/Fuse/issues/133#issuecomment-318692703
  * @param {string} str
  * @return {string}
  */
 export const asciifold = (str) => {
-	return decompose(str)
+
+	return toArray(str).reduce(
+		/**
+		 * @param {string} result
+		 * @param {string} char
+		 */
+		(result, char) =>{
+			return result + _asciifold(char)
+		},
+		''
+	);
+};
+
+export const _asciifold = (str) => {
+	str = normalize(str)
 		.toLowerCase()
 		.replace(convert_pat,(/** @type {string} */ char) => {
 			return latin_convert[char] || '';
-		});
+		})
+
+	//return str;
+	return normalize(str,'NFC')
 };
 
 
diff --git a/test/matching.js b/test/matching.js
@@ -100,20 +100,45 @@ describe('Matching', function(){
 		testCombos(combos);
 	});
 
+	// https://github.com/orchidjs/tom-select/discussions/491
+	it('full-width characters',()=>{
+		let combos = ['アプ'];
+		testCombos(combos);
+	});
+
 	it('diacritic list',()=>{
 
+		let missing = {};
+
 		for( let folded in diacritics ){
 			let chars = Array.from(diacritics[folded]);
 
 			for( let char of chars){
 				folded		= folded.toLowerCase();
 				char		= char.toLowerCase();
 
-				let combos	= [folded,char];
-				testCombos(combos);
+				let regex = regExp(folded);
+				let r		= regex.test(char);
+				if( r ){
+					continue;
+				}
+
+				let temp = missing[folded] || '';
+
+				if( temp.indexOf(char) >= 0 ){
+					continue;
+				}
+
+				missing[folded] = temp + char;
+
 			}
 		}
 
+		if( Object.keys(missing).length > 0 ){
+			console.log('missing characters',JSON.stringify(missing,null,"\t"));
+			assert.equal(false, true, 'should not be missing characters. see console output');
+		}
+
 	});
 
 	it('Should match all code points individually',()=>{
