Unittest b64 decoder, make mroe robust to handle unicode properly

microsoft · weswigham · Mar 26, 2018 · Feb 15, 2018 · Feb 17, 2018 · Mar 16, 2018
commit f2c8d3fa82beb4a248b9431ef89f732e39facd02
diff --git a/Jakefile.js b/Jakefile.js
@@ -115,6 +115,7 @@ var harnessCoreSources = [
 });
 
 var harnessSources = harnessCoreSources.concat([
+    "base64.ts",
     "incrementalParser.ts",
     "jsDocParsing.ts",
     "services/colorization.ts",

diff --git a/src/compiler/utilities.ts b/src/compiler/utilities.ts
@@ -3398,7 +3398,7 @@ namespace ts {
         for (let i = 0; i < length; i++) {
             const charCode = input.charCodeAt(i);
 
-            // handel utf8
+            // handle utf8
             if (charCode < 0x80) {
                 output.push(charCode);
             }
@@ -3463,12 +3463,45 @@ namespace ts {
         return result;
     }
 
+    function getStringFromExpandedCharCodes(codes: number[]): string {
+        let output = "";
+        let i = 0;
+        const length = codes.length;
+        while (i < length) {
+            const charCode = codes[i];
+
+            if (charCode < 0x80) {
+                output += String.fromCharCode(charCode);
+                i++;
+            }
+            else if ((charCode & 0B11000000) === 0B11000000) {
+                let value = charCode & 0B00111111;
+                i++;
+                let nextCode: number = codes[i];
+                while ((nextCode & 0B11000000) === 0B10000000) {
+                    value = (value << 6) | (nextCode & 0B00111111);
+                    i++;
+                    nextCode = codes[i];
+                }
+                // `value` may be greater than 10FFFF (the maximum unicode codepoint) - JS will just make this into an invalid character for us
+                output += String.fromCharCode(value);
+            }
+            else {
+                // We don't want to kill the process when decoding fails (due to a following char byte not
+                // following a leading char), so we just print the (bad) value
+                output += String.fromCharCode(charCode);
+                i++;
+            }
+        }
+        return output;
+    }
+
     export function base64decode(host: { base64decode?(input: string): string }, input: string): string {
         if (host.base64decode) {
             return host.base64decode(input);
         }
-        let result = "";
         const length = input.length;
+        let expandedCharCodes: number[] = [];
         let i = 0;
         while (i < length) {
             // Stop decoding once padding characters are present
@@ -3481,22 +3514,22 @@ namespace ts {
             const ch3 = base64Digits.indexOf(input[i + 2]);
             const ch4 = base64Digits.indexOf(input[i + 3]);
 
-            const code1 = (ch1 & 0B00111111 << 2) | (ch2 & 0B00000011);
-            const code2 = (ch2 & 0B00001111 << 4) | (ch3 & 0B00001111);
-            const code3 = (ch3 & 0B00000011 << 6) | (ch4 & 0B00111111);
+            const code1 = ((ch1 & 0B00111111) << 2) | ((ch2 >> 4) & 0B00000011);
+            const code2 = ((ch2 & 0B00001111) << 4) | ((ch3 >> 2) & 0B00001111);
+            const code3 = ((ch3 & 0B00000011) << 6) | (ch4 & 0B00111111);
 
             if (code2 === 0 && ch3 !== 0) { // code2 decoded to zero, but ch3 was padding - elide code2 and code3
-                result += String.fromCharCode(code1);
+                expandedCharCodes.push(code1);
             }
             else if (code3 === 0 && ch4 !== 0) { // code3 decoded to zero, but ch4 was padding, elide code3
-                result += `${String.fromCharCode(code1)}${String.fromCharCode(code2)}`;
+                expandedCharCodes.push(code1, code2);
             }
             else {
-                result += `${String.fromCharCode(code1)}${String.fromCharCode(code2)}${String.fromCharCode(code3)}`;
+                expandedCharCodes.push(code1, code2, code3);
             }
             i += 4;
         }
-        return result;
+        return getStringFromExpandedCharCodes(expandedCharCodes);
     }
 
     const carriageReturnLineFeed = "\r\n";

diff --git a/src/harness/tsconfig.json b/src/harness/tsconfig.json
@@ -95,6 +95,7 @@
         "../server/session.ts",
         "../server/client.ts",
         "../server/editorServices.ts",
+        "./unittests/base64.ts",
         "./unittests/incrementalParser.ts",
         "./unittests/jsDocParsing.ts",
         "./unittests/services/colorization.ts",

diff --git a/src/harness/unittests/base64.ts b/src/harness/unittests/base64.ts
@@ -0,0 +1,23 @@
+/// <reference path="../harness.ts" />
+namespace ts {
+    describe("base64", () => {
+        describe("base64decode", () => {
+            it("can decode input strings correctly without needing a host implementation", () => {
+                const tests = [
+                    // "a",
+                    // "this is a test",
+                    // " !\"#$ %&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~",
+                    "日本語",
+                    "🐱",
+                    "\x00\x01",
+                    "\t\n\r\\\"\'\u0062",
+                    "====",
+                    "",
+                ];
+                for (const test of tests) {
+                    assert.equal(base64decode({}, convertToBase64(test)), test);
+                }
+            });
+        });
+    });
+}