krsbx · krsbx · Jun 2, 2022 · Jun 2, 2022
diff --git a/index.html b/index.html
@@ -6,9 +6,9 @@
   <link rel="icon" type="image/svg+xml" href="favicon.svg" />
   <meta name="viewport" content="width=device-width, initial-scale=1" />
   <title>Vite App</title>
-  <script type="module" src="/out/mindar-image.js"></script>
-    <script type="module" src="/out/mindar-image-aframe.js"></script>
-    <script type="module" src="/out/mindar-image-three.js"></script>
+  <script type="module" src="/src/image-target/index.ts"></script>
+  <script type="module" src="/src/image-target/aframe.ts"></script>
+  <script type="module" src="/src/image-target/three.ts"></script>
   <script src="https://aframe.io/releases/1.3.0/aframe.min.js"></script>
   <script type="module" src="/src/main.ts"></script>
 </head>

diff --git a/package.json b/package.json
@@ -15,7 +15,7 @@
     "lint": "eslint --cache --fix src/*",
     "preview": "vite preview",
     "prepare": "husky install",
-    "postinstall": "npx patch-package" 
+    "postinstall": "npx patch-package"
   },
   "devDependencies": {
     "@commitlint/cli": "^17.0.0",
@@ -61,6 +61,7 @@
     "stats-js": "^1.0.1",
     "svd-js": "^1.1.1",
     "three": "^0.140.0",
+    "three-bmfont-text": "^3.0.1",
     "tinyqueue": "^2.0.3"
   },
   "lint-staged": {

diff --git a/src/image-target/aframe/image-system.ts b/src/image-target/aframe/image-system.ts
@@ -34,8 +34,6 @@ AFRAME.registerSystem(AR_COMPONENT_NAME.IMAGE_SYSTEM, {
     this.anchorEntities = [];
   },
 
-  tick: function () {},
-
   setup: function ({
     imageTargetSrc,
     maxTrack,
@@ -80,9 +78,7 @@ AFRAME.registerSystem(AR_COMPONENT_NAME.IMAGE_SYSTEM, {
   _registerEventListener: function () {
     // Subcribe to the targetFound event
     // This event is fired when the target is found
-    this.el.addEventListener(AR_EVENT_NAME.MARKER_FOUND, () => {
-      this.ui.hideScanning();
-    });
+    this.el.addEventListener(AR_EVENT_NAME.MARKER_FOUND, this.ui.hideScanning.bind(this));
 
     // Subcribe to the targetFound event
     // This event is fired when the target is found

diff --git a/src/image-target/compiler.ts b/src/image-target/compiler.ts
@@ -43,7 +43,7 @@ class Compiler {
         processContext.drawImage(img, 0, 0, img.width, img.height);
 
         const processData = processContext.getImageData(0, 0, img.width, img.height);
-        const greyImageData = new Uint8ClampedArray(img.width * img.height);
+        const greyImageData = new Uint8Array(img.width * img.height);
 
         for (let i = 0; i < greyImageData.length; i++) {
           const offset = i * 4;
@@ -56,13 +56,13 @@ class Compiler {
           );
         }
 
-        const targetImage = {
+        const targetImage = Helper.castTo<ImageData>({
           data: greyImageData,
           height: img.height,
           width: img.width,
-        };
+        });
 
-        targetImages.push(<ImageData>targetImage);
+        targetImages.push(targetImage);
       }
 
       // compute matching data: 50% progress
@@ -76,7 +76,7 @@ class Compiler {
         const imageList = buildImageList(targetImage);
         const percentPerAction = percentPerImage / imageList.length;
 
-        const matchingData = await _extractMatchingFeatures(<ImageDataWithScale[]>imageList, () => {
+        const matchingData = await _extractMatchingFeatures(imageList, () => {
           percent += percentPerAction;
           progressCallback(percent);
         });
@@ -90,7 +90,7 @@ class Compiler {
 
       for (let i = 0; i < targetImages.length; i++) {
         const trackingImageList = buildTrackingImageList(targetImages[i]);
-        this.data[i].trackingImageList = <ImageDataWithScale[]>trackingImageList;
+        this.data[i].trackingImageList = trackingImageList;
       }
 
       // compute tracking data with worker: 50% progress
@@ -146,8 +146,8 @@ class Compiler {
     return buffer;
   }
 
-  importData(buffer: string | ArrayBuffer) {
-    const content = msgpack.decode(new Uint8Array(buffer as ArrayBuffer)) as {
+  importData(buffer: ArrayBuffer) {
+    const content = msgpack.decode(new Uint8Array(buffer)) as {
       v: number;
       dataList: IDataList[];
     };

diff --git a/src/image-target/compiler.worker.ts b/src/image-target/compiler.worker.ts
@@ -20,9 +20,9 @@ const _extractTrackingFeatures = (
       width: image.width,
       height: image.height,
       points,
-    };
+    } as ITrackingFeature;
 
-    featureSets.push(<ITrackingFeature>featureSet);
+    featureSets.push(featureSet);
 
     doneCallback(i);
   }
@@ -37,6 +37,7 @@ onmessage = (msg) => {
     case WORKER_EVENT.COMPILE:
       const { targetImages } = data;
       const percentPerImage = 50.0 / targetImages.length;
+
       let percent = 0.0;
       const list: ITrackingFeature[][] = [];
 
@@ -63,4 +64,5 @@ onmessage = (msg) => {
 
 export { _extractTrackingFeatures };
 
-export default {} as typeof Worker & (new () => Worker);
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export default null as any;
diff --git a/src/image-target/controller.ts b/src/image-target/controller.ts
@@ -130,7 +130,7 @@ class Controller {
     });
   }
 
-  addImageTargetsFromBuffer(buffer: string | ArrayBuffer) {
+  addImageTargetsFromBuffer(buffer: ArrayBuffer) {
     const compiler = new Compiler();
     const dataList = compiler.importData(buffer);
 
@@ -196,11 +196,11 @@ class Controller {
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   async _trackAndUpdate(inputT: any, lastModelViewTransform: number[][], targetIndex: number) {
-    const trackResult = this.tracker.track(inputT, lastModelViewTransform, targetIndex);
-
-    if (!trackResult) return null;
-
-    const { worldCoords, screenCoords } = trackResult;
+    const { worldCoords, screenCoords } = this.tracker.track(
+      inputT,
+      lastModelViewTransform,
+      targetIndex
+    );
 
     if (worldCoords.length < 4) return null;
 
@@ -226,7 +226,7 @@ class Controller {
 
   private async _matchImageTarget(nTracking: number, inputT: Tensor) {
     // detect and match only if less then maxTrack
-    if (nTracking > this.maxTrack) return;
+    if (nTracking >= this.maxTrack) return;
 
     const matchingIndexes: number[] = [];
 
@@ -295,7 +295,7 @@ class Controller {
     trackingState.trackCount = 0;
     trackingState.trackMiss += 1;
 
-    if (trackingState.trackMiss < this.missTolerance) return;
+    if (trackingState.trackMiss <= this.missTolerance) return;
 
     trackingState.showing = false;
     trackingState.trackingMatrix = null;
@@ -333,8 +333,8 @@ class Controller {
     while (true) {
       if (!this.processingVideo) break;
 
-      const nTracking = this.trackingStates.reduce((acc, s) => acc + (s.isTracking ? 1 : 0), 0);
       const inputT = this.inputLoader.loadInput(input);
+      const nTracking = this.trackingStates.reduce((acc, s) => acc + (s.isTracking ? 1 : 0), 0);
 
       await this._matchImageTarget(nTracking, inputT);
 
@@ -349,7 +349,7 @@ class Controller {
       }
 
       inputT.dispose();
-      this.onUpdate && this.onUpdate({ type: 'processDone' });
+      this.onUpdate?.({ type: ON_UPDATE_EVENT.DONE });
       await tfNextFrame();
     }
   }

diff --git a/src/image-target/controller.worker.ts b/src/image-target/controller.worker.ts
@@ -57,10 +57,10 @@ const match = (data: IControllerWorker['MATCH']) => {
 };
 
 const trackUpdate = (data: IControllerWorker['TRACK_UPDATE']) => {
-  const { modelViewTransform, worldCoords, screenCoords } = data;
+  const { modelViewTransform: initialModelViewTransform, worldCoords, screenCoords } = data;
 
   const finalModelViewTransform = estimator.refineEstimate({
-    initialModelViewTransform: modelViewTransform,
+    initialModelViewTransform,
     worldCoords,
     screenCoords,
   });
@@ -84,9 +84,8 @@ onmessage = (msg) => {
     case WORKER_EVENT.TRACK_UPDATE:
       trackUpdate(data);
       break;
-    default:
-      break;
   }
 };
 
-export default {} as typeof Worker & (new () => Worker);
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export default null as any;
diff --git a/src/image-target/detector/crop-detector.ts b/src/image-target/detector/crop-detector.ts
@@ -71,8 +71,9 @@ class CropDetector {
       p.y += startY;
     });
 
-    if (this.debugMode && debugExtra)
+    if (this.debugMode) {
       debugExtra.projectedImage = cropInputImageT.arraySync() as number[][];
+    }
 
     cropInputImageT.dispose();
 

diff --git a/src/image-target/detector/detector.ts b/src/image-target/detector/detector.ts
@@ -11,6 +11,7 @@ import {
   TensorInfo,
   DataTypeMap,
 } from '@tensorflow/tfjs';
+import { GPGPUProgram, MathBackendWebGL } from '@tensorflow/tfjs-backend-webgl';
 import {
   FREAK_CONPARISON_COUNT,
   MAX_FEATURES_PER_BUCKET,
@@ -83,12 +84,13 @@ class Detector {
     for (let i = 0; i < this.numOctaves; i++) {
       let image1T: Tensor;
 
-      if (i === 0) image1T = this._applyFilter(inputImageT);
-      else
+      if (i === 0) {
+        image1T = this._applyFilter(inputImageT);
+      } else {
         image1T = this._downsampleBilinear(pyramidImagesT[i - 1][pyramidImagesT[i - 1].length - 1]);
+      }
 
       const image2T = this._applyFilter(image1T);
-
       pyramidImagesT.push([image1T, image2T]);
     }
 
@@ -143,9 +145,7 @@ class Detector {
     if (this.debugMode) {
       debugExtra = {
         pyramidImages: pyramidImagesT.map((ts) => ts.map((t) => t.arraySync())) as number[][],
-        dogPyramidImages: dogPyramidImagesT.map((t) => (t ? t.arraySync() : null)) as
-          | number[]
-          | null[],
+        dogPyramidImages: dogPyramidImagesT.map((t) => (t ? (t.arraySync() as number[]) : null)),
         extremasResults: extremasResultsT.map((t) => t.arraySync()) as number[],
         extremaAngles: extremaAnglesT.arraySync() as number[],
         prunedExtremas: prunedExtremasList,
@@ -313,7 +313,7 @@ class Detector {
         const radius = ORIENTATION_GAUSSIAN_EXPANSION_FACTOR * ORIENTATION_REGION_EXPANSION_FACTOR;
         const radiusCeil = Math.ceil(radius);
 
-        const radialProperties = [];
+        const radialProperties: number[][] = [];
 
         for (let y = -radiusCeil; y <= radiusCeil; y++) {
           for (let x = -radiusCeil; x <= radiusCeil; x++) {
@@ -322,13 +322,15 @@ class Detector {
             if (distanceSquare > radius ** 2) continue;
 
             // may just assign w = 1 will do, this could be over complicated.
-            const _x = distanceSquare * gwScale;
-            // fast expontenial approx
-            const w =
-              (720 + _x * (720 + _x * (360 + _x * (120 + _x * (30 + _x * (6 + _x)))))) *
-              0.0013888888;
-
-            radialProperties.push([y, x, w]);
+            if (distanceSquare <= radius * radius) {
+              const _x = distanceSquare * gwScale;
+              // fast expontenial approx
+              const w =
+                (720 + _x * (720 + _x * (360 + _x * (120 + _x * (30 + _x * (6 + _x)))))) *
+                0.0013888888;
+
+              radialProperties.push([y, x, w]);
+            }
           }
         }
 
@@ -357,7 +359,6 @@ class Detector {
         prunedExtremasT,
         radialPropertiesT,
       ]);
-
       const result2 = this._compileAndRun(program2, [result1]);
 
       return result2;
@@ -411,7 +412,8 @@ class Detector {
         for (let j = 0; j < pixels[i].length; j++) result[i].push([]);
       }
 
-      const localizedExtremas = [];
+      const localizedExtremas: number[][] = [];
+
       for (let i = 0; i < prunedExtremasList.length; i++) {
         localizedExtremas[i] = [
           prunedExtremasList[i][0],
@@ -460,7 +462,7 @@ class Detector {
     const nFeatures = MAX_FEATURES_PER_BUCKET;
 
     if (!this.kernelCaches.applyPrune) {
-      const reductionKernels: any[] = [];
+      const reductionKernels: GPGPUProgram[] = [];
 
       // to reduce to amount of data that need to sync back to CPU by 4 times, we apply this trick:
       // the fact that there is not possible to have consecutive maximum/minimum, we can safe combine 4 pixels into 1
@@ -523,7 +525,9 @@ class Detector {
             const absScore = Math.abs(score);
 
             let tIndex = nFeatures;
-            while (tIndex >= 1 && absScore > curAbsScores[bucket][tIndex - 1]) tIndex -= 1;
+            while (tIndex >= 1 && absScore > curAbsScores[bucket][tIndex - 1]) {
+              tIndex -= 1;
+            }
 
             if (tIndex < nFeatures) {
               for (let t = nFeatures - 1; t >= tIndex + 1; t--) {
@@ -650,14 +654,18 @@ class Detector {
     });
   }
 
-  private _compileAndRun(program: any, inputs: TensorInfo[]) {
-    const outInfo = (tfBackend() as any).compileAndRun(program, inputs);
+  private _compileAndRun(program: GPGPUProgram, inputs: TensorInfo[]) {
+    const outInfo = (tfBackend() as MathBackendWebGL).compileAndRun(program, inputs);
 
     return tfEngine().makeTensorFromTensorInfo(outInfo);
   }
 
-  private _runWebGLProgram(program: any, inputs: TensorInfo[], outputType: keyof DataTypeMap) {
-    const outInfo = (tfBackend() as any).runWebGLProgram(program, inputs, outputType);
+  private _runWebGLProgram(
+    program: GPGPUProgram,
+    inputs: TensorInfo[],
+    outputType: keyof DataTypeMap
+  ) {
+    const outInfo = (tfBackend() as MathBackendWebGL).runWebGLProgram(program, inputs, outputType);
 
     return tfEngine().makeTensorFromTensorInfo(outInfo);
   }

diff --git a/src/image-target/estimation/estimate.ts b/src/image-target/estimation/estimate.ts
@@ -37,8 +37,8 @@ const estimate = ({
   const _KInvH = KInv.mmul(H);
   const KInvH = _KInvH.to1DArray();
 
-  const norm1 = Math.sqrt(KInvH[0] * KInvH[0] + KInvH[3] * KInvH[3] + KInvH[6] * KInvH[6]);
-  const norm2 = Math.sqrt(KInvH[1] * KInvH[1] + KInvH[4] * KInvH[4] + KInvH[7] * KInvH[7]);
+  const norm1 = Math.sqrt(KInvH[0] ** 2 + KInvH[3] ** 2 + KInvH[6] ** 2);
+  const norm2 = Math.sqrt(KInvH[1] ** 2 + KInvH[4] ** 2 + KInvH[7] ** 2);
   const tnorm = (norm1 + norm2) / 2;
 
   const rotate = [];
@@ -54,7 +54,7 @@ const estimate = ({
   rotate[5] = rotate[6] * rotate[1] - rotate[0] * rotate[7];
   rotate[8] = rotate[0] * rotate[4] - rotate[1] * rotate[3];
 
-  const norm3 = Math.sqrt(rotate[2] * rotate[2] + rotate[5] * rotate[5] + rotate[8] * rotate[8]);
+  const norm3 = Math.sqrt(rotate[2] ** 2 + rotate[5] ** 2 + rotate[8] ** 2);
   rotate[2] /= norm3;
   rotate[5] /= norm3;
   rotate[8] /= norm3;