diff --git a/.eslintrc.json b/.eslintrc.json index b94f9c938..7bc6aae3d 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -7,38 +7,37 @@ "standard-with-typescript", "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended", - "plugin:prettier/recommended", "prettier/@typescript-eslint" ], "parser": "@typescript-eslint/parser", "parserOptions": { + "ecmaVersion": 2020, "project": "./tsconfig.json", - "sourceType": "module", - "ecmaVersion": 2020 + "sourceType": "module" }, "plugins": ["prettier", "@typescript-eslint"], "root": true, "rules": { - "@typescript-eslint/explicit-function-return-type": "off", + "@typescript-eslint/explicit-function-return-type": "warn", "@typescript-eslint/interface-name-prefix": "off", "@typescript-eslint/member-delimiter-style": "off", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-var-requires": "off", + "@typescript-eslint/no-empty-function": "warn", + "@typescript-eslint/no-var-requires": "warn", "@typescript-eslint/semi": "off", "@typescript-eslint/space-before-function-paren": "off", - "@typescript-eslint/strict-boolean-expressions": "off", + "@typescript-eslint/strict-boolean-expressions": "warn", "arrow-parens": "off", "class-methods-use-this": "off", + "comma-dangle": ["error", "always-multiline"], "max-classes-per-file": "off", "no-continue": "off", - "no-empty-function": "off", + "no-empty-function": "warn", "no-multi-assign": "off", "no-param-reassign": "off", "no-plusplus": "off", "no-prototype-builtins": "off", "no-restricted-globals": "off", "no-underscore-dangle": "off", - "prettier/prettier": "error", "semi": "off", "standard/no-callback-literal": "off" } diff --git a/.prettierrc b/.prettierrc index 92f97e756..120fa4911 100644 --- a/.prettierrc +++ b/.prettierrc @@ -2,5 +2,5 @@ "semi": true, "singleQuote": true, "tabWidth": 2, - "trailingComma": "es5" + "trailingComma": "all" } diff --git a/__tests__/activation/index.ts b/__tests__/activation/index.ts index d57487782..19d41cf6d 100644 --- a/__tests__/activation/index.ts +++ b/__tests__/activation/index.ts @@ -1,5 +1,6 @@ import * as activation from '../../src/activation'; import * as leakyRelu from '../../src/activation/leaky-relu'; +import * as mish from '../../src/activation/mish'; import * as relu from '../../src/activation/relu'; import * as sigmoid from '../../src/activation/sigmoid'; import * as tanh from '../../src/activation/tanh'; @@ -10,5 +11,6 @@ describe('activation', () => { expect(activation.relu).toBe(relu); expect(activation.sigmoid).toBe(sigmoid); expect(activation.tanh).toBe(tanh); + expect(activation.mish).toBe(mish); }); }); diff --git a/__tests__/activation/mish.ts b/__tests__/activation/mish.ts new file mode 100644 index 000000000..f889368c5 --- /dev/null +++ b/__tests__/activation/mish.ts @@ -0,0 +1,18 @@ +import * as mish from '../../src/activation/mish'; + +describe('tanh', () => { + describe('.active()', () => { + it('matches for value 4', () => { + expect(mish.activate(4).toFixed(5)).toBe('3.99741'); + expect(mish.activate(1).toFixed(5)).toBe('0.86510'); + expect(mish.activate(0).toFixed(5)).toBe('0.00000'); + expect(mish.activate(-10).toFixed(5)).toBe('-0.00045'); + }); + }); + + describe('.measure()', () => { + it('matches for value .7', () => { + expect(mish.measure(-10).toFixed(5)).toBe('-0.00041'); + }); + }); +}); diff --git a/__tests__/layer/mish.ts b/__tests__/layer/mish.ts new file mode 100644 index 000000000..cbf6e5eb4 --- /dev/null +++ b/__tests__/layer/mish.ts @@ -0,0 +1,291 @@ +import { gpuMock } from 'gpu-mock.js'; +import { GPU } from 'gpu.js'; +import * as tanhActivation from '../../src/activation/tanh'; +import { ILayerSettings } from '../../src/layer/base-layer'; +import { + compare2D, + compare3D, + Mish, + mish, + predict2D, + predict3D, +} from '../../src/layer/mish'; +import { makeKernel, setup, teardown } from '../../src/utilities/kernel'; +import { randos2D } from '../../src/utilities/randos'; +import { mockLayer, mockPraxis, shave2D, shave3D } from '../test-utils'; + +jest.mock('../../src/utilities/kernel', () => { + return { + setup: jest.fn(), + teardown: jest.fn(), + makeKernel: jest.fn(() => { + return [[1]]; + }), + release: jest.fn(), + clear: jest.fn(), + }; +}); + +describe('Tanh Layer', () => { + describe('predict2D() (forward propagation)', () => { + test('can tanh a simple matrix', () => { + const inputs = [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.9, 1, 1.1, 1.2], + ]; + const width = 4; + const height = 3; + const results = gpuMock(predict2D, { output: [width, height] })( + inputs, + ) as Float32Array[]; + expect(results.length).toBe(height); + expect(results[0].length).toBe(width); + expect(shave2D(results)).toEqual( + shave2D([ + Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), + Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), + Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), + ]), + ); + }); + }); + + describe('predict3D() (forward propagation)', () => { + test('can tanh a simple matrix', () => { + const inputs = [ + [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.9, 1, 1.1, 1.2], + ], + [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.9, 1, 1.1, 1.2], + ], + ]; + const width = 4; + const height = 3; + const depth = 2; + const results = gpuMock(predict3D, { output: [width, height, depth] })( + inputs, + ) as Float32Array[][]; + + expect(results.length).toBe(depth); + expect(results[0].length).toBe(height); + expect(results[0][0].length).toBe(width); + expect(shave3D(results)).toEqual( + shave3D([ + [ + Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), + Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), + Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), + ], + [ + Float32Array.from([0.099668, 0.19737533, 0.29131261, 0.37994897]), + Float32Array.from([0.46211717, 0.53704959, 0.60436779, 0.66403675]), + Float32Array.from([0.71629786, 0.76159418, 0.80049902, 0.83365458]), + ], + ]), + ); + }); + }); + + describe('compare2D() (back propagation)', () => { + test('can tanh a simple matrix', () => { + const inputs = [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0.8], + [0.9, 1, 1.1, 1.2], + ]; + const deltas = [ + [1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1], + ]; + const width = 4; + const height = 3; + const results = gpuMock(compare2D, { output: [width, height] })( + inputs, + deltas, + ) as Float32Array[]; + expect(results.length).toBe(height); + expect(results[0].length).toBe(width); + expect(shave2D(results)).toEqual( + shave2D([ + Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), + Float32Array.from([0.75, 0.63999999, 0.50999999, 0.36000001]), + Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), + ]), + ); + }); + }); + + describe('compare3D() (back propagation)', () => { + test('can tanh a simple matrix', () => { + const inputs = [ + [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0, 8], + [0.9, 1, 1.1, 1.2], + ], + [ + [0.1, 0.2, 0.3, 0.4], + [0.5, 0.6, 0.7, 0, 8], + [0.9, 1, 1.1, 1.2], + ], + ]; + const deltas = [ + [ + [1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1], + ], + [ + [1, 1, 1, 1], + [1, 1, 1, 1], + [1, 1, 1, 1], + ], + ]; + const width = 4; + const height = 3; + const depth = 2; + const results = gpuMock(compare3D, { output: [width, height, depth] })( + inputs, + deltas, + ) as Float32Array[][]; + expect(results.length).toBe(depth); + expect(results[0].length).toBe(height); + expect(results[0][0].length).toBe(width); + expect(shave3D(results)).toEqual( + shave3D([ + [ + Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), + Float32Array.from([0.75, 0.63999999, 0.50999999, 1]), + Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), + ], + [ + Float32Array.from([0.99000001, 0.95999998, 0.91000003, 0.83999997]), + Float32Array.from([0.75, 0.63999999, 0.50999999, 1]), + Float32Array.from([0.19, 0.0, -0.20999999, -0.44]), + ], + ]), + ); + }); + }); + + describe('.setupKernels()', () => { + beforeEach(() => { + setup( + new GPU({ + mode: 'cpu', + }), + ); + }); + afterEach(() => { + teardown(); + }); + describe('2d', () => { + it('sets up kernels correctly', () => { + const width = 3; + const height = 4; + const mockInputLayer = mockLayer({ width, height }); + const l = new Mish(mockInputLayer); + expect(l.predictKernel).toBe(null); + expect(l.compareKernel).toBe(null); + l.setupKernels(); + expect(l.predictKernel).not.toBe(null); + expect(l.compareKernel).not.toBe(null); + expect(makeKernel).toHaveBeenCalledWith(predict2D, { + functions: [tanhActivation.activate], + immutable: true, + output: [3, 4], + }); + expect(makeKernel).toHaveBeenCalledWith(compare2D, { + functions: [tanhActivation.measure], + immutable: true, + output: [3, 4], + }); + }); + }); + describe('3d', () => { + it('sets up kernels correctly', () => { + const width = 3; + const height = 4; + const depth = 5; + const mockInputLayer = mockLayer({ width, height, depth }); + const l = new Mish(mockInputLayer); + expect(l.predictKernel).toBe(null); + expect(l.compareKernel).toBe(null); + l.setupKernels(); + expect(l.predictKernel).not.toBe(null); + expect(l.compareKernel).not.toBe(null); + expect(makeKernel).toHaveBeenCalledWith(predict3D, { + functions: [tanhActivation.activate], + immutable: true, + output: [3, 4, 5], + }); + expect(makeKernel).toHaveBeenCalledWith(compare3D, { + functions: [tanhActivation.measure], + immutable: true, + output: [3, 4, 5], + }); + }); + }); + }); + + describe('.predict()', () => { + it('calls this.predictKernel() with this.inputLayer.weights', () => { + const mockWeights = randos2D(1, 1); + const mockInputLayer = mockLayer({ + weights: mockWeights, + width: 1, + height: 1, + depth: 1, + }); + const l = new Mish(mockInputLayer); + (l as any).predictKernel = jest.fn((weights) => weights); + l.predict(); + expect(l.predictKernel).toBeCalledWith(mockWeights); + expect(l.weights).toBe(mockWeights); + }); + }); + + describe('.compare()', () => { + it('calls this.compareKernel() with this.inputLayer.weights & this.inputLayer.deltas', () => { + const mockWeights = randos2D(1, 1); + const mockDeltas = randos2D(1, 1); + const mockInputLayer = mockLayer({ + width: 1, + height: 1, + depth: 1, + }); + const l = new Mish(mockInputLayer); + l.weights = mockWeights; + l.deltas = mockDeltas; + const expected = randos2D(1, 1); + (l as any).compareKernel = jest.fn((weights, deltas) => expected); + l.compare(); + expect(l.compareKernel).toBeCalledWith(mockWeights, mockDeltas); + expect(l.inputLayer.deltas).toBe(expected); + }); + }); + + describe('tanh lambda', () => { + test('creates a new instance of Tanh', () => { + const width = 3; + const height = 4; + const depth = 5; + const mockInputLayer = mockLayer({ width, height, depth }); + const mockPraxisInstance = mockPraxis(mockInputLayer); + const settings: ILayerSettings = { initPraxis: () => mockPraxisInstance }; + const l = mish(mockInputLayer, settings); + expect(l.constructor).toBe(Mish); + expect(l.width).toBe(width); + expect(l.height).toBe(height); + expect(l.depth).toBe(depth); + expect(l.praxis).toBe(mockPraxisInstance); + }); + }); +}); diff --git a/package-lock.json b/package-lock.json index 7232642f3..16b581fdb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5266,9 +5266,9 @@ "integrity": "sha512-+lbp8rQ0p1nTa6Gk6HoLiw4yM6JTpql82U+nCF3sZbX4FJWP9PzzF1018dW8K+pbmqRmhLHbn6Bjc6i6tgUpbA==" }, "gpu.js": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/gpu.js/-/gpu.js-2.11.0.tgz", - "integrity": "sha512-Yz99xie7/WwtGvX2rHgfpo11IoW2J3XJ9Vta3S81YfFgJVPHe9mg5E6/cux6wArOMQFdGXwt6OumcuANV52p9w==", + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/gpu.js/-/gpu.js-2.11.1.tgz", + "integrity": "sha512-vSiND00XiSSs+bX3s4+96y88RiNs6oRGQ5SuOIc7f31ZfW0YL+huzbNGE/iufL25m1qQiGUItI5LEvDyjjz6zQ==", "requires": { "acorn": "^7.1.1", "gl": "^4.5.2", diff --git a/src/activation/index.ts b/src/activation/index.ts index 624af43f1..650856cc3 100644 --- a/src/activation/index.ts +++ b/src/activation/index.ts @@ -1,4 +1,5 @@ +export * as leakyRelu from './leaky-relu'; +export * as mish from './mish'; export * as relu from './relu'; export * as sigmoid from './sigmoid'; export * as tanh from './tanh'; -export * as leakyRelu from './leaky-relu'; diff --git a/src/activation/mish.ts b/src/activation/mish.ts new file mode 100644 index 000000000..b4580cc85 --- /dev/null +++ b/src/activation/mish.ts @@ -0,0 +1,26 @@ +/** + * mish activation + */ +export function activate(weight: number): number { + return ( + (weight * + (Math.exp(Math.log(1 + Math.exp(weight))) - + Math.exp(-Math.log(1 + Math.exp(weight))))) / + (Math.exp(Math.log(1 + Math.exp(weight))) + + Math.exp(-Math.log(1 + Math.exp(weight)))) + ); +} + +/** + * mish derivative + */ +export function measure(weight: number): number { + const omega = + Math.exp(3 * weight) + + 4 * Math.exp(2 * weight) + + (6 + 4 * weight) * Math.exp(weight) + + 4 * (1 + weight); + const delta = 1 + Math.pow(Math.exp(weight) + 1, 2); + + return (Math.exp(weight) * omega) / Math.pow(delta, 2); +} diff --git a/src/index.ts b/src/index.ts index 19e32b62c..f50eeff58 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,7 +6,7 @@ import { layerTypes } from './layer'; import { likely } from './likely'; import { lookup } from './lookup'; import { NeuralNetwork } from './neural-network'; -import NeuralNetworkGPU from './neural-network-gpu'; +import { NeuralNetworkGPU } from './neural-network-gpu'; import * as praxis from './praxis'; import { Recurrent } from './recurrent'; import { GRU } from './recurrent/gru'; @@ -47,7 +47,7 @@ export const brain = { GRUTimeStep, RNN, LSTM, - GRU, + GRU }, utilities: { max, @@ -61,8 +61,8 @@ export const brain = { toArray, DataFormatter, zeros, - toSVG, - }, + toSVG + } }; declare global { diff --git a/src/layer/mish.ts b/src/layer/mish.ts new file mode 100644 index 000000000..6290ce11b --- /dev/null +++ b/src/layer/mish.ts @@ -0,0 +1,85 @@ +import { IKernelFunctionThis, IKernelRunShortcut } from 'gpu.js'; +import { activate, measure } from '../activation/mish'; +import { clear, makeKernel, release } from '../utilities/kernel'; +import { Activation } from './activation'; +import { ILayer, ILayerSettings } from './base-layer'; + +export function predict2D( + this: IKernelFunctionThis, + inputs: number[][], +): number { + return activate(inputs[this.thread.y][this.thread.x]); +} + +export function predict3D( + this: IKernelFunctionThis, + inputs: number[][][], +): number { + return activate(inputs[this.thread.z][this.thread.y][this.thread.x]); +} + +export function compare2D( + this: IKernelFunctionThis, + weights: number[][], + errors: number[][], +): number { + return measure(weights[this.thread.y][this.thread.x]); +} + +export function compare3D( + this: IKernelFunctionThis, + weights: number[][][], + errors: number[][][], +): number { + return measure(weights[this.thread.z][this.thread.y][this.thread.x]); +} + +export class Mish extends Activation { + setupKernels(): void { + if (this.depth > 0) { + this.predictKernel = makeKernel(predict3D, { + output: [this.width, this.height, this.depth], + functions: [activate], + immutable: true, + }); + + this.compareKernel = makeKernel(compare3D, { + output: [this.width, this.height, this.depth], + functions: [measure], + immutable: true, + }); + } else { + this.predictKernel = makeKernel(predict2D, { + output: [this.width, this.height], + functions: [activate], + immutable: true, + }); + + this.compareKernel = makeKernel(compare2D, { + output: [this.width, this.height], + functions: [measure], + immutable: true, + }); + } + } + + predict(): void { + release(this.weights); + this.weights = (this.predictKernel as IKernelRunShortcut)( + this.inputLayer.weights, + ); + clear(this.deltas); + } + + compare(): void { + release(this.inputLayer.deltas); + this.inputLayer.deltas = (this.compareKernel as IKernelRunShortcut)( + this.weights, + this.deltas, + ); + } +} + +export function mish(inputLayer: ILayer, settings?: ILayerSettings): Mish { + return new Mish(inputLayer, settings); +}