From 6f4c6c2d8afc7090c27ebbeabb045182e971af75 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 09:51:01 -0700 Subject: [PATCH 01/18] first working kernel finder --- .../jupyter/kernels/kernelSelections.ts | 4 +- .../kernel-launcher/kernelFinder.ts | 126 +++++++++++++++++- .../datascience/kernel-launcher/types.ts | 2 +- 3 files changed, 127 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/jupyter/kernels/kernelSelections.ts b/src/client/datascience/jupyter/kernels/kernelSelections.ts index cb584f4e0e26..99754b290996 100644 --- a/src/client/datascience/jupyter/kernels/kernelSelections.ts +++ b/src/client/datascience/jupyter/kernels/kernelSelections.ts @@ -122,10 +122,10 @@ export class InstalledJupyterKernelSelectionListProvider implements IKernelSelec export class InstalledRawKernelSelectionListProvider implements IKernelSelectionListProvider { constructor(private readonly kernelFinder: IKernelFinder, private readonly pathUtils: IPathUtils) {} public async getKernelSelections( - _resource: Resource, + resource: Resource, cancelToken?: CancellationToken ): Promise { - const items = await this.kernelFinder.listKernelSpecs(cancelToken); + const items = await this.kernelFinder.listKernelSpecs(resource, cancelToken); return items .filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) .map((item) => getQuickPickItemForKernelSpec(item, this.pathUtils)); diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 931946b4865a..e794c523da26 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -46,8 +46,15 @@ export function findIndexOfConnectionFile(kernelSpec: Readonly>(); + + // Store any json file that we have loaded from disk before + private pathToKernelSpec = new Map>(); + constructor( @inject(IInterpreterService) private interpreterService: IInterpreterService, @inject(IInterpreterLocatorService) @@ -113,8 +120,123 @@ export class KernelFinder implements IKernelFinder { } // Search all our local file system locations for installed kernel specs and return them - public async listKernelSpecs(_cancelToken?: CancellationToken): Promise { - throw new Error('Not yet implmented'); + public async listKernelSpecs(resource: Resource, _cancelToken?: CancellationToken): Promise { + // If we have not already searched for this resource, then generate the search + if (!this.resourceToKernels.has(resource)) { + this.resourceToKernels.set(resource, this.findResourceKernelSpecs(resource, _cancelToken)); + } + + // ! as the has and set above verify that we have a return here + return this.resourceToKernels.get(resource)!; + } + + private async findResourceKernelSpecs( + resource: Resource, + _cancelToken?: CancellationToken + ): Promise { + const results: IJupyterKernelSpec[] = []; + + // Find all the possible places to look for this resource + const paths = await this.findAllResourcePossibleKernelPaths(resource); + + // Next search to find what actual json files there are + // IANHU: Finder also doing something similar + const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath)); + const searchResults = await Promise.all(promises); + + searchResults.forEach((result, i) => { + result.forEach(async (jsonpath) => { + // We are not using the cache for list all, but add the items that we find so the finder knows about them + // Only push if it's not there already + if (!this.cache.includes(jsonpath)) { + // IANHU: Don't mess with the cache while I'm still testing this + //this.cache.push(jsonpath); + } + + const specPath = path.join(paths[i], jsonpath); + const kernelspec = await this.getKernelSpec(specPath); + results.push(kernelspec); + }); + }); + + return results; + } + + // IANHU: have the finder code use this as well + private async getKernelSpec(specPath: string): Promise { + // If we have not already searched for this resource, then generate the search + if (!this.pathToKernelSpec.has(specPath)) { + this.pathToKernelSpec.set(specPath, this.loadKernelSpec(specPath)); + } + + // ! as the has and set above verify that we have a return here + return this.pathToKernelSpec.get(specPath)!; + } + + // Load a kernelspec from disk + private async loadKernelSpec(specPath: string): Promise { + const kernelJson = JSON.parse(await this.file.readFile(specPath)); + return new JupyterKernelSpec(kernelJson, specPath); + } + + private async findAllResourcePossibleKernelPaths( + resource: Resource, + _cancelToken?: CancellationToken + ): Promise { + const possiblePaths: string[] = []; + possiblePaths.push(...(await this.getActiveInterpreterPath(resource))); + // IANHU: Also add in interpreter paths and disk paths + // Use Promise.all to let them run together + possiblePaths.push(...(await this.getInterpreterPaths(resource))); + possiblePaths.push(...(await this.getDiskPaths())); + + return possiblePaths; + } + + // IANHU: Have the finder use this as well? + private async getActiveInterpreterPath(resource: Resource): Promise { + const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource); + if (activeInterpreter) { + return [path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')]; + } + + return []; + } + + // IANHU: also combine with finder + private async getInterpreterPaths(resource: Resource): Promise { + const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false }); + const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix); + const interpreterPaths = interpreterPrefixPaths.map((prefixPath) => + path.join(prefixPath, kernelPaths.get('kernel')!) + ); + return interpreterPaths; + } + + // IANHU: Also combine with finder code + private async getDiskPaths(): Promise { + let paths = []; + + if (this.platformService.isWindows) { + paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)]; + + if (process.env.ALLUSERSPROFILE) { + paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels')); + } + } else { + // Unix based + const secondPart = this.platformService.isMac + ? kernelPaths.get('macJupyterPath')! + : kernelPaths.get('linuxJupyterPath')!; + + paths = [ + path.join('usr', 'share', 'jupyter', 'kernels'), + path.join('usr', 'local', 'share', 'jupyter', 'kernels'), + path.join(this.pathUtils.home, secondPart) + ]; + } + + return paths; } // For the given kernelspec return back the kernelspec with ipykernel installed into it or error diff --git a/src/client/datascience/kernel-launcher/types.ts b/src/client/datascience/kernel-launcher/types.ts index 9656ef790296..2eda298a6068 100644 --- a/src/client/datascience/kernel-launcher/types.ts +++ b/src/client/datascience/kernel-launcher/types.ts @@ -49,7 +49,7 @@ export interface IKernelFinder { kernelName?: string, cancelToken?: CancellationToken ): Promise; - listKernelSpecs(cancelToken?: CancellationToken): Promise; + listKernelSpecs(resource: Resource, cancelToken?: CancellationToken): Promise; } /** From 9ff896605b9ca6a9e69f63cbcfddbd6f728997aa Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 10:36:04 -0700 Subject: [PATCH 02/18] path fix --- src/client/datascience/kernel-launcher/kernelFinder.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index e794c523da26..3ef8bdcdce41 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -148,12 +148,12 @@ export class KernelFinder implements IKernelFinder { result.forEach(async (jsonpath) => { // We are not using the cache for list all, but add the items that we find so the finder knows about them // Only push if it's not there already - if (!this.cache.includes(jsonpath)) { + const specPath = path.join(paths[i], jsonpath); + if (!this.cache.includes(specPath)) { // IANHU: Don't mess with the cache while I'm still testing this - //this.cache.push(jsonpath); + //this.cache.push(specPath); } - const specPath = path.join(paths[i], jsonpath); const kernelspec = await this.getKernelSpec(specPath); results.push(kernelspec); }); From 33d538bcffdf9cdea69c629f37eb48dec6284eed Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 10:43:36 -0700 Subject: [PATCH 03/18] promise all for getting paths --- .../datascience/kernel-launcher/kernelFinder.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 3ef8bdcdce41..fb79b007056f 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -183,14 +183,13 @@ export class KernelFinder implements IKernelFinder { resource: Resource, _cancelToken?: CancellationToken ): Promise { - const possiblePaths: string[] = []; - possiblePaths.push(...(await this.getActiveInterpreterPath(resource))); - // IANHU: Also add in interpreter paths and disk paths - // Use Promise.all to let them run together - possiblePaths.push(...(await this.getInterpreterPaths(resource))); - possiblePaths.push(...(await this.getDiskPaths())); - - return possiblePaths; + const [activePath, interpreterPaths, diskPaths] = await Promise.all([ + this.getActiveInterpreterPath(resource), + this.getInterpreterPaths(resource), + this.getDiskPaths() + ]); + + return [...activePath, ...interpreterPaths, ...diskPaths]; } // IANHU: Have the finder use this as well? From 6eb4feb8c47aaa94b671b977222aa2a3f97f0237 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 11:04:45 -0700 Subject: [PATCH 04/18] merge path functions in --- .vscode/launch.json | 1 + .../kernel-launcher/kernelFinder.ts | 73 +++++++++---------- .../datascience/kernelFinder.unit.test.ts | 2 +- 3 files changed, 35 insertions(+), 41 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 95cc798affcb..b72b20e2071c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -240,6 +240,7 @@ "--recursive", "--colors", // "--grep", "", + "--grep", "IANHU", "--timeout=300000", "--fast" ], diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index fb79b007056f..bb28feae0076 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -86,7 +86,7 @@ export class KernelFinder implements IKernelFinder { // Check in active interpreter first if (activeInterpreter) { - kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, activeInterpreter); + kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource); } if (kernelSpec) { @@ -95,12 +95,9 @@ export class KernelFinder implements IKernelFinder { } const diskSearch = this.findDiskPath(kernelName); - const interpreterSearch = this.interpreterLocator - .getInterpreters(resource, { ignoreCache: false }) - .then((interpreters) => { - const interpreterPaths = interpreters.map((interp) => interp.sysPrefix); - return this.findInterpreterPath(interpreterPaths, kernelName); - }); + const interpreterSearch = this.getInterpreterPaths(resource).then((interpreterPaths) => { + return this.findInterpreterPath(interpreterPaths, kernelName); + }); let result = await Promise.race([diskSearch, interpreterSearch]); if (!result) { @@ -179,6 +176,7 @@ export class KernelFinder implements IKernelFinder { return new JupyterKernelSpec(kernelJson, specPath); } + // For the given resource, find atll the file paths for kernel specs that wewant to associate with this private async findAllResourcePossibleKernelPaths( resource: Resource, _cancelToken?: CancellationToken @@ -192,9 +190,9 @@ export class KernelFinder implements IKernelFinder { return [...activePath, ...interpreterPaths, ...diskPaths]; } - // IANHU: Have the finder use this as well? private async getActiveInterpreterPath(resource: Resource): Promise { const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource); + if (activeInterpreter) { return [path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')]; } @@ -202,17 +200,12 @@ export class KernelFinder implements IKernelFinder { return []; } - // IANHU: also combine with finder private async getInterpreterPaths(resource: Resource): Promise { const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false }); const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix); - const interpreterPaths = interpreterPrefixPaths.map((prefixPath) => - path.join(prefixPath, kernelPaths.get('kernel')!) - ); - return interpreterPaths; + return interpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, kernelPaths.get('kernel')!)); } - // IANHU: Also combine with finder code private async getDiskPaths(): Promise { let paths = []; @@ -264,12 +257,10 @@ export class KernelFinder implements IKernelFinder { private async getKernelSpecFromActiveInterpreter( kernelName: string, - activeInterpreter: PythonInterpreter + resource: Resource ): Promise { - return this.getKernelSpecFromDisk( - [path.join(activeInterpreter.sysPrefix, 'share', 'jupyter', 'kernels')], - kernelName - ); + const activePath = await this.getActiveInterpreterPath(resource); + return this.getKernelSpecFromDisk(activePath, kernelName); } private async findInterpreterPath( @@ -277,7 +268,8 @@ export class KernelFinder implements IKernelFinder { kernelName: string ): Promise { const promises = interpreterPaths.map((intPath) => - this.getKernelSpecFromDisk([path.join(intPath, kernelPaths.get('kernel')!)], kernelName) + //this.getKernelSpecFromDisk([path.join(intPath, kernelPaths.get('kernel')!)], kernelName) + this.getKernelSpecFromDisk([intPath], kernelName) ); const specs = await Promise.all(promises); @@ -287,26 +279,27 @@ export class KernelFinder implements IKernelFinder { // Jupyter looks for kernels in these paths: // https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs private async findDiskPath(kernelName: string): Promise { - let paths = []; - - if (this.platformService.isWindows) { - paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)]; - - if (process.env.ALLUSERSPROFILE) { - paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels')); - } - } else { - // Unix based - const secondPart = this.platformService.isMac - ? kernelPaths.get('macJupyterPath')! - : kernelPaths.get('linuxJupyterPath')!; - - paths = [ - path.join('usr', 'share', 'jupyter', 'kernels'), - path.join('usr', 'local', 'share', 'jupyter', 'kernels'), - path.join(this.pathUtils.home, secondPart) - ]; - } + //let paths = []; + + //if (this.platformService.isWindows) { + //paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)]; + + //if (process.env.ALLUSERSPROFILE) { + //paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels')); + //} + //} else { + //// Unix based + //const secondPart = this.platformService.isMac + //? kernelPaths.get('macJupyterPath')! + //: kernelPaths.get('linuxJupyterPath')!; + + //paths = [ + //path.join('usr', 'share', 'jupyter', 'kernels'), + //path.join('usr', 'local', 'share', 'jupyter', 'kernels'), + //path.join(this.pathUtils.home, secondPart) + //]; + //} + const paths = await this.getDiskPaths(); return this.getKernelSpecFromDisk(paths, kernelName); } diff --git a/src/test/datascience/kernelFinder.unit.test.ts b/src/test/datascience/kernelFinder.unit.test.ts index a435c9f66b14..eac3c40e658c 100644 --- a/src/test/datascience/kernelFinder.unit.test.ts +++ b/src/test/datascience/kernelFinder.unit.test.ts @@ -22,7 +22,7 @@ import { PythonInterpreter } from '../../client/interpreter/contracts'; -suite('Kernel Finder', () => { +suite('IANHU Kernel Finder', () => { let interpreterService: typemoq.IMock; let interpreterLocator: typemoq.IMock; let fileSystem: typemoq.IMock; From 03039fb5814a18f00dce05aeba4db01204b25e42 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 11:32:15 -0700 Subject: [PATCH 05/18] better glob search in finder --- .../kernel-launcher/kernelFinder.ts | 59 +++++++------------ 1 file changed, 22 insertions(+), 37 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index bb28feae0076..c1ff5bf2b262 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -22,6 +22,8 @@ import { JupyterKernelSpec } from '../jupyter/kernels/jupyterKernelSpec'; import { IJupyterKernelSpec } from '../types'; import { getKernelInterpreter } from './helpers'; import { IKernelFinder } from './types'; +// tslint:disable-next-line:no-require-imports no-var-requires +const flatten = require('lodash/flatten') as typeof import('lodash/flatten'); const kernelPaths = new Map([ ['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')], @@ -136,24 +138,12 @@ export class KernelFinder implements IKernelFinder { // Find all the possible places to look for this resource const paths = await this.findAllResourcePossibleKernelPaths(resource); - // Next search to find what actual json files there are - // IANHU: Finder also doing something similar - const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath)); - const searchResults = await Promise.all(promises); - - searchResults.forEach((result, i) => { - result.forEach(async (jsonpath) => { - // We are not using the cache for list all, but add the items that we find so the finder knows about them - // Only push if it's not there already - const specPath = path.join(paths[i], jsonpath); - if (!this.cache.includes(specPath)) { - // IANHU: Don't mess with the cache while I'm still testing this - //this.cache.push(specPath); - } + const searchResults = await this.kernelGlobSearch(paths); - const kernelspec = await this.getKernelSpec(specPath); - results.push(kernelspec); - }); + searchResults.forEach(async (resultPath) => { + // IANHU: Add to cache here as well? + const kernelspec = await this.getKernelSpec(resultPath); + results.push(kernelspec); }); return results; @@ -231,6 +221,21 @@ export class KernelFinder implements IKernelFinder { return paths; } + // IANHU: Have the finder code use this as well for searching + private async kernelGlobSearch(paths: string[]): Promise { + const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath)); + const searchResults = await Promise.all(promises); + + // Append back on the start of each path so we have the full path in the results + const fullPathResults = searchResults.map((result, index) => { + return result.map((partialSpecPath) => { + return path.join(paths[index], partialSpecPath); + }); + }); + + return flatten(fullPathResults); + } + // For the given kernelspec return back the kernelspec with ipykernel installed into it or error private async verifyIpyKernel( kernelSpec: IJupyterKernelSpec, @@ -279,26 +284,6 @@ export class KernelFinder implements IKernelFinder { // Jupyter looks for kernels in these paths: // https://jupyter-client.readthedocs.io/en/stable/kernels.html#kernel-specs private async findDiskPath(kernelName: string): Promise { - //let paths = []; - - //if (this.platformService.isWindows) { - //paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)]; - - //if (process.env.ALLUSERSPROFILE) { - //paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels')); - //} - //} else { - //// Unix based - //const secondPart = this.platformService.isMac - //? kernelPaths.get('macJupyterPath')! - //: kernelPaths.get('linuxJupyterPath')!; - - //paths = [ - //path.join('usr', 'share', 'jupyter', 'kernels'), - //path.join('usr', 'local', 'share', 'jupyter', 'kernels'), - //path.join(this.pathUtils.home, secondPart) - //]; - //} const paths = await this.getDiskPaths(); return this.getKernelSpecFromDisk(paths, kernelName); From acd249a888bfb0d3711c6b6449bc36abe56c1f92 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 11:34:23 -0700 Subject: [PATCH 06/18] update comment --- src/client/datascience/kernel-launcher/kernelFinder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index c1ff5bf2b262..a9d3f395c14d 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -151,7 +151,7 @@ export class KernelFinder implements IKernelFinder { // IANHU: have the finder code use this as well private async getKernelSpec(specPath: string): Promise { - // If we have not already searched for this resource, then generate the search + // If we have not already loaded this kernel spec, then load it if (!this.pathToKernelSpec.has(specPath)) { this.pathToKernelSpec.set(specPath, this.loadKernelSpec(specPath)); } From 1f1466a2b45c7e846aaa1bcf8fdb33a6ac0ec514 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 14:20:00 -0700 Subject: [PATCH 07/18] both finds use glob search --- .../kernel-launcher/kernelFinder.ts | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index a9d3f395c14d..959b87682e39 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -140,11 +140,13 @@ export class KernelFinder implements IKernelFinder { const searchResults = await this.kernelGlobSearch(paths); - searchResults.forEach(async (resultPath) => { - // IANHU: Add to cache here as well? - const kernelspec = await this.getKernelSpec(resultPath); - results.push(kernelspec); - }); + await Promise.all( + searchResults.map(async (resultPath) => { + // IANHU: Add to cache as well? + const kernelspec = await this.getKernelSpec(resultPath); + results.push(kernelspec); + }) + ); return results; } @@ -221,9 +223,9 @@ export class KernelFinder implements IKernelFinder { return paths; } - // IANHU: Have the finder code use this as well for searching + // Given a set of paths, search for kernel.json files and return back the full paths of all of them that we find private async kernelGlobSearch(paths: string[]): Promise { - const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath)); + const promises = paths.map((kernelPath) => this.file.search(`**${path.sep}kernel.json`, kernelPath)); const searchResults = await Promise.all(promises); // Append back on the start of each path so we have the full path in the results @@ -272,10 +274,7 @@ export class KernelFinder implements IKernelFinder { interpreterPaths: string[], kernelName: string ): Promise { - const promises = interpreterPaths.map((intPath) => - //this.getKernelSpecFromDisk([path.join(intPath, kernelPaths.get('kernel')!)], kernelName) - this.getKernelSpecFromDisk([intPath], kernelName) - ); + const promises = interpreterPaths.map((intPath) => this.getKernelSpecFromDisk([intPath], kernelName)); const specs = await Promise.all(promises); return specs.find((sp) => sp !== undefined); @@ -290,15 +289,11 @@ export class KernelFinder implements IKernelFinder { } private async getKernelSpecFromDisk(paths: string[], kernelName: string): Promise { - const promises = paths.map((kernelPath) => this.file.search('**/kernel.json', kernelPath)); - const searchResults = await Promise.all(promises); - searchResults.forEach((result, i) => { - result.forEach((res) => { - const specPath = path.join(paths[i], res); - if (!this.cache.includes(specPath)) { - this.cache.push(specPath); - } - }); + const searchResults = await this.kernelGlobSearch(paths); + searchResults.forEach((specPath) => { + if (!this.cache.includes(specPath)) { + this.cache.push(specPath); + } }); return this.searchCache(kernelName); From ea2355736f1e43ea751bb624c22079d5835808f8 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 14:25:44 -0700 Subject: [PATCH 08/18] change path constants --- .../kernel-launcher/kernelFinder.ts | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 959b87682e39..257d7f6df34b 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -25,12 +25,17 @@ import { IKernelFinder } from './types'; // tslint:disable-next-line:no-require-imports no-var-requires const flatten = require('lodash/flatten') as typeof import('lodash/flatten'); -const kernelPaths = new Map([ - ['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')], - ['linuxJupyterPath', path.join('.local', 'share', 'jupyter', 'kernels')], - ['macJupyterPath', path.join('Library', 'Jupyter', 'kernels')], - ['kernel', path.join('share', 'jupyter', 'kernels')] -]); +//const kernelPaths = new Map([ +//['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')], +//['linuxJupyterPath', path.join('.local', 'share', 'jupyter', 'kernels')], +//['macJupyterPath', path.join('Library', 'Jupyter', 'kernels')], +//['kernel', path.join('share', 'jupyter', 'kernels')] +//]); +const winJupyterPath = path.join('AppData', 'Roaming', 'jupyter', 'kernels'); +const linuxJupyterPath = path.join('.local', 'share', 'jupyter', 'kernels'); +const macJupyterPath = path.join('Library', 'Jupyter', 'kernels'); +const baseKernelPath = path.join('share', 'jupyter', 'kernels'); + const cacheFile = 'kernelSpecPathCache.json'; const defaultSpecName = 'python_defaultSpec_'; @@ -195,23 +200,21 @@ export class KernelFinder implements IKernelFinder { private async getInterpreterPaths(resource: Resource): Promise { const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false }); const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix); - return interpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, kernelPaths.get('kernel')!)); + return interpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, baseKernelPath)); } private async getDiskPaths(): Promise { let paths = []; if (this.platformService.isWindows) { - paths = [path.join(this.pathUtils.home, kernelPaths.get('winJupyterPath')!)]; + paths = [path.join(this.pathUtils.home, winJupyterPath)]; if (process.env.ALLUSERSPROFILE) { paths.push(path.join(process.env.ALLUSERSPROFILE, 'jupyter', 'kernels')); } } else { // Unix based - const secondPart = this.platformService.isMac - ? kernelPaths.get('macJupyterPath')! - : kernelPaths.get('linuxJupyterPath')!; + const secondPart = this.platformService.isMac ? macJupyterPath : linuxJupyterPath; paths = [ path.join('usr', 'share', 'jupyter', 'kernels'), From 84fe7fb3546f45ae10f1e42711fdaf89946ceb53 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 15:13:43 -0700 Subject: [PATCH 09/18] add result to search cache --- .../datascience/kernel-launcher/kernelFinder.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 257d7f6df34b..32d9713f872d 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -25,12 +25,6 @@ import { IKernelFinder } from './types'; // tslint:disable-next-line:no-require-imports no-var-requires const flatten = require('lodash/flatten') as typeof import('lodash/flatten'); -//const kernelPaths = new Map([ -//['winJupyterPath', path.join('AppData', 'Roaming', 'jupyter', 'kernels')], -//['linuxJupyterPath', path.join('.local', 'share', 'jupyter', 'kernels')], -//['macJupyterPath', path.join('Library', 'Jupyter', 'kernels')], -//['kernel', path.join('share', 'jupyter', 'kernels')] -//]); const winJupyterPath = path.join('AppData', 'Roaming', 'jupyter', 'kernels'); const linuxJupyterPath = path.join('.local', 'share', 'jupyter', 'kernels'); const macJupyterPath = path.join('Library', 'Jupyter', 'kernels'); @@ -53,7 +47,6 @@ export function findIndexOfConnectionFile(kernelSpec: Readonly { - // IANHU: Add to cache as well? + // Add these into our path cache to speed up later finds + if (!this.cache.includes(resultPath)) { + this.cache.push(resultPath); + } const kernelspec = await this.getKernelSpec(resultPath); results.push(kernelspec); }) From f8eca0400aca9c6854ade24b447d6f44b3388897 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 15:41:25 -0700 Subject: [PATCH 10/18] handle errors and have find use new load mechanism --- .../kernel-launcher/kernelFinder.ts | 66 +++++++++++-------- 1 file changed, 40 insertions(+), 26 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 32d9713f872d..bf2e57fad537 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -7,15 +7,10 @@ import { inject, injectable, named } from 'inversify'; import * as path from 'path'; import { CancellationToken, CancellationTokenSource } from 'vscode'; import { wrapCancellationTokens } from '../../common/cancellation'; -import { traceInfo } from '../../common/logger'; +import { traceError, traceInfo } from '../../common/logger'; import { IFileSystem, IPlatformService } from '../../common/platform/types'; import { IExtensionContext, IInstaller, InstallerResponse, IPathUtils, Product, Resource } from '../../common/types'; -import { - IInterpreterLocatorService, - IInterpreterService, - KNOWN_PATH_SERVICE, - PythonInterpreter -} from '../../interpreter/contracts'; +import { IInterpreterLocatorService, IInterpreterService, KNOWN_PATH_SERVICE } from '../../interpreter/contracts'; import { captureTelemetry } from '../../telemetry'; import { Telemetry } from '../constants'; import { JupyterKernelSpec } from '../jupyter/kernels/jupyterKernelSpec'; @@ -53,7 +48,7 @@ export class KernelFinder implements IKernelFinder { private resourceToKernels = new Map>(); // Store any json file that we have loaded from disk before - private pathToKernelSpec = new Map>(); + private pathToKernelSpec = new Map>(); constructor( @inject(IInterpreterService) private interpreterService: IInterpreterService, @@ -75,7 +70,6 @@ export class KernelFinder implements IKernelFinder { ): Promise { this.cache = await this.readCache(); let foundKernel: IJupyterKernelSpec | undefined; - const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource); if (kernelName && !kernelName.includes(defaultSpecName)) { let kernelSpec = await this.searchCache(kernelName); @@ -85,9 +79,7 @@ export class KernelFinder implements IKernelFinder { } // Check in active interpreter first - if (activeInterpreter) { - kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource); - } + kernelSpec = await this.getKernelSpecFromActiveInterpreter(kernelName, resource); if (kernelSpec) { this.writeCache(this.cache).ignoreErrors(); @@ -105,9 +97,9 @@ export class KernelFinder implements IKernelFinder { result = both[0] ? both[0] : both[1]; } - foundKernel = result ? result : await this.getDefaultKernelSpec(activeInterpreter); + foundKernel = result ? result : await this.getDefaultKernelSpec(resource); } else { - foundKernel = await this.getDefaultKernelSpec(activeInterpreter); + foundKernel = await this.getDefaultKernelSpec(resource); } this.writeCache(this.cache).ignoreErrors(); @@ -145,27 +137,45 @@ export class KernelFinder implements IKernelFinder { this.cache.push(resultPath); } const kernelspec = await this.getKernelSpec(resultPath); - results.push(kernelspec); + + if (kernelspec) { + results.push(kernelspec); + } }) ); return results; } - // IANHU: have the finder code use this as well - private async getKernelSpec(specPath: string): Promise { + // Load the IJupyterKernelSpec for a given spec path, check the ones that we have already loaded first + private async getKernelSpec(specPath: string): Promise { // If we have not already loaded this kernel spec, then load it if (!this.pathToKernelSpec.has(specPath)) { this.pathToKernelSpec.set(specPath, this.loadKernelSpec(specPath)); } // ! as the has and set above verify that we have a return here - return this.pathToKernelSpec.get(specPath)!; + return this.pathToKernelSpec.get(specPath)!.then((value) => { + if (value) { + return value; + } + + // If we failed to get a kernelspec pull path from our cache and loaded list + this.pathToKernelSpec.delete(specPath); + this.cache = this.cache.filter((itempath) => itempath !== specPath); + return undefined; + }); } - // Load a kernelspec from disk - private async loadKernelSpec(specPath: string): Promise { - const kernelJson = JSON.parse(await this.file.readFile(specPath)); + // Load kernelspec json from disk + private async loadKernelSpec(specPath: string): Promise { + let kernelJson; + try { + kernelJson = JSON.parse(await this.file.readFile(specPath)); + } catch { + traceError(`Failed to parse kernelspec ${specPath}`); + return undefined; + } return new JupyterKernelSpec(kernelJson, specPath); } @@ -298,7 +308,9 @@ export class KernelFinder implements IKernelFinder { return this.searchCache(kernelName); } - private async getDefaultKernelSpec(activeInterpreter?: PythonInterpreter): Promise { + private async getDefaultKernelSpec(resource: Resource): Promise { + const activeInterpreter = await this.interpreterService.getActiveInterpreter(resource); + // This creates a default kernel spec. When launched, 'python' argument will map to using the interpreter // associated with the current resource for launching. const defaultSpec: Kernel.ISpecModel = { @@ -339,10 +351,12 @@ export class KernelFinder implements IKernelFinder { }); if (kernelJsonFile) { - const kernelJson = JSON.parse(await this.file.readFile(kernelJsonFile)); - const spec = new JupyterKernelSpec(kernelJson, kernelJsonFile); - spec.name = kernelName; - return spec; + const spec = await this.getKernelSpec(kernelJsonFile); + + if (spec) { + spec.name = kernelName; + return spec; + } } return undefined; From db09c1ea2f85383e26f64b5169b17b50ea6d8562 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 15:59:30 -0700 Subject: [PATCH 11/18] remove cancellation --- .../datascience/jupyter/kernels/kernelSelections.ts | 4 ++-- src/client/datascience/kernel-launcher/kernelFinder.ts | 9 +++------ src/client/datascience/kernel-launcher/types.ts | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/client/datascience/jupyter/kernels/kernelSelections.ts b/src/client/datascience/jupyter/kernels/kernelSelections.ts index 99754b290996..db810aaaf443 100644 --- a/src/client/datascience/jupyter/kernels/kernelSelections.ts +++ b/src/client/datascience/jupyter/kernels/kernelSelections.ts @@ -123,9 +123,9 @@ export class InstalledRawKernelSelectionListProvider implements IKernelSelection constructor(private readonly kernelFinder: IKernelFinder, private readonly pathUtils: IPathUtils) {} public async getKernelSelections( resource: Resource, - cancelToken?: CancellationToken + _cancelToken?: CancellationToken ): Promise { - const items = await this.kernelFinder.listKernelSpecs(resource, cancelToken); + const items = await this.kernelFinder.listKernelSpecs(resource); return items .filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) .map((item) => getQuickPickItemForKernelSpec(item, this.pathUtils)); diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index bf2e57fad537..89059691411f 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -109,20 +109,17 @@ export class KernelFinder implements IKernelFinder { } // Search all our local file system locations for installed kernel specs and return them - public async listKernelSpecs(resource: Resource, _cancelToken?: CancellationToken): Promise { + public async listKernelSpecs(resource: Resource): Promise { // If we have not already searched for this resource, then generate the search if (!this.resourceToKernels.has(resource)) { - this.resourceToKernels.set(resource, this.findResourceKernelSpecs(resource, _cancelToken)); + this.resourceToKernels.set(resource, this.findResourceKernelSpecs(resource)); } // ! as the has and set above verify that we have a return here return this.resourceToKernels.get(resource)!; } - private async findResourceKernelSpecs( - resource: Resource, - _cancelToken?: CancellationToken - ): Promise { + private async findResourceKernelSpecs(resource: Resource): Promise { const results: IJupyterKernelSpec[] = []; // Find all the possible places to look for this resource diff --git a/src/client/datascience/kernel-launcher/types.ts b/src/client/datascience/kernel-launcher/types.ts index 2eda298a6068..f6340f32b05c 100644 --- a/src/client/datascience/kernel-launcher/types.ts +++ b/src/client/datascience/kernel-launcher/types.ts @@ -49,7 +49,7 @@ export interface IKernelFinder { kernelName?: string, cancelToken?: CancellationToken ): Promise; - listKernelSpecs(resource: Resource, cancelToken?: CancellationToken): Promise; + listKernelSpecs(resource: Resource): Promise; } /** From 3a8aba2e27fee4a0f8bed749919d66162d36e74c Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 1 May 2020 16:14:28 -0700 Subject: [PATCH 12/18] use workspace folder not resource --- .../kernel-launcher/kernelFinder.ts | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 89059691411f..d2b16379f7e9 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -6,6 +6,7 @@ import { Kernel } from '@jupyterlab/services'; import { inject, injectable, named } from 'inversify'; import * as path from 'path'; import { CancellationToken, CancellationTokenSource } from 'vscode'; +import { IWorkspaceService } from '../../common/application/types'; import { wrapCancellationTokens } from '../../common/cancellation'; import { traceError, traceInfo } from '../../common/logger'; import { IFileSystem, IPlatformService } from '../../common/platform/types'; @@ -45,7 +46,8 @@ export class KernelFinder implements IKernelFinder { private cache: string[] = []; // Store our results when listing all possible kernelspecs for a resource - private resourceToKernels = new Map>(); + //private resourceToKernels = new Map>(); + private workspaceToKernels = new Map>(); // Store any json file that we have loaded from disk before private pathToKernelSpec = new Map>(); @@ -59,7 +61,8 @@ export class KernelFinder implements IKernelFinder { @inject(IFileSystem) private file: IFileSystem, @inject(IPathUtils) private readonly pathUtils: IPathUtils, @inject(IInstaller) private installer: IInstaller, - @inject(IExtensionContext) private readonly context: IExtensionContext + @inject(IExtensionContext) private readonly context: IExtensionContext, + @inject(IWorkspaceService) private readonly workspaceService: IWorkspaceService ) {} @captureTelemetry(Telemetry.KernelFinderPerf) @@ -110,13 +113,21 @@ export class KernelFinder implements IKernelFinder { // Search all our local file system locations for installed kernel specs and return them public async listKernelSpecs(resource: Resource): Promise { + if (!resource) { + // We need a resource to search for related kernel specs + return []; + } + + // Get an id for the workspace folder, if we don't have one, use the fsPath of the resource + const workspaceFolderId = this.workspaceService.getWorkspaceFolderIdentifier(resource, resource.fsPath); + // If we have not already searched for this resource, then generate the search - if (!this.resourceToKernels.has(resource)) { - this.resourceToKernels.set(resource, this.findResourceKernelSpecs(resource)); + if (!this.workspaceToKernels.has(workspaceFolderId)) { + this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource)); } // ! as the has and set above verify that we have a return here - return this.resourceToKernels.get(resource)!; + return this.workspaceToKernels.get(workspaceFolderId)!; } private async findResourceKernelSpecs(resource: Resource): Promise { From 1a011c01016c0aee2fffe5108e89bc9881feeb20 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Sat, 2 May 2020 05:21:59 -0700 Subject: [PATCH 13/18] basic test passing --- .../datascience/kernelFinder.unit.test.ts | 676 ++++++++++++------ 1 file changed, 440 insertions(+), 236 deletions(-) diff --git a/src/test/datascience/kernelFinder.unit.test.ts b/src/test/datascience/kernelFinder.unit.test.ts index eac3c40e658c..dbff6978b373 100644 --- a/src/test/datascience/kernelFinder.unit.test.ts +++ b/src/test/datascience/kernelFinder.unit.test.ts @@ -9,12 +9,14 @@ import { anything, instance, mock, when } from 'ts-mockito'; import * as typemoq from 'typemoq'; import { Uri } from 'vscode'; +import { IWorkspaceService } from '../../client/common/application/types'; import { IFileSystem, IPlatformService } from '../../client/common/platform/types'; import { IExtensionContext, IInstaller, IPathUtils, Resource } from '../../client/common/types'; import { Architecture } from '../../client/common/utils/platform'; import { JupyterKernelSpec } from '../../client/datascience/jupyter/kernels/jupyterKernelSpec'; import { KernelFinder } from '../../client/datascience/kernel-launcher/kernelFinder'; import { IKernelFinder } from '../../client/datascience/kernel-launcher/types'; +import { IJupyterKernelSpec } from '../../client/datascience/types'; import { IInterpreterLocatorService, IInterpreterService, @@ -22,7 +24,7 @@ import { PythonInterpreter } from '../../client/interpreter/contracts'; -suite('IANHU Kernel Finder', () => { +suite('Kernel Finder', () => { let interpreterService: typemoq.IMock; let interpreterLocator: typemoq.IMock; let fileSystem: typemoq.IMock; @@ -30,6 +32,7 @@ suite('IANHU Kernel Finder', () => { let pathUtils: typemoq.IMock; let context: typemoq.IMock; let installer: IInstaller; + let workspaceService: IWorkspaceService; let kernelFinder: IKernelFinder; let activeInterpreter: PythonInterpreter; const interpreters: PythonInterpreter[] = []; @@ -63,264 +66,465 @@ suite('IANHU Kernel Finder', () => { ); } - setup(() => { - interpreterService = typemoq.Mock.ofType(); - interpreterService - .setup((is) => is.getActiveInterpreter(typemoq.It.isAny())) - .returns(() => Promise.resolve(activeInterpreter)); - interpreterService - .setup((is) => is.getInterpreterDetails(typemoq.It.isAny())) - .returns(() => Promise.resolve(activeInterpreter)); - - interpreterLocator = typemoq.Mock.ofType(); - interpreterLocator - .setup((il) => il.getInterpreters(typemoq.It.isAny(), typemoq.It.isAny())) - .returns(() => Promise.resolve(interpreters)); - - fileSystem = typemoq.Mock.ofType(); - platformService = typemoq.Mock.ofType(); - platformService.setup((ps) => ps.isWindows).returns(() => true); - platformService.setup((ps) => ps.isMac).returns(() => true); + function setupFindFileSystem() { + fileSystem + .setup((fs) => fs.writeFile(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .returns(() => Promise.resolve()); + fileSystem.setup((fs) => fs.getSubDirectories(typemoq.It.isAnyString())).returns(() => Promise.resolve([''])); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .returns(() => + Promise.resolve([ + path.join(kernel.name, 'kernel.json'), + path.join('kernelA', 'kernel.json'), + path.join('kernelB', 'kernel.json') + ]) + ); + } + setup(() => { pathUtils = typemoq.Mock.ofType(); pathUtils.setup((pu) => pu.home).returns(() => './'); context = typemoq.Mock.ofType(); context.setup((c) => c.globalStoragePath).returns(() => './'); + fileSystem = typemoq.Mock.ofType(); installer = mock(); when(installer.isInstalled(anything(), anything())).thenResolve(true); - activeInterpreter = { - path: context.object.globalStoragePath, - displayName: 'activeInterpreter', - sysPrefix: '1', - envName: '1', - sysVersion: '3.1.1.1', - architecture: Architecture.x64, - type: InterpreterType.Unknown - }; - for (let i = 0; i < 10; i += 1) { - interpreters.push({ - path: `${context.object.globalStoragePath}_${i}`, - sysPrefix: '1', + platformService = typemoq.Mock.ofType(); + platformService.setup((ps) => ps.isWindows).returns(() => true); + platformService.setup((ps) => ps.isMac).returns(() => true); + }); + + suite('listKernelSpecs', () => { + let activeKernelA: IJupyterKernelSpec; + let activeKernelB: IJupyterKernelSpec; + let interpreter0Kernel: IJupyterKernelSpec; + let interpreter1Kernel: IJupyterKernelSpec; + let globalKernel: IJupyterKernelSpec; + setup(() => { + activeInterpreter = { + path: context.object.globalStoragePath, + displayName: 'activeInterpreter', + sysPrefix: 'active', envName: '1', sysVersion: '3.1.1.1', architecture: Architecture.x64, type: InterpreterType.Unknown - }); - } - interpreters.push(activeInterpreter); - resource = Uri.file(context.object.globalStoragePath); - - kernelFinder = new KernelFinder( - interpreterService.object, - interpreterLocator.object, - platformService.object, - fileSystem.object, - pathUtils.object, - instance(installer), - context.object - ); - }); + }; + for (let i = 0; i < 2; i += 1) { + interpreters.push({ + path: `${context.object.globalStoragePath}_${i}`, + sysPrefix: `Interpreter${i}`, + envName: '1', + sysVersion: '3.1.1.1', + architecture: Architecture.x64, + type: InterpreterType.Unknown + }); + } - test('KernelSpec is in cache', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((param: string) => { - if (param.includes(cacheFile)) { - return Promise.resolve(`["${kernel.name}"]`); - } - return Promise.resolve(JSON.stringify(kernel)); - }); - const spec = await kernelFinder.findKernelSpec(resource, kernelName); - assert.deepEqual(spec, kernel, 'The found kernel spec is not the same.'); - fileSystem.reset(); - }); + // Our defaultresource + resource = Uri.file('abc'); - test('KernelSpec is in the active interpreter', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve(JSON.stringify(kernel)); - }); - const spec = await kernelFinder.findKernelSpec(resource, kernelName); - expect(spec).to.deep.include(kernel); - fileSystem.reset(); - }); + // Set our active interpreter + interpreterService = typemoq.Mock.ofType(); + interpreterService + .setup((is) => is.getActiveInterpreter(typemoq.It.isAny())) + .returns(() => Promise.resolve(activeInterpreter)); - test('No kernel name given. Default spec returned should match the interpreter selected.', async () => { - setupFileSystem(); - - // Create a second active interpreter to return on the second call - const activeInterpreter2 = { - path: context.object.globalStoragePath, - displayName: 'activeInterpreter2', - sysPrefix: '1', - envName: '1', - sysVersion: '3.1.1.1', - architecture: Architecture.x64, - type: InterpreterType.Unknown - }; - // Record a second call to getActiveInterpreter, will play after the first - interpreterService - .setup((is) => is.getActiveInterpreter(typemoq.It.isAny())) - .returns(() => Promise.resolve(activeInterpreter2)); + // Set our workspace interpreters + interpreterLocator = typemoq.Mock.ofType(); + interpreterLocator + .setup((il) => il.getInterpreters(typemoq.It.isAny(), typemoq.It.isAny())) + .returns(() => Promise.resolve(interpreters)); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve(JSON.stringify(kernel)); - }); - let spec = await kernelFinder.findKernelSpec(resource); - expect(spec.display_name).to.equal(activeInterpreter.displayName); - - spec = await kernelFinder.findKernelSpec(resource); - expect(spec.display_name).to.equal(activeInterpreter2.displayName); - fileSystem.reset(); - }); + activeKernelA = { + name: 'activeKernelA', + language: 'python', + path: '', + display_name: 'Python 3', + metadata: {}, + env: {}, + argv: ['', '-m', 'ipykernel_launcher', '-f', '{connection_file}'] + }; - test('KernelSpec is in the interpreters', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) - .returns(() => Promise.resolve([])); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve(JSON.stringify(kernel)); - }); - const spec = await kernelFinder.findKernelSpec(activeInterpreter, kernelName); - expect(spec).to.deep.include(kernel); - fileSystem.reset(); - }); + activeKernelB = { + name: 'activeKernelB', + language: 'python', + path: '', + display_name: 'Python 3', + metadata: {}, + env: {}, + argv: ['', '-m', 'ipykernel_launcher', '-f', '{connection_file}'] + }; - test('KernelSpec is in disk', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) - .returns(() => Promise.resolve([kernelName])); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve(JSON.stringify(kernel)); - }); - const spec = await kernelFinder.findKernelSpec(activeInterpreter, kernelName); - expect(spec).to.deep.include(kernel); - fileSystem.reset(); - }); + interpreter0Kernel = { + name: 'interpreter0Kernel', + language: 'python', + path: '', + display_name: 'Python 3', + metadata: {}, + env: {}, + argv: ['', '-m', 'ipykernel_launcher', '-f', '{connection_file}'] + }; - test('KernelSpec not found, returning default', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve('{}'); - }); - // get default kernel - const spec = await kernelFinder.findKernelSpec(resource); - assert.equal(spec.name.includes('python_defaultSpec'), true); - fileSystem.reset(); - }); + interpreter1Kernel = { + name: 'interpreter1Kernel', + language: 'python', + path: '', + display_name: 'Python 3', + metadata: {}, + env: {}, + argv: ['', '-m', 'ipykernel_launcher', '-f', '{connection_file}'] + }; - test('KernelSpec not found, returning default, then search for it again and find it in the cache', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } - return Promise.resolve('{}'); - }); - - // get default kernel - const spec = await kernelFinder.findKernelSpec(resource); - assert.equal(spec.name.includes('python_defaultSpec'), true); - fileSystem.reset(); - - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve(`["${spec.path}"]`); - } - return Promise.resolve(JSON.stringify(spec)); - }) - .verifiable(typemoq.Times.once()); - - // get the same kernel, but from cache - const spec2 = await kernelFinder.findKernelSpec(resource, spec.name); - assert.notStrictEqual(spec, spec2); - - fileSystem.verifyAll(); - fileSystem.reset(); + globalKernel = { + name: 'globalKernel', + language: 'python', + path: '', + display_name: 'Python 3', + metadata: {}, + env: {}, + argv: ['', '-m', 'ipykernel_launcher', '-f', '{connection_file}'] + }; + + workspaceService = mock(); + when(workspaceService.getWorkspaceFolderIdentifier(anything(), resource.fsPath)).thenReturn( + resource.fsPath + ); + + // Setup file system + const activePath = 'active/share/jupyter/kernels'; + const activePathA = path.join(activePath, activeKernelA.name, 'kernel.json'); + const activePathB = path.join(activePath, activeKernelB.name, 'kernel.json'); + fileSystem + .setup((fs) => fs.writeFile(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .returns(() => Promise.resolve()); + fileSystem + .setup((fs) => fs.getSubDirectories(typemoq.It.isAnyString())) + .returns(() => Promise.resolve([''])); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), activePath)) + .returns(() => + Promise.resolve([ + path.join(activeKernelA.name, 'kernel.json'), + path.join(activeKernelB.name, 'kernel.json') + ]) + ); + const interpreter0Path = 'Interpreter0/share/jupyter/kernels'; + const interpreter0FullPath = path.join(interpreter0Path, interpreter0Kernel.name, 'kernel.json'); + const interpreter1Path = 'Interpreter1/share/jupyter/kernels'; + const interpreter1FullPath = path.join(interpreter1Path, interpreter1Kernel.name, 'kernel.json'); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), interpreter0Path)) + .returns(() => Promise.resolve([path.join(interpreter0Kernel.name, 'kernel.json')])); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), interpreter1Path)) + .returns(() => Promise.resolve([path.join(interpreter1Kernel.name, 'kernel.json')])); + + const globalPath = 'AppData/Roaming/jupyter/kernels'; + const globalFullPath = path.join(globalPath, globalKernel.name, 'kernel.json'); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), globalPath)) + .returns(() => Promise.resolve([path.join(globalKernel.name, 'kernel.json')])); + + // Set the file system to return our kernelspec json + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((param: string) => { + switch (param) { + case activePathA: + return Promise.resolve(JSON.stringify(activeKernelA)); + case activePathB: + return Promise.resolve(JSON.stringify(activeKernelB)); + case interpreter0FullPath: + return Promise.resolve(JSON.stringify(interpreter0Kernel)); + case interpreter1FullPath: + return Promise.resolve(JSON.stringify(interpreter1Kernel)); + case globalFullPath: + return Promise.resolve(JSON.stringify(globalKernel)); + default: + return Promise.resolve(''); + } + }); + + kernelFinder = new KernelFinder( + interpreterService.object, + interpreterLocator.object, + platformService.object, + fileSystem.object, + pathUtils.object, + instance(installer), + context.object, + instance(workspaceService) + ); + }); + + test('IANHU Basic listKernelSpecs', async () => { + setupFindFileSystem(); + const specs = await kernelFinder.listKernelSpecs(resource); + expect(specs[0]).to.deep.include(activeKernelA); + expect(specs[1]).to.deep.include(activeKernelB); + expect(specs[2]).to.deep.include(interpreter0Kernel); + expect(specs[3]).to.deep.include(interpreter1Kernel); + expect(specs[4]).to.deep.include(globalKernel); + fileSystem.reset(); + }); }); - test('Look for KernelA with no cache, find KernelA and KenelB, then search for KernelB and find it in cache', async () => { - setupFileSystem(); - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve('[]'); - } else if (pathParam.includes('kernelA')) { - const specA = { - ...kernel, - name: 'kernelA' - }; - return Promise.resolve(JSON.stringify(specA)); - } - return Promise.resolve(''); - }); - - const spec = await kernelFinder.findKernelSpec(resource, 'kernelA'); - assert.equal(spec.name.includes('kernelA'), true); - fileSystem.reset(); - - setupFileSystem(); - fileSystem - .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) - .verifiable(typemoq.Times.never()); // this never executing means the kernel was found in cache - fileSystem - .setup((fs) => fs.readFile(typemoq.It.isAnyString())) - .returns((pathParam: string) => { - if (pathParam.includes(cacheFile)) { - return Promise.resolve( - JSON.stringify([ - path.join('kernels', kernel.name, 'kernel.json'), - path.join('kernels', 'kernelA', 'kernel.json'), - path.join('kernels', 'kernelB', 'kernel.json') - ]) - ); - } else if (pathParam.includes('kernelB')) { - const specB = { - ...kernel, - name: 'kernelB' - }; - return Promise.resolve(JSON.stringify(specB)); - } - return Promise.resolve('{}'); - }); - const spec2 = await kernelFinder.findKernelSpec(resource, 'kernelB'); - assert.equal(spec2.name.includes('kernelB'), true); + suite('findKernelSpec', () => { + setup(() => { + interpreterService = typemoq.Mock.ofType(); + interpreterService + .setup((is) => is.getActiveInterpreter(typemoq.It.isAny())) + .returns(() => Promise.resolve(activeInterpreter)); + interpreterService + .setup((is) => is.getInterpreterDetails(typemoq.It.isAny())) + .returns(() => Promise.resolve(activeInterpreter)); + + interpreterLocator = typemoq.Mock.ofType(); + interpreterLocator + .setup((il) => il.getInterpreters(typemoq.It.isAny(), typemoq.It.isAny())) + .returns(() => Promise.resolve(interpreters)); + + fileSystem = typemoq.Mock.ofType(); + + activeInterpreter = { + path: context.object.globalStoragePath, + displayName: 'activeInterpreter', + sysPrefix: '1', + envName: '1', + sysVersion: '3.1.1.1', + architecture: Architecture.x64, + type: InterpreterType.Unknown + }; + for (let i = 0; i < 10; i += 1) { + interpreters.push({ + path: `${context.object.globalStoragePath}_${i}`, + sysPrefix: '1', + envName: '1', + sysVersion: '3.1.1.1', + architecture: Architecture.x64, + type: InterpreterType.Unknown + }); + } + interpreters.push(activeInterpreter); + resource = Uri.file(context.object.globalStoragePath); + + workspaceService = mock(); + + kernelFinder = new KernelFinder( + interpreterService.object, + interpreterLocator.object, + platformService.object, + fileSystem.object, + pathUtils.object, + instance(installer), + context.object, + instance(workspaceService) + ); + }); + + test('KernelSpec is in cache', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((param: string) => { + if (param.includes(cacheFile)) { + return Promise.resolve(`["${kernel.name}"]`); + } + return Promise.resolve(JSON.stringify(kernel)); + }); + const spec = await kernelFinder.findKernelSpec(resource, kernelName); + assert.deepEqual(spec, kernel, 'The found kernel spec is not the same.'); + fileSystem.reset(); + }); + + test('KernelSpec is in the active interpreter', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve(JSON.stringify(kernel)); + }); + const spec = await kernelFinder.findKernelSpec(resource, kernelName); + expect(spec).to.deep.include(kernel); + fileSystem.reset(); + }); + + test('No kernel name given. Default spec returned should match the interpreter selected.', async () => { + setupFileSystem(); + + // Create a second active interpreter to return on the second call + const activeInterpreter2 = { + path: context.object.globalStoragePath, + displayName: 'activeInterpreter2', + sysPrefix: '1', + envName: '1', + sysVersion: '3.1.1.1', + architecture: Architecture.x64, + type: InterpreterType.Unknown + }; + // Record a second call to getActiveInterpreter, will play after the first + interpreterService + .setup((is) => is.getActiveInterpreter(typemoq.It.isAny())) + .returns(() => Promise.resolve(activeInterpreter2)); + + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve(JSON.stringify(kernel)); + }); + let spec = await kernelFinder.findKernelSpec(resource); + expect(spec.display_name).to.equal(activeInterpreter.displayName); + + spec = await kernelFinder.findKernelSpec(resource); + expect(spec.display_name).to.equal(activeInterpreter2.displayName); + fileSystem.reset(); + }); + + test('KernelSpec is in the interpreters', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .returns(() => Promise.resolve([])); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve(JSON.stringify(kernel)); + }); + const spec = await kernelFinder.findKernelSpec(activeInterpreter, kernelName); + expect(spec).to.deep.include(kernel); + fileSystem.reset(); + }); + + test('KernelSpec is in disk', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .returns(() => Promise.resolve([kernelName])); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve(JSON.stringify(kernel)); + }); + const spec = await kernelFinder.findKernelSpec(activeInterpreter, kernelName); + expect(spec).to.deep.include(kernel); + fileSystem.reset(); + }); + + test('KernelSpec not found, returning default', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve('{}'); + }); + // get default kernel + const spec = await kernelFinder.findKernelSpec(resource); + assert.equal(spec.name.includes('python_defaultSpec'), true); + fileSystem.reset(); + }); + + test('KernelSpec not found, returning default, then search for it again and find it in the cache', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } + return Promise.resolve('{}'); + }); + + // get default kernel + const spec = await kernelFinder.findKernelSpec(resource); + assert.equal(spec.name.includes('python_defaultSpec'), true); + fileSystem.reset(); + + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve(`["${spec.path}"]`); + } + return Promise.resolve(JSON.stringify(spec)); + }) + .verifiable(typemoq.Times.once()); + + // get the same kernel, but from cache + const spec2 = await kernelFinder.findKernelSpec(resource, spec.name); + assert.notStrictEqual(spec, spec2); + + fileSystem.verifyAll(); + fileSystem.reset(); + }); + + test('Look for KernelA with no cache, find KernelA and KenelB, then search for KernelB and find it in cache', async () => { + setupFileSystem(); + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve('[]'); + } else if (pathParam.includes('kernelA')) { + const specA = { + ...kernel, + name: 'kernelA' + }; + return Promise.resolve(JSON.stringify(specA)); + } + return Promise.resolve(''); + }); + + const spec = await kernelFinder.findKernelSpec(resource, 'kernelA'); + assert.equal(spec.name.includes('kernelA'), true); + fileSystem.reset(); + + setupFileSystem(); + fileSystem + .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) + .verifiable(typemoq.Times.never()); // this never executing means the kernel was found in cache + fileSystem + .setup((fs) => fs.readFile(typemoq.It.isAnyString())) + .returns((pathParam: string) => { + if (pathParam.includes(cacheFile)) { + return Promise.resolve( + JSON.stringify([ + path.join('kernels', kernel.name, 'kernel.json'), + path.join('kernels', 'kernelA', 'kernel.json'), + path.join('kernels', 'kernelB', 'kernel.json') + ]) + ); + } else if (pathParam.includes('kernelB')) { + const specB = { + ...kernel, + name: 'kernelB' + }; + return Promise.resolve(JSON.stringify(specB)); + } + return Promise.resolve('{}'); + }); + const spec2 = await kernelFinder.findKernelSpec(resource, 'kernelB'); + assert.equal(spec2.name.includes('kernelB'), true); + }); }); }); From 3ec2a81d9b1e3480deaf19ad175fdc0c4425d6cf Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Sat, 2 May 2020 05:47:11 -0700 Subject: [PATCH 14/18] couple of basic unit tests for find --- .../datascience/kernelFinder.unit.test.ts | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/test/datascience/kernelFinder.unit.test.ts b/src/test/datascience/kernelFinder.unit.test.ts index dbff6978b373..82ea20ea9093 100644 --- a/src/test/datascience/kernelFinder.unit.test.ts +++ b/src/test/datascience/kernelFinder.unit.test.ts @@ -35,7 +35,7 @@ suite('Kernel Finder', () => { let workspaceService: IWorkspaceService; let kernelFinder: IKernelFinder; let activeInterpreter: PythonInterpreter; - const interpreters: PythonInterpreter[] = []; + let interpreters: PythonInterpreter[] = []; let resource: Resource; const kernelName = 'testKernel'; const cacheFile = 'kernelSpecPathCache.json'; @@ -71,15 +71,6 @@ suite('Kernel Finder', () => { .setup((fs) => fs.writeFile(typemoq.It.isAnyString(), typemoq.It.isAnyString())) .returns(() => Promise.resolve()); fileSystem.setup((fs) => fs.getSubDirectories(typemoq.It.isAnyString())).returns(() => Promise.resolve([''])); - fileSystem - .setup((fs) => fs.search(typemoq.It.isAnyString(), typemoq.It.isAnyString())) - .returns(() => - Promise.resolve([ - path.join(kernel.name, 'kernel.json'), - path.join('kernelA', 'kernel.json'), - path.join('kernelB', 'kernel.json') - ]) - ); } setup(() => { @@ -104,6 +95,7 @@ suite('Kernel Finder', () => { let interpreter0Kernel: IJupyterKernelSpec; let interpreter1Kernel: IJupyterKernelSpec; let globalKernel: IJupyterKernelSpec; + let loadError = false; setup(() => { activeInterpreter = { path: context.object.globalStoragePath, @@ -114,6 +106,7 @@ suite('Kernel Finder', () => { architecture: Architecture.x64, type: InterpreterType.Unknown }; + interpreters = []; for (let i = 0; i < 2; i += 1) { interpreters.push({ path: `${context.object.globalStoragePath}_${i}`, @@ -236,7 +229,11 @@ suite('Kernel Finder', () => { .returns((param: string) => { switch (param) { case activePathA: - return Promise.resolve(JSON.stringify(activeKernelA)); + if (!loadError) { + return Promise.resolve(JSON.stringify(activeKernelA)); + } else { + return Promise.resolve(''); + } case activePathB: return Promise.resolve(JSON.stringify(activeKernelB)); case interpreter0FullPath: @@ -262,7 +259,7 @@ suite('Kernel Finder', () => { ); }); - test('IANHU Basic listKernelSpecs', async () => { + test('Basic listKernelSpecs', async () => { setupFindFileSystem(); const specs = await kernelFinder.listKernelSpecs(resource); expect(specs[0]).to.deep.include(activeKernelA); @@ -272,6 +269,17 @@ suite('Kernel Finder', () => { expect(specs[4]).to.deep.include(globalKernel); fileSystem.reset(); }); + + test('listKernelSpecs load error', async () => { + setupFindFileSystem(); + loadError = true; + const specs = await kernelFinder.listKernelSpecs(resource); + expect(specs[0]).to.deep.include(activeKernelB); + expect(specs[1]).to.deep.include(interpreter0Kernel); + expect(specs[2]).to.deep.include(interpreter1Kernel); + expect(specs[3]).to.deep.include(globalKernel); + fileSystem.reset(); + }); }); suite('findKernelSpec', () => { From 00c649969f3ba88a0887f5d8994fc3407af580b4 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Sat, 2 May 2020 05:49:48 -0700 Subject: [PATCH 15/18] remove old comments --- .vscode/launch.json | 1 - src/client/datascience/kernel-launcher/kernelFinder.ts | 1 - 2 files changed, 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index b72b20e2071c..95cc798affcb 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -240,7 +240,6 @@ "--recursive", "--colors", // "--grep", "", - "--grep", "IANHU", "--timeout=300000", "--fast" ], diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index d2b16379f7e9..61aabdedfaa7 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -46,7 +46,6 @@ export class KernelFinder implements IKernelFinder { private cache: string[] = []; // Store our results when listing all possible kernelspecs for a resource - //private resourceToKernels = new Map>(); private workspaceToKernels = new Map>(); // Store any json file that we have loaded from disk before From 893d8d9fda7bf36eb63204522aced74fd3aae483 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Sat, 2 May 2020 06:31:57 -0700 Subject: [PATCH 16/18] de-dupe interpreters list --- src/client/datascience/kernel-launcher/kernelFinder.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 61aabdedfaa7..37dc5d59b110 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -213,7 +213,9 @@ export class KernelFinder implements IKernelFinder { private async getInterpreterPaths(resource: Resource): Promise { const interpreters = await this.interpreterLocator.getInterpreters(resource, { ignoreCache: false }); const interpreterPrefixPaths = interpreters.map((interpreter) => interpreter.sysPrefix); - return interpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, baseKernelPath)); + // We can get many duplicates here, so de-dupe the list + const uniqueInterpreterPrefixPaths = [...new Set(interpreterPrefixPaths)]; + return uniqueInterpreterPrefixPaths.map((prefixPath) => path.join(prefixPath, baseKernelPath)); } private async getDiskPaths(): Promise { From dcc6efacd5f5a59b700020a956aef07b49b879bf Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 4 May 2020 09:45:21 -0700 Subject: [PATCH 17/18] revert glob pattern back to slash --- src/client/datascience/kernel-launcher/kernelFinder.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 37dc5d59b110..28b6ff23b54b 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -243,7 +243,7 @@ export class KernelFinder implements IKernelFinder { // Given a set of paths, search for kernel.json files and return back the full paths of all of them that we find private async kernelGlobSearch(paths: string[]): Promise { - const promises = paths.map((kernelPath) => this.file.search(`**${path.sep}kernel.json`, kernelPath)); + const promises = paths.map((kernelPath) => this.file.search(`**/kernel.json`, kernelPath)); const searchResults = await Promise.all(promises); // Append back on the start of each path so we have the full path in the results From 6fcd79a8bf5990edc337be5741a0ea0e31e3bc7c Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 4 May 2020 13:11:28 -0700 Subject: [PATCH 18/18] only write cache if it changes, and update cache after we do our find --- .../kernel-launcher/kernelFinder.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelFinder.ts b/src/client/datascience/kernel-launcher/kernelFinder.ts index 28b6ff23b54b..70140a6eb80e 100644 --- a/src/client/datascience/kernel-launcher/kernelFinder.ts +++ b/src/client/datascience/kernel-launcher/kernelFinder.ts @@ -44,6 +44,7 @@ export function findIndexOfConnectionFile(kernelSpec: Readonly>(); @@ -125,6 +126,8 @@ export class KernelFinder implements IKernelFinder { this.workspaceToKernels.set(workspaceFolderId, this.findResourceKernelSpecs(resource)); } + this.writeCache(this.cache).ignoreErrors(); + // ! as the has and set above verify that we have a return here return this.workspaceToKernels.get(workspaceFolderId)!; } @@ -140,9 +143,7 @@ export class KernelFinder implements IKernelFinder { await Promise.all( searchResults.map(async (resultPath) => { // Add these into our path cache to speed up later finds - if (!this.cache.includes(resultPath)) { - this.cache.push(resultPath); - } + this.updateCache(resultPath); const kernelspec = await this.getKernelSpec(resultPath); if (kernelspec) { @@ -309,9 +310,7 @@ export class KernelFinder implements IKernelFinder { private async getKernelSpecFromDisk(paths: string[], kernelName: string): Promise { const searchResults = await this.kernelGlobSearch(paths); searchResults.forEach((specPath) => { - if (!this.cache.includes(specPath)) { - this.cache.push(specPath); - } + this.updateCache(specPath); }); return this.searchCache(kernelName); @@ -345,8 +344,18 @@ export class KernelFinder implements IKernelFinder { } } + private updateCache(newPath: string) { + if (!this.cache.includes(newPath)) { + this.cache.push(newPath); + this.cacheDirty = true; + } + } + private async writeCache(cache: string[]) { - await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache)); + if (this.cacheDirty) { + await this.file.writeFile(path.join(this.context.globalStoragePath, cacheFile), JSON.stringify(cache)); + this.cacheDirty = false; + } } private async searchCache(kernelName: string): Promise {