diff --git a/build/ci/templates/steps/build_compile.yml b/build/ci/templates/steps/build_compile.yml index 1ada0fcb400c..5abf5c0bcdaa 100644 --- a/build/ci/templates/steps/build_compile.yml +++ b/build/ci/templates/steps/build_compile.yml @@ -36,15 +36,6 @@ steps: displayName: "pip install requirements" condition: and(succeeded(), eq(variables['build'], 'true')) - - task: PythonScript@0 - displayName: "Install PTVSD wheels" - inputs: - scriptSource: "filePath" - scriptPath: "./pythonFiles/install_ptvsd.py" - arguments: "--ci" - failOnStderr: true - condition: and(succeeded(), eq(variables['build'], 'true')) - - bash: npm run clean displayName: "Clean" condition: and(succeeded(), eq(variables['build'], 'true')) diff --git a/build/ci/templates/test_phases.yml b/build/ci/templates/test_phases.yml index 532be9bef2a6..1b4e9fd15657 100644 --- a/build/ci/templates/test_phases.yml +++ b/build/ci/templates/test_phases.yml @@ -138,6 +138,15 @@ steps: displayName: 'pip install ipython requirements' condition: and(succeeded(), eq(variables['NeedsIPythonReqs'], 'true')) + - task: PythonScript@0 + displayName: "Install PTVSD wheels" + inputs: + scriptSource: "filePath" + scriptPath: "./pythonFiles/install_ptvsd.py" + arguments: "--ci" + failOnStderr: true + condition: contains(variables['TestsToRun'], 'testUnitTests') + # Run the Python unit tests in our codebase. Produces a JUnit-style log file that # will be uploaded after all tests are complete. # diff --git a/build/test-requirements.txt b/build/test-requirements.txt index 37abe5e0a827..ffc4cf1d3287 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -15,3 +15,4 @@ rope flask django isort +packaging==19.2 diff --git a/pythonFiles/install_ptvsd.py b/pythonFiles/install_ptvsd.py index 7065293f6018..5521b4e8dd4d 100644 --- a/pythonFiles/install_ptvsd.py +++ b/pythonFiles/install_ptvsd.py @@ -5,25 +5,30 @@ import urllib.request import sys -ROOT_DIRNAME = path.dirname(path.dirname(path.abspath(__file__))) -REQUIREMENTS_PATH = path.join(ROOT_DIRNAME, "requirements.txt") -PYTHONFILES_PATH = path.join(ROOT_DIRNAME, "pythonFiles", "lib", "python") +ROOT = path.dirname(path.dirname(path.abspath(__file__))) +REQUIREMENTS = path.join(ROOT, "requirements.txt") +PYTHONFILES = path.join(ROOT, "pythonFiles", "lib", "python") PYPI_PTVSD_URL = "https://pypi.org/pypi/ptvsd/json" def install_ptvsd(): - # If we are in CI use the packaging module installed in PYTHONFILES_PATH. + # If we are in CI use the packaging module installed in PYTHONFILES. if len(sys.argv) == 2 and sys.argv[1] == "--ci": - sys.path.insert(0, PYTHONFILES_PATH) + sys.path.insert(0, PYTHONFILES) from packaging.requirements import Requirement - with open(REQUIREMENTS_PATH, "r", encoding="utf-8") as requirements: - for line in requirements: - package_requirement = Requirement(line) - if package_requirement.name != "ptvsd": - continue - requirement_specifier = package_requirement.specifier - ptvsd_version = next(requirement_specifier.__iter__()).version + with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile: + for line in reqsfile: + pkgreq = Requirement(line) + if pkgreq.name == "ptvsd": + specs = pkgreq.specifier + version = next(iter(specs)).version + break + + try: + version + except NameError: + raise Exception("ptvsd requirement not found.") # Response format: https://warehouse.readthedocs.io/api-reference/json/#project with urllib.request.urlopen(PYPI_PTVSD_URL) as response: @@ -31,24 +36,25 @@ def install_ptvsd(): releases = json_response["releases"] # Release metadata format: https://github.com/pypa/interoperability-peps/blob/master/pep-0426-core-metadata.rst - for wheel_info in releases[ptvsd_version]: + for wheel_info in releases[version]: # Download only if it's a 3.7 wheel. if not wheel_info["python_version"].endswith(("37", "3.7")): continue filename = wheel_info["filename"].rpartition(".")[0] # Trim the file extension. - ptvsd_path = path.join(PYTHONFILES_PATH, filename) + ptvsd_path = path.join(PYTHONFILES, filename) with urllib.request.urlopen(wheel_info["url"]) as wheel_response: wheel_file = BytesIO(wheel_response.read()) - # Extract only the contents of the ptvsd subfolder. - prefix = path.join(f"ptvsd-{ptvsd_version}.data", "purelib", "ptvsd") + # Extract only the contents of the purelib subfolder (parent folder of ptvsd), + # since ptvsd files rely on the presence of a 'ptvsd' folder. + prefix = path.join(f"ptvsd-{version}.data", "purelib") with ZipFile(wheel_file, "r") as wheel: for zip_info in wheel.infolist(): - if not zip_info.filename.startswith(prefix): - continue + # Normalize path for Windows, the wheel folder structure uses forward slashes. + normalized = path.normpath(zip_info.filename) # Flatten the folder structure. - zip_info.filename = zip_info.filename.split(prefix)[-1] + zip_info.filename = normalized.split(prefix)[-1] wheel.extract(zip_info, ptvsd_path) diff --git a/pythonFiles/ptvsd_folder_name.py b/pythonFiles/ptvsd_folder_name.py new file mode 100644 index 000000000000..8b8dce53c52a --- /dev/null +++ b/pythonFiles/ptvsd_folder_name.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import sys +import os.path + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PYTHONFILES = os.path.join(ROOT, "pythonFiles", "lib", "python") +REQUIREMENTS = os.path.join(ROOT, "requirements.txt") + +sys.path.insert(0, PYTHONFILES) + +from packaging.requirements import Requirement +from packaging.tags import sys_tags + +sys.path.remove(PYTHONFILES) + + +def ptvsd_folder_name(): + """Return the folder name for the bundled PTVSD wheel compatible with the new debug adapter.""" + + with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile: + for line in reqsfile: + pkgreq = Requirement(line) + if pkgreq.name == "ptvsd": + specs = pkgreq.specifier + try: + spec, = specs + version = spec.version + except: + # Fallpack to use base PTVSD path. + print(PYTHONFILES, end="") + return + break + + try: + for tag in sys_tags(): + folder_name = f"ptvsd-{version}-{tag.interpreter}-{tag.abi}-{tag.platform}" + folder_path = os.path.join(PYTHONFILES, folder_name) + if os.path.exists(folder_path): + print(folder_path, end="") + return + except: + # Fallback to use base PTVSD path no matter the exception. + print(PYTHONFILES, end="") + return + + # Default fallback to use base PTVSD path. + print(PYTHONFILES, end="") + + +if __name__ == "__main__": + ptvsd_folder_name() diff --git a/pythonFiles/tests/__init__.py b/pythonFiles/tests/__init__.py index 5b7f7a925cc0..a400865e05bd 100644 --- a/pythonFiles/tests/__init__.py +++ b/pythonFiles/tests/__init__.py @@ -1,2 +1,13 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import os.path + +TEST_ROOT = os.path.dirname(__file__) +SRC_ROOT = os.path.dirname(TEST_ROOT) +PROJECT_ROOT = os.path.dirname(SRC_ROOT) +IPYTHON_ROOT = os.path.join(SRC_ROOT, "ipython") +TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, "testing_tools") +DEBUG_ADAPTER_ROOT = os.path.join(SRC_ROOT, "debug_adapter") + +PYTHONFILES = os.path.join(SRC_ROOT, "lib", "python") +REQUIREMENTS = os.path.join(PROJECT_ROOT, "requirements.txt") diff --git a/pythonFiles/tests/__main__.py b/pythonFiles/tests/__main__.py index 5b140cc521bb..14086978c9af 100644 --- a/pythonFiles/tests/__main__.py +++ b/pythonFiles/tests/__main__.py @@ -2,26 +2,22 @@ # Licensed under the MIT License. import argparse -import os.path import sys import pytest - -TEST_ROOT = os.path.dirname(__file__) -SRC_ROOT = os.path.dirname(TEST_ROOT) -PROJECT_ROOT = os.path.dirname(SRC_ROOT) -IPYTHON_ROOT = os.path.join(SRC_ROOT, 'ipython') -TESTING_TOOLS_ROOT = os.path.join(SRC_ROOT, 'testing_tools') +from . import DEBUG_ADAPTER_ROOT, IPYTHON_ROOT, SRC_ROOT, TEST_ROOT, TESTING_TOOLS_ROOT def parse_args(): parser = argparse.ArgumentParser() # To mark a test as functional: (decorator) @pytest.mark.functional - parser.add_argument('--functional', dest='markers', - action='append_const', const='functional') - parser.add_argument('--no-functional', dest='markers', - action='append_const', const='not functional') + parser.add_argument( + "--functional", dest="markers", action="append_const", const="functional" + ) + parser.add_argument( + "--no-functional", dest="markers", action="append_const", const="not functional" + ) args, remainder = parser.parse_known_args() ns = vars(args) @@ -32,20 +28,18 @@ def parse_args(): def main(pytestargs, markers=None): sys.path.insert(1, IPYTHON_ROOT) sys.path.insert(1, TESTING_TOOLS_ROOT) + sys.path.insert(1, DEBUG_ADAPTER_ROOT) - pytestargs = [ - '--rootdir', SRC_ROOT, - TEST_ROOT, - ] + pytestargs + pytestargs = ["--rootdir", SRC_ROOT, TEST_ROOT] + pytestargs for marker in reversed(markers or ()): pytestargs.insert(0, marker) - pytestargs.insert(0, '-m') + pytestargs.insert(0, "-m") ec = pytest.main(pytestargs) return ec -if __name__ == '__main__': +if __name__ == "__main__": mainkwargs, pytestargs = parse_args() ec = main(pytestargs, **mainkwargs) sys.exit(ec) diff --git a/pythonFiles/tests/debug_adapter/__init__.py b/pythonFiles/tests/debug_adapter/__init__.py new file mode 100644 index 000000000000..5b7f7a925cc0 --- /dev/null +++ b/pythonFiles/tests/debug_adapter/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. diff --git a/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name.py b/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name.py new file mode 100644 index 000000000000..e823d74d8e95 --- /dev/null +++ b/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name.py @@ -0,0 +1,81 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +import sys + +if sys.version_info[:2] != (3, 7): + import unittest + + raise unittest.SkipTest("PTVSD wheels shipped for Python 3.7 only") + +import os.path +import pytest +import re + +from unittest.mock import patch, mock_open +from packaging.tags import sys_tags +from ptvsd_folder_name import ptvsd_folder_name + +from .. import PYTHONFILES, REQUIREMENTS + + +def open_requirements_with_ptvsd(): + return patch( + "ptvsd_folder_name.open", mock_open(read_data="jedi==0.15.1\nptvsd==5.0.0") + ) + + +def open_requirements_without_ptvsd(): + return patch("ptvsd_folder_name.open", mock_open(read_data="jedi==0.15.1\n")) + + +class TestPtvsdFolderName: + """Unit tests for the script retrieving the PTVSD folder name for the PTVSD wheels experiment.""" + + def test_requirement_exists_folder_exists(self, capsys): + # Return the first constructed folder path as existing. + + patcher = patch("os.path.exists") + mock_exists = patcher.start() + mock_exists.side_effect = lambda p: True + tag = next(sys_tags()) + folder = "ptvsd-5.0.0-{}-{}-{}".format(tag.interpreter, tag.abi, tag.platform) + + with open_requirements_with_ptvsd(): + ptvsd_folder_name() + + patcher.stop() + expected = os.path.join(PYTHONFILES, folder) + captured = capsys.readouterr() + assert captured.out == expected + + def test_ptvsd_requirement_once(self): + reqs = [ + line + for line in open(REQUIREMENTS, "r", encoding="utf-8") + if re.match("ptvsd==", line) + ] + assert len(reqs) == 1 + + def test_no_ptvsd_requirement(self, capsys): + with open_requirements_without_ptvsd() as p: + ptvsd_folder_name() + + expected = PYTHONFILES + captured = capsys.readouterr() + assert captured.out == expected + + def test_no_wheel_folder(self, capsys): + # Return none of of the constructed paths as existing, + # ptvsd_folder_name() should return the path to default ptvsd. + patcher = patch("os.path.exists") + mock_no_exist = patcher.start() + mock_no_exist.side_effect = lambda p: False + + with open_requirements_with_ptvsd() as p: + ptvsd_folder_name() + + patcher.stop() + expected = PYTHONFILES + captured = capsys.readouterr() + assert captured.out == expected + diff --git a/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name_functional.py b/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name_functional.py new file mode 100644 index 000000000000..32231a74f744 --- /dev/null +++ b/pythonFiles/tests/debug_adapter/test_ptvsd_folder_name_functional.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import sys + +if sys.version_info[:2] != (3, 7): + import unittest + + raise unittest.SkipTest("PTVSD wheels shipped for Python 3.7 only") + +import os.path +import pytest +import subprocess + +from packaging.requirements import Requirement +from .. import PYTHONFILES, REQUIREMENTS, SRC_ROOT + +ARGV = ["python", os.path.join(SRC_ROOT, "ptvsd_folder_name.py")] +PREFIX = "ptvsd==" + +with open(REQUIREMENTS, "r", encoding="utf-8") as reqsfile: + for line in reqsfile: + if line.startswith(PREFIX): + VERSION = line[len(PREFIX) :].strip() + break + + +def ptvsd_paths(*platforms): + paths = set() + for platform in platforms: + folder = "ptvsd-{}-cp37-cp37m-{}".format(VERSION, platform) + paths.add(os.path.join(PYTHONFILES, folder)) + return paths + + +@pytest.mark.functional +class TestPtvsdFolderNameFunctional: + """Functional tests for the script retrieving the PTVSD folder name for the PTVSD wheels experiment.""" + + def test_ptvsd_folder_name_nofail(self): + output = subprocess.check_output(ARGV, universal_newlines=True) + assert output != PYTHONFILES + + @pytest.mark.skipif(sys.platform != "darwin", reason="macOS functional test") + def test_ptvsd_folder_name_macos(self): + output = subprocess.check_output(ARGV, universal_newlines=True) + assert output in ptvsd_paths("macosx_10_13_x86_64") + + @pytest.mark.skipif(sys.platform != "win32", reason="Windows functional test") + def test_ptvsd_folder_name_windows(self): + output = subprocess.check_output(ARGV, universal_newlines=True) + assert output in ptvsd_paths("win32", "win_amd64") + + @pytest.mark.skipif(sys.platform != "linux", reason="Linux functional test") + def test_ptvsd_folder_name_linux(self): + output = subprocess.check_output(ARGV, universal_newlines=True) + assert output in ptvsd_paths( + "manylinux1_i686", "manylinux1_x86_64", "manylinux2010_x86_64" + ) diff --git a/requirements.txt b/requirements.txt index 02c7f815e8b8..229c3e140f94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ isort==4.3.21 ptvsd==5.0.0a4 pyparsing==2.4.0 six==1.12.0 -packaging==19.1 +packaging==19.2 diff --git a/src/client/api.ts b/src/client/api.ts index 177dc997397d..511808c939c7 100644 --- a/src/client/api.ts +++ b/src/client/api.ts @@ -41,6 +41,8 @@ export function buildApi(ready: Promise) { return Promise.reject(ex); }), debug: { + // tslint:disable-next-line:no-suspicious-comment + // TODO: Add support for ptvsd wheels experiment, see https://github.com/microsoft/vscode-python/issues/7549 async getRemoteLauncherCommand(host: string, port: number, waitUntilDebuggerAttaches: boolean = true): Promise { return new RemoteDebuggerExternalLauncherScriptProvider().getLauncherArgs({ host, port, waitUntilDebuggerAttaches }); } diff --git a/src/client/debugger/extension/adapter/factory.ts b/src/client/debugger/extension/adapter/factory.ts index 73ee16730df5..243030663575 100644 --- a/src/client/debugger/extension/adapter/factory.ts +++ b/src/client/debugger/extension/adapter/factory.ts @@ -3,44 +3,86 @@ 'use strict'; +import * as fs from 'fs'; import { inject, injectable } from 'inversify'; +import * as path from 'path'; +import { parse } from 'semver'; +import { promisify } from 'util'; import { DebugAdapterDescriptor, DebugAdapterExecutable, DebugSession, WorkspaceFolder } from 'vscode'; import { IApplicationShell } from '../../../common/application/types'; +import { PVSC_EXTENSION_ID } from '../../../common/constants'; import { DebugAdapterNewPtvsd } from '../../../common/experimentGroups'; import { traceVerbose } from '../../../common/logger'; -import { IExperimentsManager } from '../../../common/types'; +import { IPythonExecutionFactory } from '../../../common/process/types'; +import { IExperimentsManager, IExtensions, IPersistentStateFactory } from '../../../common/types'; +import { EXTENSION_ROOT_DIR } from '../../../constants'; import { IInterpreterService } from '../../../interpreter/contracts'; import { AttachRequestArguments, LaunchRequestArguments } from '../../types'; -import { IDebugAdapterDescriptorFactory } from '../types'; +import { DebugAdapterPtvsdPathInfo, IDebugAdapterDescriptorFactory } from '../types'; + +export const ptvsdPathStorageKey = 'PTVSD_PATH_STORAGE_KEY'; @injectable() export class DebugAdapterDescriptorFactory implements IDebugAdapterDescriptorFactory { constructor( @inject(IInterpreterService) private readonly interpreterService: IInterpreterService, @inject(IApplicationShell) private readonly appShell: IApplicationShell, - @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager + @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager, + @inject(IPythonExecutionFactory) private readonly executionFactory: IPythonExecutionFactory, + @inject(IPersistentStateFactory) private readonly stateFactory: IPersistentStateFactory, + @inject(IExtensions) private readonly extensions: IExtensions ) {} public async createDebugAdapterDescriptor(session: DebugSession, executable: DebugAdapterExecutable | undefined): Promise { const configuration = session.configuration as (LaunchRequestArguments | AttachRequestArguments); const pythonPath = await this.getPythonPath(configuration, session.workspaceFolder); - const interpreterInfo = await this.interpreterService.getInterpreterDetails(pythonPath); - if (interpreterInfo && interpreterInfo.version && interpreterInfo.version.raw.startsWith('3.7') && this.experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)) { - traceVerbose('Compute and return the path to the correct PTVSD folder (use packaging module)'); - // const ptvsdPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'lib', 'python', 'ptvsd'); - // // tslint:disable-next-line: no-any - // const ptvsdPathToUse = 'ptvsd' in configuration ? (configuration as any).ptvsd : ptvsdPath; - // traceVerbose(`Using Python Debug Adapter with PTVSD ${ptvsdPathToUse}`); - // return new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPathToUse, 'adapter'), ...logArgs]); - return new DebugAdapterExecutable(pythonPath); + if (await this.useNewPtvsd(pythonPath)) { + // If logToFile is set in the debug config then pass --log-dir when launching the debug adapter. + const logArgs = configuration.logToFile ? ['--log-dir', EXTENSION_ROOT_DIR] : []; + const ptvsdPathToUse = await this.getPtvsdPath(pythonPath); + return new DebugAdapterExecutable(`${pythonPath}`, [ptvsdPathToUse, ...logArgs]); } + + // Use the Node debug adapter (and ptvsd_launcher.py) if (executable) { - traceVerbose('Using Node Debug Adapter'); return executable; } // Unlikely scenario. throw new Error('Debug Adapter Executable not provided'); } + + /** + * Check and return whether the user is in the PTVSD wheels experiment or not. + * + * @param {string} pythonPath Path to the python executable used to launch the Python Debug Adapter (result of `this.getPythonPath()`) + * @returns {Promise} Whether the user is in the experiment or not. + * @memberof DebugAdapterDescriptorFactory + */ + public async useNewPtvsd(pythonPath: string): Promise { + if (!this.experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)) { + return false; + } + + const interpreterInfo = await this.interpreterService.getInterpreterDetails(pythonPath); + if (!interpreterInfo || !interpreterInfo.version || !interpreterInfo.version.raw.startsWith('3.7')) { + return false; + } + + return true; + } + + public async getPtvsdPath(pythonPath: string): Promise { + let ptvsdPathToUse: string; + + try { + ptvsdPathToUse = await this.getPtvsdFolder(pythonPath); + } catch { + ptvsdPathToUse = path.join(EXTENSION_ROOT_DIR, 'pythonFiles'); + } + + return path.join(ptvsdPathToUse, 'ptvsd', 'adapter'); + } + /** * Get the python executable used to launch the Python Debug Adapter. * In the case of `attach` scenarios, just use the workspace interpreter, else first available one. @@ -49,7 +91,7 @@ export class DebugAdapterDescriptorFactory implements IDebugAdapterDescriptorFac * @private * @param {(LaunchRequestArguments | AttachRequestArguments)} configuration * @param {WorkspaceFolder} [workspaceFolder] - * @returns {Promise} + * @returns {Promise} Path to the python interpreter for this workspace. * @memberof DebugAdapterDescriptorFactory */ private async getPythonPath(configuration: LaunchRequestArguments | AttachRequestArguments, workspaceFolder?: WorkspaceFolder): Promise { @@ -72,6 +114,7 @@ export class DebugAdapterDescriptorFactory implements IDebugAdapterDescriptorFac traceVerbose(`Picking first available interpreter to launch the DA '${interpreters[0].path}'`); return interpreters[0].path; } + /** * Notify user about the requirement for Python. * Unlikely scenario, as ex expect users to have Python in order to use the extension. @@ -84,4 +127,45 @@ export class DebugAdapterDescriptorFactory implements IDebugAdapterDescriptorFac // tslint:disable-next-line: messages-must-be-localized await this.appShell.showErrorMessage('Please install Python or select a Python Interpereter to use the debugger.'); } + + /** + * Return the folder name for the bundled PTVSD wheel compatible with the new debug adapter. + * Use `ptvsd_folder_name.py` to compute the experimental PTVSD folder name in 2 cases: + * - It has never been computed before; + * - The extension number has changed since the last time it was cached. + * + * Return a cached path otherwise, since we pin the PTVSD version with each extension release, + * and other factors on folder selection (like host platform) won't change. + * + * @private + * @param {string} pythonPath Path to the python executable used to launch the Python Debug Adapter (result of `this.getPythonPath()`) + * @returns {Promise} Path to the PTVSD version to use in the debug adapter. + * @memberof DebugAdapterDescriptorFactory + */ + private async getPtvsdFolder(pythonPath: string): Promise { + const persistentState = this.stateFactory.createGlobalPersistentState(ptvsdPathStorageKey, undefined); + const extension = this.extensions.getExtension(PVSC_EXTENSION_ID)!; + const version = parse(extension.packageJSON.version)!; + + if (persistentState.value && version.raw === persistentState.value.extensionVersion) { + const cachedPath = persistentState.value.ptvsdPath; + const access = promisify(fs.access); + try { + await access(cachedPath, fs.constants.F_OK); + return cachedPath; + } catch (err) { + // The path to the cached folder didn't exist (ptvsd requirements updated during development), so run the script again. + } + } + + // The ptvsd path wasn't cached, so run the script and cache it. + const pathToScript = path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'ptvsd_folder_name.py'); + const pythonProcess = await this.executionFactory.create({ pythonPath }); + const executionResult = await pythonProcess.exec([pathToScript], {}); + const pathToPtvsd = executionResult.stdout.trim(); + + await persistentState.updateValue({ extensionVersion: version.raw, ptvsdPath: pathToPtvsd }); + + return pathToPtvsd; + } } diff --git a/src/client/debugger/extension/serviceRegistry.ts b/src/client/debugger/extension/serviceRegistry.ts index 7c8302377245..b3facb549d05 100644 --- a/src/client/debugger/extension/serviceRegistry.ts +++ b/src/client/debugger/extension/serviceRegistry.ts @@ -26,23 +26,11 @@ import { IDebugConfigurationProviderFactory, IDebugConfigurationResolver } from import { ChildProcessAttachEventHandler } from './hooks/childProcessAttachHandler'; import { ChildProcessAttachService } from './hooks/childProcessAttachService'; import { IChildProcessAttachService, IDebugSessionEventHandlers } from './hooks/types'; -import { - DebugConfigurationType, - IDebugAdapterDescriptorFactory, - IDebugConfigurationProvider, - IDebugConfigurationService, - IDebuggerBanner -} from './types'; +import { DebugConfigurationType, IDebugAdapterDescriptorFactory, IDebugConfigurationProvider, IDebugConfigurationService, IDebuggerBanner } from './types'; export function registerTypes(serviceManager: IServiceManager) { - serviceManager.addSingleton( - IExtensionSingleActivationService, - LaunchJsonCompletionProvider - ); - serviceManager.addSingleton( - IExtensionSingleActivationService, - LaunchJsonUpdaterService - ); + serviceManager.addSingleton(IExtensionSingleActivationService, LaunchJsonCompletionProvider); + serviceManager.addSingleton(IExtensionSingleActivationService, LaunchJsonUpdaterService); serviceManager.addSingleton(IDebugConfigurationService, PythonDebugConfigurationService); serviceManager.addSingleton(IDebuggerBanner, DebuggerBanner); serviceManager.addSingleton(IChildProcessAttachService, ChildProcessAttachService); @@ -57,9 +45,6 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IDebugConfigurationProvider, ModuleLaunchDebugConfigurationProvider, DebugConfigurationType.launchModule); serviceManager.addSingleton(IDebugConfigurationProvider, PyramidLaunchDebugConfigurationProvider, DebugConfigurationType.launchPyramid); serviceManager.addSingleton(IDebugEnvironmentVariablesService, DebugEnvironmentVariablesHelper); - serviceManager.addSingleton( - IExtensionSingleActivationService, - DebugAdapterActivator - ); + serviceManager.addSingleton(IExtensionSingleActivationService, DebugAdapterActivator); serviceManager.addSingleton(IDebugAdapterDescriptorFactory, DebugAdapterDescriptorFactory); } diff --git a/src/client/debugger/extension/types.ts b/src/client/debugger/extension/types.ts index 7f02db408e77..f5fd4ae5d893 100644 --- a/src/client/debugger/extension/types.ts +++ b/src/client/debugger/extension/types.ts @@ -35,11 +35,7 @@ export enum PythonPathSource { settingsJson = 'settings.json' } -export enum ExtensionSingleActivationServiceType { - jsonCompletionProvider = 'jsonCompletionProvider', - jsonUpdaterService = 'jsonUpdaterService', - debugAdapterActivator = 'debugAdapterActivator' -} - export const IDebugAdapterDescriptorFactory = Symbol('IDebugAdapterDescriptorFactory'); export interface IDebugAdapterDescriptorFactory extends DebugAdapterDescriptorFactory {} + +export type DebugAdapterPtvsdPathInfo = { extensionVersion: string; ptvsdPath: string }; diff --git a/src/test/debugger/extension/adapter/factory.unit.test.ts b/src/test/debugger/extension/adapter/factory.unit.test.ts index a45789d5b7b7..738a0d808b61 100644 --- a/src/test/debugger/extension/adapter/factory.unit.test.ts +++ b/src/test/debugger/extension/adapter/factory.unit.test.ts @@ -11,15 +11,22 @@ import { SemVer } from 'semver'; import { anyString, anything, instance, mock, verify, when } from 'ts-mockito'; import { DebugAdapterExecutable, DebugConfiguration, DebugSession, WorkspaceFolder } from 'vscode'; import { ApplicationShell } from '../../../../client/common/application/applicationShell'; +import { Extensions } from '../../../../client/common/application/extensions'; import { IApplicationShell } from '../../../../client/common/application/types'; import { DebugAdapterNewPtvsd } from '../../../../client/common/experimentGroups'; import { ExperimentsManager } from '../../../../client/common/experiments'; -import { IExperimentsManager } from '../../../../client/common/types'; +import { PersistentState, PersistentStateFactory } from '../../../../client/common/persistentState'; +import { PythonExecutionFactory } from '../../../../client/common/process/pythonExecutionFactory'; +import { PythonExecutionService } from '../../../../client/common/process/pythonProcess'; +import { IPythonExecutionFactory } from '../../../../client/common/process/types'; +import { IExperimentsManager, IExtensions, IPersistentState, IPersistentStateFactory } from '../../../../client/common/types'; import { Architecture } from '../../../../client/common/utils/platform'; -import { DebugAdapterDescriptorFactory } from '../../../../client/debugger/extension/adapter/factory'; -import { IDebugAdapterDescriptorFactory } from '../../../../client/debugger/extension/types'; +import { EXTENSION_ROOT_DIR } from '../../../../client/constants'; +import { DebugAdapterDescriptorFactory, ptvsdPathStorageKey } from '../../../../client/debugger/extension/adapter/factory'; +import { DebugAdapterPtvsdPathInfo, IDebugAdapterDescriptorFactory } from '../../../../client/debugger/extension/types'; import { IInterpreterService, InterpreterType } from '../../../../client/interpreter/contracts'; import { InterpreterService } from '../../../../client/interpreter/interpreterService'; + use(chaiAsPromised); // tslint:disable-next-line: max-func-body-length @@ -28,24 +35,55 @@ suite('Debugging - Adapter Factory', () => { let interpreterService: IInterpreterService; let appShell: IApplicationShell; let experimentsManager: IExperimentsManager; + let executionFactory: IPythonExecutionFactory; + let stateFactory: IPersistentStateFactory; + let debugAdapterPersistentState: IPersistentState; + let extensions: IExtensions; + const nodeExecutable = { command: 'node', args: [] }; + const mockExtensionVersion = new SemVer('2019.9.0'); + const ptvsdPath = path.join(EXTENSION_ROOT_DIR, 'pythonFiles'); + const pythonPath = path.join('path', 'to', 'python', 'interpreter'); + const interpreter = { + architecture: Architecture.Unknown, + path: pythonPath, + sysPrefix: '', + sysVersion: '', + type: InterpreterType.Unknown, + version: new SemVer('3.7.4-test') + }; setup(() => { interpreterService = mock(InterpreterService); - const interpreter = { - architecture: Architecture.Unknown, - path: path.join('path', 'to', 'active', 'interpreter'), - sysPrefix: '', - sysVersion: '', - type: InterpreterType.Unknown - }; - when(interpreterService.getInterpreters(anything())).thenResolve([interpreter]); - appShell = mock(ApplicationShell); appShell = mock(ApplicationShell); experimentsManager = mock(ExperimentsManager); - factory = new DebugAdapterDescriptorFactory(instance(interpreterService), instance(appShell), instance(experimentsManager)); + executionFactory = mock(PythonExecutionFactory); + stateFactory = mock(PersistentStateFactory); + debugAdapterPersistentState = mock(PersistentState); + extensions = mock(Extensions); + + // tslint:disable-next-line: no-any + when(extensions.getExtension(anything())).thenReturn({ packageJSON: { version: mockExtensionVersion } } as any); + when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreter); + when(interpreterService.getInterpreters(anything())).thenResolve([interpreter]); + + factory = new DebugAdapterDescriptorFactory( + instance(interpreterService), + instance(appShell), + instance(experimentsManager), + instance(executionFactory), + instance(stateFactory), + instance(extensions) + ); }); + function mockPtvsdInfoPersistentState(sameVersion: boolean) { + const debugAdapterInfo: DebugAdapterPtvsdPathInfo = { extensionVersion: sameVersion ? mockExtensionVersion.raw : '2019.10.0-dev', ptvsdPath }; + + when(stateFactory.createGlobalPersistentState(ptvsdPathStorageKey, undefined)).thenReturn(instance(debugAdapterPersistentState)); + when(debugAdapterPersistentState.value).thenReturn(debugAdapterInfo); + } + function createSession(config: Partial, workspaceFolder?: WorkspaceFolder): DebugSession { return { configuration: { name: '', request: 'launch', type: 'python', ...config }, @@ -58,19 +96,10 @@ suite('Debugging - Adapter Factory', () => { } test('Return the value of configuration.pythonPath as the current python path if it exists and if we are in the experiment', async () => { - const pythonPath = path.join('session', 'path', 'to', 'python'); const session = createSession({ pythonPath }); - const interpreterDetails = { - architecture: Architecture.Unknown, - path: pythonPath, - sysPrefix: '', - sysVersion: '', - type: InterpreterType.Unknown, - version: new SemVer('3.7.4-test') - }; - const debugExecutable = new DebugAdapterExecutable(pythonPath); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); - when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreterDetails); + mockPtvsdInfoPersistentState(true); when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); @@ -79,21 +108,12 @@ suite('Debugging - Adapter Factory', () => { }); test('Return the path of the active interpreter as the current python path if we are in the experiment, it exists and configuration.pythonPath is not defined', async () => { - const pythonPath = path.join('active', 'python', 'interpreter'); const session = createSession({}); - const interpreterDetails = { - architecture: Architecture.Unknown, - path: pythonPath, - sysPrefix: '', - sysVersion: '', - type: InterpreterType.Unknown, - version: new SemVer('3.7.4-test') - }; - const debugExecutable = new DebugAdapterExecutable(pythonPath); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); - when(interpreterService.getActiveInterpreter(anything())).thenResolve(interpreterDetails); - when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreterDetails); + mockPtvsdInfoPersistentState(true); when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + when(interpreterService.getActiveInterpreter(anything())).thenResolve(interpreter); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); @@ -101,20 +121,10 @@ suite('Debugging - Adapter Factory', () => { }); test('Return the path of the first available interpreter as the current python path if we are in the experiment, configuration.pythonPath is not defined and there is no active interpreter', async () => { - const pythonPath = path.join('first', 'available', 'interpreter'); const session = createSession({}); - const interpreterDetails = { - architecture: Architecture.Unknown, - path: pythonPath, - sysPrefix: '', - sysVersion: '', - type: InterpreterType.Unknown, - version: new SemVer('3.7.4-test') - }; - const debugExecutable = new DebugAdapterExecutable(pythonPath); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); - when(interpreterService.getInterpreters(anything())).thenResolve([interpreterDetails]); - when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreterDetails); + mockPtvsdInfoPersistentState(true); when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); @@ -139,8 +149,8 @@ suite('Debugging - Adapter Factory', () => { }); test('Return old node debugger when the active interpreter is not Python 3.7', async () => { - const pythonPath = path.join('path', 'to', 'active', 'interpreter'); - const interpreterDetails = { + const python36Path = path.join('path', 'to', 'active', 'interpreter'); + const interpreterPython36Details = { architecture: Architecture.Unknown, path: pythonPath, sysPrefix: '', @@ -150,28 +160,18 @@ suite('Debugging - Adapter Factory', () => { }; const session = createSession({}); - when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreterDetails); + when(interpreterService.getInterpreterDetails(python36Path)).thenResolve(interpreterPython36Details); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); assert.deepEqual(descriptor, nodeExecutable); }); - test('Return debug adapter executable when in the experiment and with the active interpreter being Python 3.7', async () => { - // will be updated when we support the new debug adapter - const pythonPath = path.join('path', 'to', 'active', 'interpreter'); - const interpreterDetails = { - architecture: Architecture.Unknown, - path: pythonPath, - sysPrefix: '', - sysVersion: '', - type: InterpreterType.Unknown, - version: { raw: '3.7.4', major: 3, minor: 7, build: ['foo'], patch: 0, prerelease: ['bar'] } - }; - const debugExecutable = new DebugAdapterExecutable(pythonPath); + test('Return Python debug adapter executable when in the experiment and with the active interpreter being Python 3.7', async () => { + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); const session = createSession({}); - when(interpreterService.getInterpreterDetails(pythonPath)).thenResolve(interpreterDetails); + mockPtvsdInfoPersistentState(true); when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); @@ -179,10 +179,121 @@ suite('Debugging - Adapter Factory', () => { assert.deepEqual(descriptor, debugExecutable); }); - test('Throw an error if the executable has not been defined', async () => { + test('Throw an error if the Node debugger adapter executable has not been defined', async () => { const session = createSession({}); const promise = factory.createDebugAdapterDescriptor(session, undefined); await expect(promise).to.eventually.be.rejectedWith('Debug Adapter Executable not provided'); }); + + test('Save the PTVSD path in persistent storage if it doesn\'t exist in the cache', async () => { + const persistentPtvsdPath = path.join('persistent', 'ptvsd', 'path'); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(persistentPtvsdPath, 'ptvsd', 'adapter')]); + const session = createSession({}); + let execCalled = false; + + when(stateFactory.createGlobalPersistentState(ptvsdPathStorageKey, undefined)).thenReturn(instance(debugAdapterPersistentState)); + when(debugAdapterPersistentState.value).thenReturn(undefined); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + const pythonExecService = ({ + exec: () => { + execCalled = true; + return Promise.resolve({ stdout: persistentPtvsdPath }); + } + // tslint:disable-next-line: no-any + } as any) as PythonExecutionService; + when(executionFactory.create(anything())).thenResolve(pythonExecService); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + assert.equal(execCalled, true); + verify(executionFactory.create(anything())).once(); + verify(debugAdapterPersistentState.updateValue(anything())).once(); + }); + + test('Save the PTVSD path in persistent storage if the extension version in the cache is different from the actual one', async () => { + const session = createSession({}); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); + let execCalled = false; + + mockPtvsdInfoPersistentState(false); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + const pythonExecService = ({ + exec: () => { + execCalled = true; + return Promise.resolve({ stdout: ptvsdPath }); + } + // tslint:disable-next-line: no-any + } as any) as PythonExecutionService; + when(executionFactory.create(anything())).thenResolve(pythonExecService); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + assert.equal(execCalled, true); + verify(executionFactory.create(anything())).once(); + verify(debugAdapterPersistentState.updateValue(anything())).once(); + }); + + test('Use the cached path to PTVSD if it exists and the extension version hasn\'t changed since the value was saved', async () => { + const session = createSession({}); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); + let execCalled = false; + + mockPtvsdInfoPersistentState(true); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + const pythonExecService = ({ + exec: () => { + execCalled = true; + return Promise.resolve({ stdout: ptvsdPath }); + } + // tslint:disable-next-line: no-any + } as any) as PythonExecutionService; + when(executionFactory.create(anything())).thenResolve(pythonExecService); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + assert.equal(execCalled, false); + verify(executionFactory.create(anything())).never(); + verify(debugAdapterPersistentState.updateValue(anything())).never(); + verify(debugAdapterPersistentState.value).thrice(); + }); + + test('Pass the --log-dir argument to PTVSD is configuration.logToFile is set', async () => { + const session = createSession({ logToFile: true }); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter'), '--log-dir', EXTENSION_ROOT_DIR]); + + mockPtvsdInfoPersistentState(true); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + }); + + test('Don\'t pass the --log-dir argument to PTVSD is configuration.logToFile is not set', async () => { + const session = createSession({}); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); + + mockPtvsdInfoPersistentState(true); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + }); + + test('Don\'t pass the --log-dir argument to PTVSD is configuration.logToFile is set but false', async () => { + const session = createSession({ logToFile: false }); + const debugExecutable = new DebugAdapterExecutable(pythonPath, [path.join(ptvsdPath, 'ptvsd', 'adapter')]); + + mockPtvsdInfoPersistentState(true); + when(experimentsManager.inExperiment(DebugAdapterNewPtvsd.experiment)).thenReturn(true); + + const descriptor = await factory.createDebugAdapterDescriptor(session, nodeExecutable); + + assert.deepEqual(descriptor, debugExecutable); + }); });