diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..be006de9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly diff --git a/.github/workflows/check-code-and-run-tests.yml b/.github/workflows/check-code-and-run-tests.yml new file mode 100644 index 00000000..36c971bd --- /dev/null +++ b/.github/workflows/check-code-and-run-tests.yml @@ -0,0 +1,45 @@ +name: Check code and run tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - run: pip install --user ruff + - run: ruff check --ignore=E401,E402,E501,E701,E721,E722,E731,E741,F401,F403,F405,F523,F524,F811,F841 + --output-format=github --target-version=py310 . + build: + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] # ["2.7", "3.6", "3.10"] + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools + pip install ".[testing]" + - name: Analysing the code with flake8 + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics + - name: Running tests + run: | + pytest --version + # make install test work... + export PYTHONPATH=$(python -m site --user-site) + pytest tests/ --ignore=tests/system/data/ --showlocals --verbose --show-capture=all --log-level=debug + diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index c55fd813..00000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - -name: Python package - -on: - push: - branches: [ master, dev ] - pull_request: - branches: [ master, dev ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: [2.7, 3.6, 3.8] - - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install --upgrade flake8>=3.7.9 pyparsing==2.4.4 pytest==4.6.6 - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics - # show pytest version - py.test --version - # install - pip install ".[testing]" - - name: Test with pytest - run: | - pytest tests/ --ignore=tests/system/data/ --showlocals --verbose --show-capture=all --log-level=debug diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a7eaa028..00000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: python -python: - - "2.7" - - "3.6" - - "3.8" -jobs: - allow_failures: - - python: "3.8" -# command to install dependencies -install: - - "pip install --upgrade flake8>=3.7.9 pyparsing==2.4.4 pytest==4.6.6" -before_script: - # stop the build if there are Python syntax errors or undefined names - - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. - - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=88 --statistics - # show pytest version - - py.test --version - # install - - pip install ".[testing]" -# command to run tests -script: pytest tests/ --ignore=tests/system/data/ --showlocals --verbose --show-capture=all --log-level=debug - diff --git a/bin/git.py b/bin/git.py index 8d563dec..4e36d06f 100644 --- a/bin/git.py +++ b/bin/git.py @@ -63,7 +63,7 @@ def call(*args, **kwargs): if AUTODOWNLOAD_DEPENDENCIES: libpath = os.path.join(os.environ['STASH_ROOT'], 'lib') - if not libpath in sys.path: + if libpath not in sys.path: sys.path.insert(1, libpath) download_dulwich = False @@ -381,10 +381,10 @@ def git_reset(args): commit = ns.commit # first arg was really a file paths = ns.paths or [] - if not commit in repo and os.path.exists(commit): #really specified a path + if commit not in repo and os.path.exists(commit): #really specified a path paths = [commit] + paths commit = None - elif not commit in repo and not commit in repo.branches and not commit in repo.remote_branches and not os.path.exists( + elif commit not in repo and commit not in repo.branches and commit not in repo.remote_branches and not os.path.exists( commit): raise Exception('{} is not a valid commit or file'.format(commit)) if not commit: @@ -429,7 +429,7 @@ def get_config_or_prompt(repo, section, name, prompt, save=None): value = config.get(section, name) except KeyError: value = input(prompt).encode() - if save == None: + if save is None: reply = input('Save this setting? [y/n]') save = reply == 'y' if save: @@ -496,7 +496,13 @@ def git_commit(args): def git_clone(args): if len(args) > 0: url = args[0] - repo = Gittle.clone(args[0], args[1] if len(args) > 1 else os.path.split(args[0])[-1].rstrip('.git'), bare=False) + if len(args) > 1: + args_1 = args[1] + else: + args_1 = os.path.split(args[0])[-1] + if args_1.endswith('.git'): + args_1 = args_1[:-4] + repo = Gittle.clone(args[0], args_1, bare=False) #Set the origin config = repo.repo.get_config() diff --git a/bin/httpserver.py b/bin/httpserver.py index 3a92ea50..bcba8d23 100644 --- a/bin/httpserver.py +++ b/bin/httpserver.py @@ -88,7 +88,7 @@ def deal_post_data(self): remainbytes = int(self.headers['content-length']) line = self.rfile.readline() remainbytes -= len(line) - if not boundary in line: + if boundary not in line: return (False, "Content NOT begin with boundary") line = self.rfile.readline() remainbytes -= len(line) diff --git a/bin/mail.py b/bin/mail.py index bc59b3c3..3db775ce 100644 --- a/bin/mail.py +++ b/bin/mail.py @@ -159,7 +159,7 @@ def send(self, sendto='', subject='', attach='', body=' '): ) args = ap.parse_args() smail = Mail(CONFIG, args.verbose) - if args.e == True: + if args.e is True: smail.edit_cfg() elif args.message or args.file and args.sendto: if args.message == '-': diff --git a/bin/mount.py b/bin/mount.py index 82226f98..246a085a 100644 --- a/bin/mount.py +++ b/bin/mount.py @@ -64,7 +64,7 @@ def list_mounts(): else: manager.enable_patches() - if not ns.type in FILESYSTEM_TYPES: + if ns.type not in FILESYSTEM_TYPES: print(_stash.text_color("Error: Unknown Filesystem-Type!", "red")) sys.exit(1) diff --git a/bin/mv.py b/bin/mv.py index 7be6d6a9..8699e1a3 100644 --- a/bin/mv.py +++ b/bin/mv.py @@ -30,7 +30,7 @@ def main(args): for src in ns.src: try: # Attempt to move every source into destination - shutil.move(src, os.path.join(ns.dest, os.path.basename(src))) + shutil.move(src, ns.dest) except Exception as err: print("mv: {}: {!s}".format(type(err).__name__, err), file=sys.stderr) status = 1 @@ -44,27 +44,17 @@ def main(args): src = ns.src[0] if os.path.exists(src): # Source must exist - if os.path.exists(ns.dest): - # If destination exists... - if os.path.isdir(ns.dest): - # ...it must be a folder - try: - # Attempt to move source into destination - shutil.move(src, os.path.join(ns.dest, os.path.basename(src))) - except Exception as err: - print("mv: {}: {!s}".format(type(err).__name__, err), file=sys.stderr) - status = 1 - else: - # Won't overwrite unasked - print("mv: {}: file exists".format(ns.dest), file=sys.stderr) - else: - # Destination doesn't exist + if not os.path.isfile(ns.dest): + # Python will rename source if it doesn't exists + # And will move source into destination if it is a directory try: - # Try to rename source to destination shutil.move(src, ns.dest) except Exception as err: print("mv: {}: {!s}".format(type(err).__name__, err), file=sys.stderr) status = 1 + else: + # Won't overwrite unasked + print("mv: {}: file exists".format(ns.dest), file=sys.stderr) else: print("mv: {}: no such file or directory".format(src), file=sys.stderr) status = 1 diff --git a/bin/ping.py b/bin/ping.py index 35fef910..f9ca6d33 100644 --- a/bin/ping.py +++ b/bin/ping.py @@ -218,7 +218,7 @@ def verbose_ping(dest_addr, timeout=2, count=4, interval=1.0): print("failed. (socket error: '%s')" % e[1]) break - if delay == None: + if delay is None: print("failed. (timeout within %ssec.)" % timeout) else: time.sleep(min(0, interval - delay)) diff --git a/bin/pip.py b/bin/pip.py index 7031ae5b..44afe882 100644 --- a/bin/pip.py +++ b/bin/pip.py @@ -45,7 +45,9 @@ SITE_PACKAGES_FOLDER = _stash.libdist.SITE_PACKAGES_FOLDER OLD_SITE_PACKAGES_FOLDER = _stash.libdist.SITE_PACKAGES_FOLDER_6 BUNDLED_MODULES = _stash.libdist.BUNDLED_MODULES -BLACKLIST_PATH = os.path.join(os.path.expandvars("$STASH_ROOT"), "data", "pip_blacklist.json") +BLOCKLIST_PATH = os.path.join(os.path.expandvars("$STASH_ROOT"), "data", "pip_blocklist.json") +PIP_INDEX_FILE = os.path.join(SITE_PACKAGES_FOLDER,'pip_index.json') +PIP_INFO_FILE = os.path.join(SITE_PACKAGES_FOLDER, '.package_info', '%s.json') # Some packages use wrong name for their dependencies PACKAGE_NAME_FIXER = { @@ -59,7 +61,7 @@ FLAG_DIST_ALLOW_WHL = 2 FLAG_DIST_PREFER_SRC = 4 FLAG_DIST_PREFER_WHL = 8 -FLAG_IGNORE_BLACKLIST = 16 +FLAG_IGNORE_BLOCKLIST = 16 DEFAULT_FLAGS = FLAG_DIST_ALLOW_SRC | FLAG_DIST_ALLOW_WHL | FLAG_DIST_PREFER_WHL @@ -82,16 +84,16 @@ class PackageAlreadyInstalled(PipError): pass -class PackageBlacklisted(PipError): +class PackageBlocklisted(PipError): """ - Error raised when a package is fataly blacklisted - :param pkg_name: name of blacklisted package + Error raised when a package is fataly blocklisted + :param pkg_name: name of blocklisted package :type pkg_name: str - :param reason: reason for blacklisting + :param reason: reason for blocklisting :type reason: str """ def __init__(self, pkg_name, reason): - s = "Package '{}' blacklisted. Reason: {}".format(pkg_name, reason) + s = "Package '{}' blocklisted. Reason: {}".format(pkg_name, reason) PipError.__init__(self, s) @@ -213,6 +215,116 @@ def __getattr__(self, item): return PackageFinder.find return OmniClass() +def get_requires(package, index_file=PIP_INDEX_FILE): + """ + get require of the package + :param package: package name + :type package: str + :return: a list of requires package + :rtype: list + """ + if os.path.exists(index_file): + with open(index_file) as f: + index=json.load(f) + try: + return index[package] + except KeyError:# no such package in index file + raise PipError("Cannot find packages in index file. Try to using 'pip dev update-index' to update index file") + + else: + raise PipError("Cannot find index file. Try to using 'pip dev update-index' to update index file") + +def get_req_by(package, index_file=PIP_INDEX_FILE): + """ + get the packages that require this package + :param package: package name + :type package: str + :return: a list of packages that require this package + :rtype: list + """ + if os.path.exists(index_file): + + with open(index_file) as f: + index=json.load(f) + required_by=[] + for pkg, req in index.items(): + if package in req: + required_by.append(pkg) + else: + + raise PipError("Cannot find index file. Try to using 'pip dev update-index' to update index file") + + return required_by + + + + + +def print_info(package, pip_info_file=PIP_INFO_FILE, site_packages=SITE_PACKAGES_FOLDER): + info_file = pip_info_file % package + if os.path.exists(info_file): + with open(info_file) as f: + info = json.load(f) + print('Name: {}'.format(info['name'])) + print('Version: {}'.format(info['version'])) + print('Summary: {}'.format(info['summary'])) + print('Home-page: {}'.format(info['project_urls']['Homepage'])) + print('Author: {}'.format(info['author'])) + print('Author-email: {}'.format(info['author_email'])) + print('License: {}'.format(info['license'])) + print('Location: {}'.format(site_packages)) + + requires = get_requires(package) + required_by = get_req_by(package) + print('requires: {}'.format(', '.join(requires))) + print('required-by: {}'.format(', '.join(required_by))) + else: #no info_file + print(_stash.text_color('Package not found: {}'.format(package), 'yellow')) + + + +def download_info(pkg_name, pip_info_file=PIP_INFO_FILE, site_packages=SITE_PACKAGES_FOLDER): + info_file = pip_info_file % pkg_name + r=requests.get('https://pypi.python.org/pypi/{}/json'.format(pkg_name)) + info=r.json()['info'] + info_folder = os.path.split(info_file)[0] + if not os.path.exists(info_folder): + os.mkdir(info_folder) + with open(info_file, 'w') as f: + json.dump(info, f) + update_req_index() + +def update_req_index(pip_info_file=PIP_INFO_FILE, site_packages=SITE_PACKAGES_FOLDER, index_file=PIP_INDEX_FILE): + ''' + update package requires index file + ''' + repository = get_repository('pypi', site_packages=site_packages) + info_list = repository.list() + + req_index={} # a dict of {package:requires_list} type:dict of {str:list of str} + for package, info in info_list: + info_file = pip_info_file % package + if os.path.exists(info_file): + # load info + with open(info_file) as f: + info=json.load(f) + # filte requires + requires = [] + try: + for req in info['requires_dist']: + if ';' not in req: + #Remove package version + requires.append(req.split(' ')[0]) + except TypeError:# some package may have no require + pass + req_index[package]=requires + + else: # info file not exists + print(_stash.text_color('Info file of Package {} not found'.format(package), 'yellow')) + + + with open(index_file, 'w') as f: + json.dump(req_index,f) def fake_module(new_module): """ @@ -670,7 +782,7 @@ def _run_setup_file(self, filename, extras=[]): if self.verbose: print("Handling commandline script: {s}".format(s=script)) cmdname = script.replace(os.path.dirname(script), "").replace("/", "") - if not "." in cmdname: + if '.' not in cmdname: cmdname += ".py" scriptpath = os.path.join(source_folder, script) with open(scriptpath, "r") as fin: @@ -981,30 +1093,30 @@ def __init__(self, *args, **kwargs): self.pypi = xmlrpclib.ServerProxy('https://pypi.python.org/pypi') self.standard_package_names = {} - def _check_blacklist(self, pkg_name): + def _check_blocklist(self, pkg_name): """ - Check if a package is blacklisted. + Check if a package is blocklisted. The result is a tuple: - - element 0 is True if the package is blacklisted + - element 0 is True if the package is blocklisted - element 1 is the reason - element 2 is True if the install should fail due to this - element 3 is an optional alternative package to use instead. :param pkg_name: name of package to check :type pkg_name: str - :return: a tuple of (blacklisted, reason, fatal, alt). + :return: a tuple of (blocklisted, reason, fatal, alt). :rtype: (bool, str, bool, str or None) """ - if (BLACKLIST_PATH is None) or (not os.path.exists(BLACKLIST_PATH)): - # blacklist not available + if (BLOCKLIST_PATH is None) or (not os.path.exists(BLOCKLIST_PATH)): + # blocklist not available return (False, "", False, None) - with open(BLACKLIST_PATH) as fin: + with open(BLOCKLIST_PATH) as fin: content = json.load(fin) - if pkg_name not in content["blacklist"]: - # package not blacklisted + if pkg_name not in content["blocklist"]: + # package not blocklisted return (False, "", False, None) else: - # package blacklisted - reasonid, fatal, alt = content["blacklist"][pkg_name] + # package blocklisted + reasonid, fatal, alt = content["blocklist"][pkg_name] reason = content["reasons"].get(reasonid, reasonid) return (True, reason, fatal, alt) @@ -1127,25 +1239,25 @@ def download(self, pkg_name, ver_spec, flags=DEFAULT_FLAGS): return os.path.join(os.getenv('TMPDIR'), target['filename']), pkg_info - def install(self, pkg_name, ver_spec, flags=DEFAULT_FLAGS, extras=[]): + def install(self, pkg_name, ver_spec, flags=DEFAULT_FLAGS, pip_info_file=PIP_INFO_FILE, extras=[]): pkg_name = self.get_standard_package_name(pkg_name) - # check if package is blacklisted + # check if package is blocklisted # we only do this for PyPI installs, since non-PyPI installs # may have the same pkg name for a different package. # TODO: should this be changed? - blacklisted, reason, fatal, alt = self._check_blacklist(pkg_name) - if blacklisted and not (flags & FLAG_IGNORE_BLACKLIST > 0): + blocklisted, reason, fatal, alt = self._check_blocklist(pkg_name) + if blocklisted and not (flags & FLAG_IGNORE_BLOCKLIST > 0): if fatal: # raise an exception. print( _stash.text_color( - "Package {} is blacklisted and marked fatal. Failing install.".format(pkg_name), + "Package {} is blocklisted and marked fatal. Failing install.".format(pkg_name), "red", ) ) print(_stash.text_color("Reason: " + reason, "red")) - raise PackageBlacklisted(pkg_name, reason) + raise PackageBlocklisted(pkg_name, reason) elif alt is not None: # an alternative package exposing the same functionality # and API is known. Print a warning and use this instead. @@ -1166,17 +1278,24 @@ def install(self, pkg_name, ver_spec, flags=DEFAULT_FLAGS, extras=[]): # we should print a warning, but continue anyway print( _stash.text_color( - "Warning: package '{}' is blacklisted, but marked as non-fatal.".format(pkg_name), + "Warning: package '{}' is blocklisted, but marked as non-fatal.".format(pkg_name), "yellow", ) ) print("This probably means that the dependency can not be installed, but pythonista ships with the package preinstalled.") - print("Reason for blacklisting: " + reason) + print("Reason for blocklisting: " + reason) return if not self.config.module_exists(pkg_name): archive_filename, pkg_info = self.download(pkg_name, ver_spec, flags=flags) self._install(pkg_name, pkg_info, archive_filename, dependency_flags=flags, extras=extras) + # save json file of info + info_file = pip_info_file % pkg_name + info_folder = os.path.split(info_file)[0] + if not os.path.exists(info_folder): + os.mkdir(info_folder) + with open(info_file, 'w') as f: + json.dump(pkg_info, f) else: # todo: maybe update package? raise PackageAlreadyInstalled('Package already installed') @@ -1435,7 +1554,6 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): if __name__ == '__main__': import argparse - ap = argparse.ArgumentParser() ap.add_argument('--verbose', action='store_true', help='be more chatty') @@ -1454,6 +1572,10 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): metavar='sub-command', help='"pip sub-command -h" for more help on a sub-command' ) + + show_parser = subparsers.add_parser('show', help='show information of package ') + show_parser.add_argument('package', help='package name to show') + show_parser.add_argument('-f', '--force', action="store_true", dest="forcedownload", help='force to download info file from pypi') list_parser = subparsers.add_parser('list', help='list packages installed') @@ -1483,7 +1605,7 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): help="Prefer older binary packages over newer source packages", # TODO: do we actually check older sources/wheels? dest="preferbinary", ) - install_parser.add_argument("--ignore-blacklist", action="store_true", help="Ignore blacklist", dest="ignoreblacklist") + install_parser.add_argument("--ignore-blocklist", action="store_true", help="Ignore blocklist", dest="ignoreblocklist") download_parser = subparsers.add_parser('download', help='download packages') download_parser.add_argument( @@ -1514,6 +1636,9 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): update_parser = subparsers.add_parser('update', help='update an installed package') update_parser.add_argument('packages', nargs="+", help='the package name') + dev_parser = subparsers.add_parser('dev') + dev_parser.add_argument('opt') + ns = ap.parse_args() if ns.site_packages is None: @@ -1565,8 +1690,8 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): flags = flags | FLAG_DIST_PREFER_WHL | FLAG_DIST_ALLOW_WHL flags = flags & ~FLAG_DIST_PREFER_SRC - if ns.ignoreblacklist: - flags = flags | FLAG_IGNORE_BLACKLIST + if ns.ignoreblocklist: + flags = flags | FLAG_IGNORE_BLOCKLIST for requirement in ns.requirements: repository = get_repository(requirement, site_packages=site_packages, verbose=ns.verbose) @@ -1580,6 +1705,7 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): # start with what we have installed (i.e. in the config file) sys.modules['setuptools']._installed_requirements_ = repository.config.list_modules() repository.install(pkg_name, ver_spec, flags=flags, extras=extras) + update_req_index() elif ns.sub_command == 'download': for requirement in ns.requirements: @@ -1609,6 +1735,7 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): for package_name in ns.packages: repository = get_repository('pypi', site_packages=ns.site_packages, verbose=ns.verbose) repository.remove(package_name) + update_req_index() elif ns.sub_command == 'update': for package_name in ns.packages: @@ -1620,6 +1747,20 @@ def get_repository(pkg_name, site_packages=SITE_PACKAGES_FOLDER, verbose=False): # start with what we have installed (i.e. in the config file) sys.modules['setuptools']._installed_requirements_ = repository.config.list_modules() repository.update(package_name) + + elif ns.sub_command == 'show': + if ns.forcedownload: + download_info(ns.package, site_packages=ns.site_packages) + print_info(ns.package, site_packages=ns.site_packages) + + elif ns.sub_command=='dev': + if ns.opt=='update-index': + update_req_index() + print('index file updated') + else: + raise PipError('unknow dev option: {}'.format(ns.opt)) + sys.exit(1) + else: raise PipError('unknown command: {}'.format(ns.sub_command)) sys.exit(1) diff --git a/core.py b/core.py index 4515eb63..e9ae7c69 100644 --- a/core.py +++ b/core.py @@ -5,7 +5,7 @@ https://github.com/ywangd/stash """ -__version__ = '0.7.4' +__version__ = '0.7.5' import imp as pyimp # rename to avoid name conflict with objc_util import logging diff --git a/data/pip_blacklist.json b/data/pip_blocklist.json similarity index 89% rename from data/pip_blacklist.json rename to data/pip_blocklist.json index 5c96b5f4..665f6826 100644 --- a/data/pip_blacklist.json +++ b/data/pip_blocklist.json @@ -10,7 +10,7 @@ "incompatible_dependency": "This package has one or more dependencies which are incompatible with Pythonista.", "bugged": "Installation of this packages is currently critically bugged." }, -"blacklist": { +"blocklist": { "stash": ["is_stash", true, null], "pip": ["is_pip", true, null], "selenium": ["processes", true, null], @@ -22,6 +22,8 @@ "pyobjc": ["C", true, null], "pyttsx3": ["incompatible_dependency", true, null], "futures": ["bugged", true, null], - "regex": ["C", true, null] + "regex": ["C", true, null], + "gevent": ["C", true, null], + "grequests": ["incompatible_dependency", true, null] } } diff --git a/docs/pip_blacklist.md b/docs/pip_blocklist.md similarity index 67% rename from docs/pip_blacklist.md rename to docs/pip_blocklist.md index c32a54f8..5432ac23 100644 --- a/docs/pip_blacklist.md +++ b/docs/pip_blocklist.md @@ -1,6 +1,6 @@ -#PIP blacklist +#PIP blocklist ----------------------- -Starting with version 0.7.5, StaSh pip includes a blacklist. +Starting with version 0.7.5, StaSh pip includes a blocklist. It indicates whether a package should not be installed and what reasons @@ -9,17 +9,17 @@ should be given. ## Motivation -The reason for this blacklist is the high number of issues regarding +The reason for this blocklist is the high number of issues regarding installation problems of known incompatible packages. I hope that by -slowly adding packages to the blacklist we can reduce the amount of +slowly adding packages to the blocklist we can reduce the amount of these issues. -The blacklist was initially discussed in issue #376. +The blocklist was initially discussed in issue #376. ## Details -This blacklist is stored as a JSON file at `$STASH_ROOT/data/pip_blacklist.json`. +This blocklist is stored as a JSON file at `$STASH_ROOT/data/pip_blocklist.json`. It is a dict with two top-level keys: @@ -27,14 +27,14 @@ It is a dict with two top-level keys: It is used to reduce redundancy of error messages. -**`blacklist`** Is a dict mapping packagename (str) to details (list). -Every package mentioned in a key of the dict is considered blacklisted. +**`blocklist`** Is a dict mapping packagename (str) to details (list). +Every package mentioned in a key of the dict is considered blocklisted. The first (`i=0`) element of the list is the reasonID (str). Use the `reasons` toplevel key to determine the actual reason. The second (`i=1`) element of the list is a bool indicating whether this -blacklisting is fatal. If true, it is considered fatal. This means that +blocklisting is fatal. If true, it is considered fatal. This means that `pip` should abort the installation. Otherwise it is considered nonfatal. This means that `pip` should skip the installation, but continue the install. This is useful if the package can not be installed, but Pythonista already diff --git a/getstash.py b/getstash.py index 2b94f30f..6c765db0 100644 --- a/getstash.py +++ b/getstash.py @@ -29,7 +29,7 @@ 'bin/pythonista.py', 'bin/cls.py', 'stash.py', - 'lib/librunner.py' + 'lib/librunner.py', 'system/shui.py', 'system/shterminal.py', 'system/dummyui.py', diff --git a/lib/git/git-branch.py b/lib/git/git-branch.py index 8b13c483..a29cc3e1 100644 --- a/lib/git/git-branch.py +++ b/lib/git/git-branch.py @@ -152,7 +152,7 @@ def format_tracking_branch_desc(repo, branchname): def edit_branch_description(branchname, description=None): description = description or input('enter description:') config = _get_repo().repo.get_config() - if not branchname in _get_repo().branches: + if branchname not in _get_repo().branches: GitError('{} is not an existing branch'.format(branchname)) config.set(('branch', branchname), 'description', description) config.write_to_path() @@ -284,7 +284,7 @@ def create_branch(new_branch, base_rev, force=False, no_track=False): #handle tracking, only if this was a remote tracking, remote_branch = (['origin'] + base_rev.split('/'))[-2:] #branch-> origin/branch. remote/branch stays as is qualified_remote_branch = os.path.sep.join([tracking, remote_branch]) - if qualified_remote_branch in repo.remote_branches and not base_rev in repo.branches: + if qualified_remote_branch in repo.remote_branches and base_rev not in repo.branches: if not no_track: add_tracking(new_branch, tracking, remote_branch) else: diff --git a/lib/libdist.py b/lib/libdist.py index 05d9caaa..c27ed478 100644 --- a/lib/libdist.py +++ b/lib/libdist.py @@ -38,7 +38,9 @@ def clipboard_set(s): # -------------- pip ---------------------- if six.PY3: - SITE_PACKAGES_DIR_NAME = "site-packages-3" + SITE_PACKAGES_DIR_NAME = "site-packages" + if sys.version_info < (3, 10): # Pythonista < v3.4 + SITE_PACKAGES_DIR_NAME += "-3" else: SITE_PACKAGES_DIR_NAME = "site-packages-2" SITE_PACKAGES_DIR_NAME_6 = "site-packages" @@ -146,7 +148,8 @@ def clipboard_set(s): pyperclip.copy(s) else: # use fake implementation - global _CLIPBOARD; _CLIPBOARD = u"" + global _CLIPBOARD + _CLIPBOARD = u"" def clipboard_get(): """ diff --git a/lib/libversion.py b/lib/libversion.py index bea19377..93c217cd 100644 --- a/lib/libversion.py +++ b/lib/libversion.py @@ -323,6 +323,14 @@ def parse_requirement(requirement): extras = [] else: extras = extra_s.split(",") + elif "[" in name: + si = name.find("[") + extra_s = name[si + 1:-1] + name = name[:si] + if len(extra_s) == 0: + extras = [] + else: + extras = extra_s.split(",") else: extras = [] splitted = specs_s.split(",") diff --git a/lib/stashutils/mount_manager.py b/lib/stashutils/mount_manager.py index c3720c02..4f8a95f3 100644 --- a/lib/stashutils/mount_manager.py +++ b/lib/stashutils/mount_manager.py @@ -79,7 +79,7 @@ def mount_fsi(self, path, fsi, readonly=False): def unmount_fsi(self, path, force=False): """unmounts a fsi.""" path = os.path.abspath(path) - if not path in self.path2fs: + if path not in self.path2fs: raise MountError("Nothing mounted there.") fsi, readonly = self.path2fs[path] if not force: diff --git a/lib/stashutils/wheels.py b/lib/stashutils/wheels.py index eefc41ce..a44c1fa9 100644 --- a/lib/stashutils/wheels.py +++ b/lib/stashutils/wheels.py @@ -336,34 +336,51 @@ def read_dependencies_from_METADATA(self, p): if ";" in t: es = t[t.find(";") + 1:].replace('"', "").replace("'", "") t = t[:t.find(";")].strip() - if VersionSpecifier is None: - # libversion not found - print( - "Warning: could not import libversion.VersionSpecifier! Ignoring version and extra dependencies." - ) - rq, v, extras = "", "???", [] - else: - rq, v, extras = VersionSpecifier.parse_requirement(es) - if rq == "python_version": - # handle python version dependencies - if not v.match(platform.python_version()): - # dependency NOT required - continue - elif rq == "extra": - # handle extra dependencies - matched = any([v.match(e) for e in self.wheel.extras]) - if not matched: - # dependency NOT required - continue + for sub_es in es.split(' and '): + if VersionSpecifier is None: + # libversion not found + print( + "Warning: could not import libversion.VersionSpecifier! Ignoring version and extra dependencies." + ) + rq, v, extras = "", "???", [] + else: + rq, v, extras = VersionSpecifier.parse_requirement(sub_es) + + if rq == "python_version": + # handle python version dependencies + if not v.match(platform.python_version()): + # dependency NOT required + break + elif rq == "extra": + # handle extra dependencies + matched = any([v.match(e) for e in self.wheel.extras]) + if not matched: + # dependency NOT required + break + else: + if self.verbose: + print("Adding dependencies for extras...") + elif rq == "platform_python_implementation": + if not v.match(platform.python_implementation()): + break + elif rq == "platform_system": + if v.match(platform.system()): + break + elif rq == "sys_platform": + if not v.match(sys.platform): + break else: - if self.verbose: - print("Adding dependencies for extras...") + # unknown requirement for dependency + # warn user and register the dependency + print("Warning: unknown dependency requirement: '{}'".format(rq)) + print("Warning: Adding dependency '{}', ignoring requirements for dependency.".format(t)) + # do not do anything here- As long as we dont use 'continue', 'break', ... the dependency will be added. else: - # unknown requirement for dependency - # warn user and register the dependency - print("Warning: unknown dependency requirement: '{}'".format(rq)) - print("Warning: Adding dependency '{}', ignoring requirements for dependency.".format(t)) - # do not do anything here- As long as we dont use 'continue', 'break', ... the dependency will be added. + # no 'break' happens + continue + # a 'break' happens, don't add dependencies and go to next 'line' + break + dependencies.append(t) return dependencies @@ -449,4 +466,4 @@ def extract_into_temppath(self): print("dependencies:") print(dep) if len(dep) > 0: - print("WARNING: Dependencies were not installed.") + print("WARNING: Dependencies were not installed.") \ No newline at end of file diff --git a/setup.py b/setup.py index fe042802..75cfb149 100644 --- a/setup.py +++ b/setup.py @@ -40,17 +40,35 @@ # =================== SETUP =================== -from distutils.core import setup -from setuptools import find_packages +from setuptools import setup,find_packages - -TEST_REQUIREMENTS = [ - "pyparsing==2.0.1", - "pytest>=3.6.0", - "flake8>=3.5.0", - "pycrypto==2.6", - "requests==2.9.1", -] +if sys.version_info.major==2: + INSTALL_REQUIREMENTS = [ + "rsa==4.5", + "six", # required by StaSh + "pyperclip", # required by libdist for copy/paste on PC + "requests==2.9.1", + "pycrypto==2.6", + "pyte==0.8.1", + ] + TEST_REQUIREMENTS = [ + "pyparsing==2.0.2", + "pytest==4.6.11", + "flake8>=3.7.9", + ] +else: + INSTALL_REQUIREMENTS=[ + "six", # required by StaSh + "pyperclip", # required by libdist for copy/paste on PC + "requests", + "pycrypto", + "pyte", + ], + TEST_REQUIREMENTS = [ + "pyparsing", + "pytest", + "flake8>=3.7.9", + ] PARENT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -125,12 +143,7 @@ def get_stash_version(corepath): }, scripts=[os.path.join(STASH_DIR, "launch_stash.py")], zip_safe=False, - install_requires=[ - "six", # required by StaSh - "pyperclip", # required by libdist for copy/paste on PC - "requests", - "pyte", - ], + install_requires=INSTALL_REQUIREMENTS, extras_require={ "testing": TEST_REQUIREMENTS, }, diff --git a/system/shcommon.py b/system/shcommon.py index 1ef2232b..3db5fc6d 100644 --- a/system/shcommon.py +++ b/system/shcommon.py @@ -17,7 +17,8 @@ if IN_PYTHONISTA: import plistlib - _properties = plistlib.readPlist(os.path.join(os.path.dirname(sys.executable), 'Info.plist')) + with open(os.path.join(os.path.dirname(sys.executable), 'Info.plist'), 'rb') as fp: + _properties = plistlib.loads(fp.read()) PYTHONISTA_VERSION = _properties['CFBundleShortVersionString'] PYTHONISTA_VERSION_LONG = _properties['CFBundleVersion'] diff --git a/system/shui/pythonista_ui.py b/system/shui/pythonista_ui.py index 05e5c64e..06b118c6 100644 --- a/system/shui/pythonista_ui.py +++ b/system/shui/pythonista_ui.py @@ -894,7 +894,7 @@ def render(self, no_wait=False): self.render_thread.cancel() self._render() else: # delayed rendering - if self.render_thread is None or not self.render_thread.isAlive(): + if self.render_thread is None or not self.render_thread.is_alive(): self.render_thread = sh_delay(self._render, self.RENDER_INTERVAL) # Do nothing if there is already a delayed rendering thread waiting diff --git a/tests/pip/test_pip.py b/tests/pip/test_pip.py index fb8b723e..f6c90dd9 100644 --- a/tests/pip/test_pip.py +++ b/tests/pip/test_pip.py @@ -60,6 +60,7 @@ def test_help(self): self.assertIn("download", output) self.assertIn("list", output) + @unittest.skip('Pip is retiring and seach is disabled') @requires_network def test_search(self): """test 'pip search '""" @@ -105,8 +106,8 @@ def test_install_pypi_simple_2(self): self.logger.info("sys.path = " + str(sys.path)) raise AssertionError("Could not import installed module: " + repr(e)) + # @expected_failure_on_py3 @requires_network - @expected_failure_on_py3 def test_install_pypi_complex_1(self): """test 'pip install ' with a complex package.""" output = self.run_command("pip --verbose install twisted", exitcode=0) @@ -119,6 +120,7 @@ def test_install_pypi_complex_1(self): self.logger.info("sys.path = " + str(sys.path)) raise AssertionError("Could not import installed module: " + repr(e)) + ''' setup.py on modern pip @requires_network def test_install_pypi_nobinary(self): """test 'pip install --no-binary :all: '.""" @@ -131,11 +133,12 @@ def test_install_pypi_nobinary(self): except ImportError as e: self.logger.info("sys.path = " + str(sys.path)) raise AssertionError("Could not import installed module: " + repr(e)) + ''' @requires_network def test_install_pypi_onlybinary(self): """test 'pip install --only-binary :all: '.""" - output = self.run_command("pip --verbose install --only-binary :all: rsa", exitcode=0) + output = self.run_command("pip --verbose install --only-binary :all: rsa==4.5", exitcode=0) self.assertIn("Downloading package", output) self.assert_did_run_setup(output, allow_source=False) self.assertIn("Package installed: rsa", output) @@ -152,7 +155,7 @@ def test_install_command(self): self.run_command("pyrsa-keygen --help", exitcode=127) # 2. install - output = self.run_command("pip --verbose install rsa", exitcode=0) + output = self.run_command("pip --verbose install rsa==4.5", exitcode=0) self.assertIn("Downloading package", output) self.assert_did_run_setup(output) self.assertIn("Package installed: rsa", output) @@ -287,20 +290,20 @@ def test_uninstall(self): # expected failure pass - def test_blacklist_fatal(self): - """test 'pip install '.""" + def test_blocklist_fatal(self): + """test 'pip install '.""" output = self.run_command("pip --verbose install pip", exitcode=1) self.assertIn("StaSh uses a custom version of PIP", output) - self.assertIn("PackageBlacklisted", output) + self.assertIn("PackageBlocklisted", output) self.assertNotIn("Package installed: pip", output) - def test_blacklist_nonfatal(self): - """test 'pip install '.""" + def test_blocklist_nonfatal(self): + """test 'pip install '.""" output = self.run_command("pip --verbose install matplotlib", exitcode=0) - self.assertIn("Warning: package 'matplotlib' is blacklisted, but marked as non-fatal.", output) + self.assertIn("Warning: package 'matplotlib' is blocklisted, but marked as non-fatal.", output) self.assertIn("This package is already bundled with Pythonista", output) - self.assertNotIn("PackageBlacklisted", output) + self.assertNotIn("PackageBlocklisted", output) self.assertNotIn("Package installed: matplotlib", output) - # TODO: add test for blacklist with alternative. + # TODO: add test for blocklist with alternative. diff --git a/tools/pythonista_reinstall.py b/tools/pythonista_reinstall.py index fee4c94e..dc2fef86 100644 --- a/tools/pythonista_reinstall.py +++ b/tools/pythonista_reinstall.py @@ -19,7 +19,7 @@ def remove_stash(): def install_stash(repo="ywangd", branch="master"): - if not "TMPDIR" in os.environ: + if "TMPDIR" not in os.environ: os.environ["TMPDIR"] = tempfile.gettempdir() ns = {"_owner": repo, "_br": branch} exec(requests.get("https://bit.ly/get-stash").content, ns, ns)