diff --git a/.coveragerc b/.coveragerc index 8cc1780e..69277de5 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,2 +1,2 @@ [run] -omit = pypiserver/bottle.py +omit = pypiserver/bottle_wrapper/bottle.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 26c9021d..c4513f2a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,6 @@ jobs: matrix: # make sure to align the `python-version`s in the Matrix with env.LAST_SUPPORTED_PYTHON python-version: [ - "3.7", "3.8", "3.9", "3.10", diff --git a/Dockerfile b/Dockerfile index 9dab88cc..374b910e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,11 @@ -FROM python:3.10-alpine3.20 as base +FROM python:3.10-alpine3.20 AS base # Copy the requirements & code and install them # Do this in a separate image in a separate directory # to not have all the build stuff in the final image FROM base AS builder_gosu -ENV GOSU_VERSION 1.12 +ENV GOSU_VERSION=1.12 RUN apk add --no-cache --virtual .build-deps \ ca-certificates \ diff --git a/Makefile b/Makefile index a5b0322d..82a2ebcc 100644 --- a/Makefile +++ b/Makefile @@ -8,16 +8,39 @@ SHELL = /bin/sh -MYPKG_SRC = fixtures/mypkg/setup.py $(shell find fixtures/mypkg/mypkg -type f -name '*.py') +MYPKG_DIR := fixtures/mypkg +MYPKG_HEAVY_DIR := fixtures/mypkg_heavy +MYPKG_HEAVY_PLACEHOLDER_PATH := $(MYPKG_HEAVY_DIR)/mypkg_heavy/generated_placeholder.py +MYPKG_SRC = $(MYPKG_DIR)/setup.py $(shell find $(MYPKG_DIR)/ -type f -name '*.py') +MYPKG_HEAVY_SRC = $(MYPKG_HEAVY_DIR)/setup.py $(shell find $(MYPKG_HEAVY_DIR)/ -type f -name '*.py') + +fixtures: mypkg mypkg_heavy # Build the test fixture package. -mypkg: fixtures/mypkg/dist/pypiserver_mypkg-1.0.0.tar.gz -mypkg: fixtures/mypkg/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl -fixtures/mypkg/dist/pypiserver_mypkg-1.0.0.tar.gz: $(MYPKG_SRC) - cd fixtures/mypkg; python setup.py sdist +mypkg: $(MYPKG_DIR)/dist/pypiserver_mypkg-1.0.0.tar.gz +mypkg: $(MYPKG_DIR)/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl + +$(MYPKG_DIR)/dist/pypiserver_mypkg-1.0.0.tar.gz: $(MYPKG_SRC) + cd $(MYPKG_DIR); python setup.py sdist +$(MYPKG_DIR)/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl: $(MYPKG_SRC) + cd $(MYPKG_DIR); python setup.py bdist_wheel + +# end + +# Build the heavy test fixture package. +mypkg_heavy: $(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0.tar.gz +mypkg_heavy: $(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0-py2.py3-none-any.whl -fixtures/mypkg/dist/pypiserver_mypkg-1.0.0-py2.py3-none-any.whl: $(MYPKG_SRC) - cd fixtures/mypkg; python setup.py bdist_wheel +$(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0.tar.gz: $(MYPKG_HEAVY_PLACEHOLDER_PATH) +$(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0.tar.gz: $(MYPKG_SRC) + cd $(MYPKG_HEAVY_DIR); python setup.py sdist +$(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0-py2.py3-none-any.whl: $(MYPKG_HEAVY_PLACEHOLDER_PATH) +$(MYPKG_HEAVY_DIR)/dist/pypiserver_mypkg_heavy-1.0.0-py2.py3-none-any.whl: $(MYPKG_SRC) + cd $(MYPKG_HEAVY_DIR); python setup.py bdist_wheel +$(MYPKG_HEAVY_PLACEHOLDER_PATH): $(MYPKG_HEAVY_DIR) + echo '"""' > $(MYPKG_HEAVY_PLACEHOLDER_PATH) + dd if=/dev/urandom bs=150M count=1 | base64 >> $(MYPKG_HEAVY_PLACEHOLDER_PATH) + echo '"""' >> $(MYPKG_HEAVY_PLACEHOLDER_PATH) # end diff --git a/README.md b/README.md index c09347f7..5c442983 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,11 @@ # [**pypiserver - minimal PyPI server for use with pip/easy_install**](#pypiserver) -[![pypi badge](https://img.shields.io/pypi/v/pypiserver.svg)](https://shields.io/) -[![ci workflow](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml/badge.svg)](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml) -[![Generic badge](https://img.shields.io/badge/python-3.6%7C3.7%7C3.8+-blue.svg)](https://pypi.org/project/pypiserver/) -[![Generic badge](https://img.shields.io/badge/license-MIT%7Czlib/libpng-blue.svg)](https://raw.githubusercontent.com/pypiserver/pypiserver/main/LICENSE.txt) +[![PyPi project badge](https://img.shields.io/pypi/v/pypiserver.svg)](https://pypi.org/project/pypiserver/) +[![Python Versions](https://img.shields.io/badge/python-3.8%7C3.9+-blue.svg)](https://pypi.org/project/pypiserver/) +[![CI workflow](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml/badge.svg)](https://github.com/pypiserver/pypiserver/actions/workflows/ci.yml) +[![Licenses](https://img.shields.io/badge/license-MIT%7Czlib/libpng-blue.svg)](https://raw.githubusercontent.com/pypiserver/pypiserver/main/LICENSE.txt) +[![Stable Docker Tag](https://img.shields.io/docker/v/pypiserver/pypiserver/latest?label=Docker%20Hub%20%28stable%29%0A)](https://hub.docker.com/r/pypiserver/pypiserver/tags) | name | description | | :---------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -51,6 +52,8 @@ Table of Contents - [Quickstart Installation and Usage](#quickstart-installation-and-usage) - [More details about pypi server run](#more-details-about-pypi-server-run) - [More details about pypi-server update](#more-details-about-pypi-server-update) + - [Experimental configuration flags](#experimental-configuration-flags) + - [Addressing #630](#addressing-630) - [Client-Side Configurations](#client-side-configurations) - [Configuring pip](#configuring-pip) - [Configuring easy_install](#configuring-easy_install) @@ -91,20 +94,26 @@ Table of Contents ## Quickstart Installation and Usage -**pypiserver** works with Python 3.6+ and PyPy3. +> [!IMPORTANT] +> **pypiserver** works with Python 3.8+ and PyPy3. + + -Older Python versions may still work, but they are not tested. +> [!WARNING] +> Older Python versions may still work, but they are not tested. +> +> For legacy Python versions, use **pypiserver-1.x** series.\ +> Note that these are not officially supported, and will not receive bugfixes or new features. -For legacy Python versions, use **pypiserver-1.x** series. Note that these are -not officially supported, and will not receive bugfixes or new features. + > [!TIP] > -> The commands below work on a unix-like operating system with a posix shell. +> The commands below work on a unix-like operating system with a posix shell.\ > The **'~'** character expands to user's home directory. - -If you're using Windows, you'll have to use their "Windows counterparts". -The same is true for the rest of this documentation. +> +> If you're using Windows, you'll have to use their "Windows counterparts".\ +> The same is true for the rest of this documentation. 1. Install **pypiserver** with this command @@ -344,12 +353,37 @@ optional arguments: containing arbitrary code. ``` +### Experimental Configuration Flags + +> [!WARNING] +> This section describes temporary and experimental features of **pypiserver**. +> +> They are likely to be promoted to standard features of the project or deprecated in the future. +> If you are using these features, please pay attention to the release notes. + +Additional features of **pypiserver** can be configured as environment variables. + +#### Addressing #630 + +> [!TIP] +> For more context, see discussion in #630. + +This flag allows to override the `MEMFILE_MAX` setting used by `bottle` under the hood. +Consider using it if you encounter: `MultipartError: Memory limit reached.` issue when uploading to **pypiserver**. + +```bash +PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES= +``` + ## Client-Side Configurations Always specifying the pypi url on the command line is a bit -cumbersome. Since **pypiserver** redirects **pip/easy_install** to the -**pypi.org** index if it doesn't have a requested package, it is a -good idea to configure them to always use your local pypi index. +cumbersome. + +> [!TIP] +> Since **pypiserver** redirects **pip/easy_install** to the **pypi.org** index +> if it doesn't have a requested package, it is a good idea to configure them to +> always use your local pypi index. ### Configuring pip @@ -362,7 +396,7 @@ export PIP_EXTRA_INDEX_URL=http://localhost:8080/simple/ or by adding the following lines to **~/.pip/pip.conf** -```shell +```ini [global] extra-index-url = http://localhost:8080/simple/ ``` @@ -379,7 +413,7 @@ extra-index-url = http://localhost:8080/simple/ For **easy_install** command you may set the following configuration in **~/.pydistutils.cfg** -```shell +```ini [easy_install] index_url = http://localhost:8080/simple/ ``` @@ -387,10 +421,10 @@ index_url = http://localhost:8080/simple/ ### Uploading Packages Remotely Instead of copying packages directly to the server's folder (i.e. with **scp**), -you may use python tools for the task, e.g. **python setup.py upload**. +you may use python tools for the task, e.g. **python setup.py upload**.\ In that case, **pypiserver** is responsible for authenticating the upload-requests. -> [!NOTE] +> [!IMPORTANT] > > We strongly advise to ***password-protect*** your uploads! @@ -405,14 +439,14 @@ To avoid lazy security decisions, read help for **-P** and **-a** options. (see next steps) ```shell - pip install passlib + pip install passlib ``` 1. Create the Apache **htpasswd** file with at least one user/password pair with this command (you'll be prompted for a password) ```shell - htpasswd -sc htpasswd.txt + htpasswd -sc htpasswd.txt ``` > [!TIP] @@ -433,9 +467,9 @@ To avoid lazy security decisions, read help for **-P** and **-a** options. > order to provide custom authentication. For example, to configure > pypiserver to authenticate using the [python-pam](https://pypi.org/project/python-pam/) > -> ```shell -> import pam -> pypiserver.default_config(auther=pam.authenticate) +> ```python +> import pam +> pypiserver.default_config(auther=pam.authenticate) > ``` Please see [`Using Ad-hoc authentication providers`](#using-ad-hoc-authentication-providers) for more information. @@ -444,34 +478,34 @@ Please see [`Using Ad-hoc authentication providers`](#using-ad-hoc-authenticatio (but user/password pairs can later be added or updated on the fly) ```shell - ./pypi-server run -p 8080 -P htpasswd.txt ~/packages & + ./pypi-server run -p 8080 -P htpasswd.txt ~/packages & ``` #### Upload with setuptools 1. On client-side, edit or create a **~/.pypirc** file with a similar content: - ```shell - [distutils] - index-servers = - pypi - local - - [pypi] - username: - password: - - [local] - repository: http://localhost:8080 - username: - password: + ```ini + [distutils] + index-servers = + pypi + local + + [pypi] + username: + password: + + [local] + repository: http://localhost:8080 + username: + password: ``` 1. Then from within the directory of the python-project you wish to upload, issue this command: ```shell - python setup.py sdist upload -r local + python setup.py sdist upload -r local ``` #### Upload with twine @@ -482,7 +516,7 @@ To avoid storing you passwords on disk, in clear text, you may either: like that ```shell - python setup.py sdist register -r local upload -r local + python setup.py sdist register -r local upload -r local ``` - use *twine* library, which @@ -491,13 +525,14 @@ To avoid storing you passwords on disk, in clear text, you may either: to **pypiserver**:: ```shell - twine upload -r local --sign -identity user_name ./foo-1.zip + twine upload -r local --sign -identity user_name ./foo-1.zip ``` ## Using the Docker Image Starting with version 1.2.5, official Docker images will be built for each push to `main`, each dev, alpha, or beta release, and each final release. + The most recent full release will always be available under the tag **latest**, and the current `main` branch will always be available under the tag **unstable**. @@ -508,7 +543,7 @@ You can always check to see what tags are currently available at our To run the most recent release of **pypiserver** with Docker, simply ```shell - docker run pypiserver/pypiserver:latest run +docker run pypiserver/pypiserver:latest run ``` This starts **pypiserver** serving packages from the **/data/packages** @@ -522,7 +557,7 @@ Of course, just running a container isn't that interesting. To map port 80 on the host to port 8080 on the container:: ```shell - docker run -p 80:8080 pypiserver/pypiserver:latest run +docker run -p 80:8080 pypiserver/pypiserver:latest run ``` You can now access your **pypiserver** at **localhost:80** in a web browser. @@ -530,13 +565,13 @@ You can now access your **pypiserver** at **localhost:80** in a web browser. To serve packages from a directory on the host, e.g. **~/packages** ```shell - docker run -p 80:8080 -v ~/packages:/data/packages pypiserver/pypiserver:latest run +docker run -p 80:8080 -v ~/packages:/data/packages pypiserver/pypiserver:latest run ``` To authenticate against a local **.htpasswd** file:: ```shell - docker run -p 80:8080 -v ~/.htpasswd:/data/.htpasswd pypiserver/pypiserver:latest run -P .htpasswd packages +docker run -p 80:8080 -v ~/.htpasswd:/data/.htpasswd pypiserver/pypiserver:latest run -P .htpasswd packages ``` You can also specify **pypiserver** to run as a Docker service using a @@ -635,7 +670,7 @@ reverse-proxy as described below in `Behind a reverse proxy`, you can easily enable caching. For example, to allow nginx to cache up to 10 gigabytes of data for up to 1 hour:: -```shell +```nginx proxy_cache_path /data/nginx/cache levels=1:2 keys_zone=pypiserver_cache:10m @@ -670,7 +705,7 @@ an easy task without a third party tool such as *NSSM*. it is an excellent option for managing the pypiserver process. An example config file for **systemd** can be seen below -```shell +```ini [Unit] Description=A minimal PyPI server for use with pip/easy_install. After=network.target @@ -709,7 +744,7 @@ More useful information about *systemd* can be found at package and as such, it provides excellent cross-platform support for process management. An example configuration file for **supervisor** is given below -```shell +```ini [program:pypi] command=/home/pypi/pypi-venv/bin/pypi-server run -p 7001 -P /home/pypi/.htpasswd /home/pypi/packages directory=/home/pypi @@ -728,7 +763,7 @@ For Windows download [NSSM](https://nssm.cc/) from unzip to a desired location such as Program Files. Decide whether you are going to use `win32` or `win64`, and add that `exe` to environment `PATH`. -Create a start_pypiserver.bat +Create a `start_pypiserver.bat` ```shell pypi-server run -p 8080 C:\Path\To\Packages & @@ -784,7 +819,7 @@ nssm start pypiserver - You may view all supported WSGI servers using the following interactive code ```python - >>> from pypiserver import bottle + >>> from pypiserver import bottle_wrapper as bottle >>> list(bottle.server_names.keys()) ['cgi', 'gunicorn', 'cherrypy', 'eventlet', 'tornado', 'geventSocketIO', 'rocket', 'diesel', 'twisted', 'wsgiref', 'fapws3', 'bjoern', 'gevent', @@ -811,7 +846,7 @@ explained in [bottle's documentation](http://bottlepy.org/docs/dev/deployment.ht 1. Adapt and place the following *Apache* configuration either into top-level scope, or inside some **``** (contributed by Thomas Waldmann): - ```shell + ```apache WSGIScriptAlias / /yoursite/wsgi/pypiserver-wsgi.py WSGIDaemonProcess pypisrv user=pypisrv group=pypisrv umask=0007 \ processes=1 threads=5 maximum-requests=500 \ @@ -826,7 +861,7 @@ explained in [bottle's documentation](http://bottlepy.org/docs/dev/deployment.ht or if using older **Apache < 2.4**, substitute the last part with this:: - ```shell + ```apache Order deny,allow Allow from all @@ -838,14 +873,12 @@ explained in [bottle's documentation](http://bottlepy.org/docs/dev/deployment.ht (**pypisrv:pypisrv** in the example) have the read permission on it ```python - import pypiserver conf = pypiserver.default_config( root = "/yoursite/packages", password_file = "/yoursite/htpasswd", ) application = pypiserver.app(**conf) - ``` > [!TIP] @@ -894,27 +927,27 @@ of packages on different paths. The following example **paste.ini** could be used to serve stable and unstable packages on different paths -```shell - [composite:main] - use = egg:Paste#urlmap - /unstable/ = unstable - / = stable - - [app:stable] - use = egg:pypiserver#main - root = ~/stable-packages - - [app:unstable] - use = egg:pypiserver#main - root = ~/stable-packages - ~/unstable-packages - - [server:main] - use = egg:gunicorn#main - host = 0.0.0.0 - port = 9000 - workers = 5 - accesslog = - +```ini +[composite:main] +use = egg:Paste#urlmap +/unstable/ = unstable +/ = stable + +[app:stable] +use = egg:pypiserver#main +root = ~/stable-packages + +[app:unstable] +use = egg:pypiserver#main +root = ~/stable-packages + ~/unstable-packages + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port = 9000 +workers = 5 +accesslog = - ``` > [!NOTE] @@ -938,7 +971,7 @@ You can run **pypiserver** behind a reverse proxy as well. Extend your nginx configuration -```shell +```nginx upstream pypi { server pypiserver.example.com:12345 fail_timeout=0; } @@ -959,7 +992,7 @@ As of pypiserver 1.3, you may also use the `X-Forwarded-Host` header in your reverse proxy config to enable changing the base URL. For example if you want to host pypiserver under a particular path on your server -```shell +```nginx upstream pypi { server localhost:8000; } @@ -981,7 +1014,7 @@ Using a reverse proxy is the preferred way of getting pypiserver behind HTTPS. For example, to put pypiserver behind HTTPS on port 443, with automatic HTTP redirection, using `nginx` -```shell +```nginx upstream pypi { server localhost:8000; } @@ -1061,15 +1094,13 @@ these steps: 1. Create a python-script along these lines - ```shell - $ cat > pypiserver-start.py + ```python + # pypiserver-start.py import pypiserver - from pypiserver import bottle + from pypiserver import bottle_wrapper as bottle import pam app = pypiserver.app(root='./packages', auther=pam.authenticate) bottle.run(app=app, host='0.0.0.0', port=80, server='auto') - - [Ctrl+ D] ``` 1. Invoke the python-script to start-up **pypiserver** @@ -1133,7 +1164,7 @@ pypi-server run --health-endpoint /action/health ```python import pypiserver -from pypiserver import bottle +from pypiserver import bottle_wrapper as bottle app = pypiserver.app(root="./packages", health_endpoint="/action/health") bottle.run(app=app, host="0.0.0.0", port=8080, server="auto") ``` diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100644 index 36c31551..00000000 --- a/bootstrap.py +++ /dev/null @@ -1,187 +0,0 @@ -############################################################################## -# -# Copyright (c) 2006 Zope Foundation and Contributors. -# All Rights Reserved. -# -# This software is subject to the provisions of the Zope Public License, -# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution. -# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED -# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS -# FOR A PARTICULAR PURPOSE. -# -############################################################################## -"""Bootstrap a buildout-based project - -Simply run this script in a directory containing a buildout.cfg. -The script accepts buildout command-line options, so you can -use the -c option to specify an alternate configuration file. -""" - -import os, shutil, sys, tempfile -from optparse import OptionParser - -tmpeggs = tempfile.mkdtemp() - -usage = """\ -[DESIRED PYTHON FOR BUILDOUT] bootstrap.py [options] - -Bootstraps a buildout-based project. - -Simply run this script in a directory containing a buildout.cfg, using the -Python that you want bin/buildout to use. - -Note that by using --setup-source and --download-base to point to -local resources, you can keep this script from going over the network. -""" - -parser = OptionParser(usage=usage) -parser.add_option("-v", "--version", help="use a specific zc.buildout version") - -parser.add_option( - "-t", - "--accept-buildout-test-releases", - dest="accept_buildout_test_releases", - action="store_true", - default=False, - help=( - "Normally, if you do not specify a --version, the " - "bootstrap script and buildout gets the newest " - "*final* versions of zc.buildout and its recipes and " - "extensions for you. If you use this flag, " - "bootstrap and buildout will get the newest releases " - "even if they are alphas or betas." - ), -) -parser.add_option( - "-c", - "--config-file", - help=("Specify the path to the buildout configuration " "file to be used."), -) -parser.add_option( - "-f", "--find-links", help="Specify a URL to search for buildout releases" -) - - -options, args = parser.parse_args() - -###################################################################### -# load/install distribute - -to_reload = False -try: - import pkg_resources, setuptools - - if not hasattr(pkg_resources, "_distribute"): - to_reload = True - raise ImportError -except ImportError: - ez = {} - - try: - from urllib.request import urlopen - except ImportError: - from urllib2 import urlopen - - exec(urlopen("http://python-distribute.org/distribute_setup.py").read(), ez) - setup_args = dict(to_dir=tmpeggs, download_delay=0, no_fake=True) - ez["use_setuptools"](**setup_args) - - if to_reload: - from importlib import reload - - reload(pkg_resources) - import pkg_resources - - # This does not (always?) update the default working set. We will - # do it. - for path in sys.path: - if path not in pkg_resources.working_set.entries: - pkg_resources.working_set.add_entry(path) - -###################################################################### -# Install buildout - -ws = pkg_resources.working_set - -cmd = [ - sys.executable, - "-c", - "from setuptools.command.easy_install import main; main()", - "-mZqNxd", - tmpeggs, -] - -find_links = os.environ.get( - "bootstrap-testing-find-links", - options.find_links - or ( - "http://downloads.buildout.org/" - if options.accept_buildout_test_releases - else None - ), -) -if find_links: - cmd.extend(["-f", find_links]) - -distribute_path = ws.find( - pkg_resources.Requirement.parse("distribute") -).location - -requirement = "zc.buildout" -version = options.version -if version is None and not options.accept_buildout_test_releases: - # Figure out the most recent final version of zc.buildout. - import setuptools.package_index - - _final_parts = "*final-", "*final" - - def _final_version(parsed_version): - for part in parsed_version: - if (part[:1] == "*") and (part not in _final_parts): - return False - return True - - index = setuptools.package_index.PackageIndex(search_path=[distribute_path]) - if find_links: - index.add_find_links((find_links,)) - req = pkg_resources.Requirement.parse(requirement) - if index.obtain(req) is not None: - best = [] - bestv = None - for dist in index[req.project_name]: - distv = dist.parsed_version - if _final_version(distv): - if bestv is None or distv > bestv: - best = [dist] - bestv = distv - elif distv == bestv: - best.append(dist) - if best: - best.sort() - version = best[-1].version -if version: - requirement = "==".join((requirement, version)) -cmd.append(requirement) - -import subprocess - -if subprocess.call(cmd, env=dict(os.environ, PYTHONPATH=distribute_path)) != 0: - raise Exception("Failed to execute command:\n%s", repr(cmd)[1:-1]) - -###################################################################### -# Import and run buildout - -ws.add_entry(tmpeggs) -ws.require(requirement) -import zc.buildout.buildout - -if not [a for a in args if "=" not in a]: - args.append("bootstrap") - -# if -c was provided, we push it back into args for buildout' main function -if options.config_file is not None: - args[0:0] = ["-c", options.config_file] - -zc.buildout.buildout.main(args) -shutil.rmtree(tmpeggs) diff --git a/buildout.cfg b/buildout.cfg deleted file mode 100644 index 7835de61..00000000 --- a/buildout.cfg +++ /dev/null @@ -1,59 +0,0 @@ -[buildout] - -develop = . - -parts = - pypi-server - -show-picked-versions = true -newest = false - -eggs = - pypiserver - waitress - passlib - -[gunicorn] -recipe = zc.recipe.egg -eggs = - gunicorn - Paste - PasteDeploy - ${pypi-server:eggs} -scripts = - gunicorn_paster - gunicorn - - -[pytest] -recipe = zc.recipe.egg -eggs = - pypiserver - pytest - webtest - beautifulsoup4 - ${buildout:eggs} -scripts = - py.test - -[yolk] -recipe = zc.recipe.egg -eggs = - yolk - ${gunicorn:eggs} - ${pytest:eggs} - -[pypi-server] -recipe = zc.recipe.egg -eggs = ${buildout:eggs} -interpreter = pypipython -scripts = - pypi-server - -[ipython] -recipe = zc.recipe.egg -eggs = - ipython - ${buildout:eggs} -scripts = - ipython diff --git a/docker/test_docker.py b/docker/test_docker.py index 0bf3cc5f..203e67b0 100644 --- a/docker/test_docker.py +++ b/docker/test_docker.py @@ -18,12 +18,16 @@ PYPISERVER_PROCESS_NAME = "pypi-server" TEST_DEMO_PIP_PACKAGE = "pypiserver-mypkg" +TEST_DEMO_HEAVY_PIP_PACKAGE = "pypiserver-mypkg-heavy" THIS_DIR = Path(__file__).parent ROOT_DIR = THIS_DIR.parent DOCKERFILE = ROOT_DIR / "Dockerfile" FIXTURES = ROOT_DIR / "fixtures" -MYPKG_ROOT = FIXTURES / "mypkg" +MYPKG_NAME = "mypkg" +MYPKG_ROOT = FIXTURES / MYPKG_NAME +MYPKG_HEAVY_NAME = "mypkg_heavy" +MYPKG_HEAVY_ROOT = FIXTURES / MYPKG_HEAVY_NAME HTPASS_FILE = FIXTURES / "htpasswd.a.a" @@ -37,6 +41,36 @@ # pylint: disable=no-self-use +class RunReturn(t.NamedTuple): + """Simple wrapper around a simple subprocess call's results.""" + + returncode: int + out: str + err: str + + +def run( + *cmd: str, + capture: bool = False, + raise_on_err: bool = True, + check_code: t.Callable[[int], bool] = lambda c: c == 0, + **popen_kwargs: t.Any, +) -> RunReturn: + """Run a command to completion.""" + stdout = subprocess.PIPE if capture else None + stderr = subprocess.PIPE if capture else None + proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, **popen_kwargs) + out, err = proc.communicate() + result = RunReturn( + proc.returncode, + "" if out is None else out.decode(), + "" if err is None else err.decode(), + ) + if raise_on_err and not check_code(result.returncode): + raise RuntimeError(result) + return result + + @pytest.fixture(scope="session") def image() -> str: """Build the docker image for pypiserver. @@ -57,34 +91,6 @@ def image() -> str: return tag -@pytest.fixture(scope="session") -def mypkg_build() -> None: - """Ensure the mypkg test fixture package is build.""" - # Use make for this so that it will skip the build step if it's not needed - run("make", "mypkg", cwd=ROOT_DIR) - - -@pytest.fixture(scope="session") -def mypkg_paths( - mypkg_build: None, # pylint: disable=unused-argument -) -> t.Dict[str, Path]: - """The path to the mypkg sdist file.""" - dist_dir = Path(MYPKG_ROOT) / "dist" - assert dist_dir.exists() - - sdist = dist_dir / "pypiserver_mypkg-1.0.0.tar.gz" - assert sdist.exists() - - wheel = dist_dir / "pypiserver_mypkg-1.0.0-py2.py3-none-any.whl" - assert wheel.exists() - - return { - "dist_dir": dist_dir, - "sdist": sdist, - "wheel": wheel, - } - - def wait_for_container(port: int) -> None: """Wait for the container to be available.""" for _ in range(60): @@ -113,37 +119,60 @@ def get_socket() -> int: return sock.getsockname()[1] -class RunReturn(t.NamedTuple): - """Simple wrapper around a simple subprocess call's results.""" +def _make_fixture_package(package: str) -> RunReturn: + # Use make for this so that it will skip the build step if it's not needed + return run("make", package, cwd=ROOT_DIR) - returncode: int - out: str - err: str +def _get_fixture_package_paths(root: Path, package: str) -> t.Dict[str, Path]: + dist_dir = Path(root) / "dist" + sdist = dist_dir / f"pypiserver_{package}-1.0.0.tar.gz" + wheel = dist_dir / f"pypiserver_{package}-1.0.0-py2.py3-none-any.whl" + + return {"dist_dir": dist_dir, "sdist": sdist, "wheel": wheel} + + +@pytest.fixture(scope="session") +def mypkg_build() -> None: + """Ensure the mypkg test fixture package is build.""" + _make_fixture_package(MYPKG_NAME) + + +@pytest.fixture(scope="session") +def mypkg_paths( + mypkg_build: None, # pylint: disable=unused-argument +) -> t.Dict[str, Path]: + """The path to the mypkg sdist file.""" + paths = _get_fixture_package_paths(MYPKG_ROOT, MYPKG_NAME) + + assert paths["dist_dir"].exists() + assert paths["sdist"].exists() + assert paths["wheel"].exists() + + return paths + + +@pytest.fixture(scope="session") +def mypkg_heavy_build() -> None: + """Ensure the mypkg_heavy test fixture package is build.""" + _make_fixture_package(MYPKG_HEAVY_NAME) -def run( - *cmd: str, - capture: bool = False, - raise_on_err: bool = True, - check_code: t.Callable[[int], bool] = lambda c: c == 0, - **popen_kwargs: t.Any, -) -> RunReturn: - """Run a command to completion.""" - stdout = subprocess.PIPE if capture else None - stderr = subprocess.PIPE if capture else None - proc = subprocess.Popen(cmd, stdout=stdout, stderr=stderr, **popen_kwargs) - out, err = proc.communicate() - result = RunReturn( - proc.returncode, - "" if out is None else out.decode(), - "" if err is None else err.decode(), - ) - if raise_on_err and not check_code(result.returncode): - raise RuntimeError(result) - return result +@pytest.fixture(scope="session") +def mypkg_heavy_paths( + mypkg_heavy_build: None, # pylint: disable=unused-argument +) -> t.Dict[str, Path]: + """The path to the mypkg sdist file.""" + paths = _get_fixture_package_paths(MYPKG_HEAVY_ROOT, MYPKG_HEAVY_NAME) + + assert paths["dist_dir"].exists() + assert paths["sdist"].exists() + assert paths["wheel"].exists() + + return paths -def uninstall_pkgs() -> None: + +def uninstall_packages() -> None: """Uninstall any packages we've installed.""" res = run("pip", "freeze", capture=True) if any( @@ -157,14 +186,18 @@ def uninstall_pkgs() -> None: def session_cleanup() -> t.Iterator[None]: """Deal with any pollution of the local env.""" yield - uninstall_pkgs() + uninstall_packages() @pytest.fixture() def cleanup() -> t.Iterator[None]: """Clean up after tests that may have affected the env.""" yield - uninstall_pkgs() + uninstall_packages() + + +# Test cases +# ---------- class TestCommands: @@ -596,3 +629,83 @@ def test_welcome(self) -> None: resp = httpx.get(f"http://localhost:{self.HOST_PORT}") assert resp.status_code == 200 assert "pypiserver" in resp.text + + +class TestHeavyPackage: + """Test for a workaround of https://github.com/pypiserver/pypiserver/issues/630.""" + + override_with_100_mb = ( + f"PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES={(2**20) * 100}" + ) + + @pytest.fixture(scope="class") + def patched_container( + self, request: pytest.FixtureRequest, image: str + ) -> t.Iterator[ContainerInfo]: + """Run the pypiserver container. + + Returns the container ID. + """ + port = get_socket() + args = ( + "docker", + "run", + "--rm", + "--env", + self.override_with_100_mb, + "--publish", + f"{port}:8080", + "--detach", + image, + "run", + "--passwords", + ".", + "--authenticate", + ".", + ) + res = run(*args, capture=True) + wait_for_container(port) + container_id = res.out.strip() + yield ContainerInfo(container_id, port, args) + run("docker", "container", "rm", "-f", container_id) + + @pytest.fixture(scope="class") + def upload_mypkg_heavy( + self, + patched_container: ContainerInfo, + mypkg_heavy_paths: t.Dict[str, Path], + ) -> None: + """Upload mypkg to the container.""" + run( + sys.executable, + "-m", + "twine", + "upload", + "--repository-url", + f"http://localhost:{patched_container.port}", + "--username", + "a", + "--password", + "a", + f"{mypkg_heavy_paths['dist_dir']}/*", + ) + + @pytest.mark.usefixtures("upload_mypkg_heavy") + def test_download(self, patched_container: ContainerInfo) -> None: + """Download mypkg_heavy from the container.""" + with tempfile.TemporaryDirectory() as tmpdir: + run( + sys.executable, + "-m", + "pip", + "download", + "--index-url", + f"http://localhost:{patched_container.port}/simple", + "--dest", + tmpdir, + "pypiserver_mypkg_heavy", + ) + assert any( + "pypiserver_mypkg_heavy" in path.name + for path in Path(tmpdir).iterdir() + ) diff --git a/fixtures/README.md b/fixtures/README.md new file mode 100644 index 00000000..f4132323 --- /dev/null +++ b/fixtures/README.md @@ -0,0 +1,7 @@ +# Testing Fixtures + +This directory contains various 'fixture' test data. + +> [!IMPORTANT] +> +> Dynamic and static test data should be placed in this folder. diff --git a/fixtures/mypkg/setup.cfg b/fixtures/mypkg/setup.cfg index 7c6e4aff..53df4012 100644 --- a/fixtures/mypkg/setup.cfg +++ b/fixtures/mypkg/setup.cfg @@ -4,4 +4,3 @@ universal=1 [mypy] follow_imports = silent ignore_missing_imports = True - diff --git a/fixtures/mypkg_heavy/mypkg_heavy/.gitignore b/fixtures/mypkg_heavy/mypkg_heavy/.gitignore new file mode 100644 index 00000000..e57dd7fc --- /dev/null +++ b/fixtures/mypkg_heavy/mypkg_heavy/.gitignore @@ -0,0 +1,2 @@ +# this ~1.4 MB file is generated using the root Makefile +generated_placeholder.py \ No newline at end of file diff --git a/fixtures/mypkg_heavy/mypkg_heavy/__init__.py b/fixtures/mypkg_heavy/mypkg_heavy/__init__.py new file mode 100644 index 00000000..77a0e05d --- /dev/null +++ b/fixtures/mypkg_heavy/mypkg_heavy/__init__.py @@ -0,0 +1,10 @@ +""" +This imitates a a heavy package. + +- The ~1.4MB `generated_placehoder.py` is generated using the root `Makefile`. +""" + + +def pkg_name() -> None: + """Print the package name.""" + print("mypkg_heavy") diff --git a/fixtures/mypkg_heavy/setup.cfg b/fixtures/mypkg_heavy/setup.cfg new file mode 100644 index 00000000..53df4012 --- /dev/null +++ b/fixtures/mypkg_heavy/setup.cfg @@ -0,0 +1,6 @@ +[wheel] +universal=1 + +[mypy] +follow_imports = silent +ignore_missing_imports = True diff --git a/fixtures/mypkg_heavy/setup.py b/fixtures/mypkg_heavy/setup.py new file mode 100644 index 00000000..effc43b3 --- /dev/null +++ b/fixtures/mypkg_heavy/setup.py @@ -0,0 +1,10 @@ +"""A simple setup file for the 'HEAVY' test package.""" + +from setuptools import setup + +setup( + name="pypiserver_mypkg_heavy", + description="Heavy test pkg", + version="1.0.0", + packages=["mypkg_heavy"], +) diff --git a/pypiserver/__init__.py b/pypiserver/__init__.py index 67091bc7..284672ac 100644 --- a/pypiserver/__init__.py +++ b/pypiserver/__init__.py @@ -4,7 +4,7 @@ import sys import typing as t -from pypiserver.bottle import Bottle +from pypiserver.bottle_wrapper import Bottle from pypiserver.config import Config, RunConfig, strtobool version = __version__ = "2.3.2" diff --git a/pypiserver/__main__.py b/pypiserver/__main__.py index a47418b1..8385b859 100644 --- a/pypiserver/__main__.py +++ b/pypiserver/__main__.py @@ -160,7 +160,7 @@ def main(argv: t.Sequence[str] = None) -> None: gevent.monkey.patch_all() - from pypiserver import bottle + from pypiserver import bottle_wrapper as bottle bottle.debug(config.verbosity > 1) bottle._stderr = ft.partial( # pylint: disable=protected-access diff --git a/pypiserver/_app.py b/pypiserver/_app.py index ed6ae200..41e13e3d 100644 --- a/pypiserver/_app.py +++ b/pypiserver/_app.py @@ -13,7 +13,7 @@ from pypiserver.config import RunConfig from . import __version__ -from .bottle import ( +from .bottle_wrapper import ( static_file, redirect, request, diff --git a/pypiserver/bottle_wrapper/__init__.py b/pypiserver/bottle_wrapper/__init__.py new file mode 100644 index 00000000..2e6346c3 --- /dev/null +++ b/pypiserver/bottle_wrapper/__init__.py @@ -0,0 +1,12 @@ +""" +This `__init__.py` allows to wrap and patch the default `bottle.py` implementation. +""" + +from pypiserver.bottle_wrapper.bottle import * +from pypiserver.environment import Environment + +BaseRequest.MEMFILE_MAX = ( + override + if (override := Environment.PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES) + else BaseRequest.MEMFILE_MAX +) diff --git a/pypiserver/bottle.py b/pypiserver/bottle_wrapper/bottle.py similarity index 100% rename from pypiserver/bottle.py rename to pypiserver/bottle_wrapper/bottle.py diff --git a/pypiserver/environment.py b/pypiserver/environment.py new file mode 100644 index 00000000..d4ab9e9d --- /dev/null +++ b/pypiserver/environment.py @@ -0,0 +1,16 @@ +import os +from typing import Optional + + +class Environment: + """This class contains various environment configurations for Pypi-Server.""" + + PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES: Optional[int] = ( + int(override) + if ( + override := os.getenv( + "PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES", + ) + ) + else None + ) diff --git a/pypiserver/plugin.py b/pypiserver/plugin.py index fc845a66..8777cc16 100644 --- a/pypiserver/plugin.py +++ b/pypiserver/plugin.py @@ -1,4 +1,4 @@ -""" NOT YET IMPLEMENTED +"""NOT YET IMPLEMENTED Plugins are callable setuptools entrypoints that are invoked at startup that a developer may use to extend the behaviour of pypiserver. A plugin for example diff --git a/pyproject.toml b/pyproject.toml index 1ff56aa4..e8dc7486 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ exclude = ''' | build | dist | venv - | pypiserver/bottle.py + | pypiserver/bottle_wrapper/bottle.py ) ) ''' diff --git a/requirements/dev.pip b/requirements/dev.pip index 420bac17..59abbf9f 100644 --- a/requirements/dev.pip +++ b/requirements/dev.pip @@ -8,5 +8,5 @@ black docopt # For `/bin/bumpver.py`. -types-setuptools # For mypy stubs +types-setuptools==75.8.0.20250225 # For mypy stubs mypy; implementation_name == 'cpython' diff --git a/setup.py b/setup.py index fa976c09..01cae518 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import re from pathlib import Path -from setuptools import setup +from setuptools import setup, find_packages tests_require = [ "pytest>=2.3", @@ -52,9 +52,9 @@ def get_version(): long_description=read_file("README.md"), long_description_content_type="text/markdown", version=get_version(), - packages=["pypiserver"], + packages=find_packages(include=["pypiserver", "pypiserver.*"]), package_data={"pypiserver": ["welcome.html"]}, - python_requires=">=3.6", + python_requires=">=3.8", install_requires=install_requires, setup_requires=setup_requires, extras_require={"passlib": ["passlib>=1.6"], "cache": ["watchdog"]}, @@ -77,7 +77,6 @@ def get_version(): "Operating System :: POSIX", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", diff --git a/tests/test_app.py b/tests/test_app.py index 230a03ea..06d773cb 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,7 +1,6 @@ #! /usr/bin/env py.test # Builtin imports -import logging import os import pathlib import xmlrpc.client as xmlrpclib @@ -13,8 +12,8 @@ # Local Imports from tests.test_pkg_helpers import files, invalid_files -from pypiserver import __main__, bottle, core, Bottle, _app -from pypiserver.backend import CachingFileBackend, SimpleFileBackend +from pypiserver import __main__, bottle_wrapper, _app +from pypiserver.backend import CachingFileBackend # Enable logging to detect any problems with it ## @@ -37,7 +36,7 @@ def app(tmpdir): @pytest.fixture def testapp(app): """Return a webtest TestApp initiated with pypiserver app""" - bottle.debug(True) + bottle_wrapper.debug(True) return webtest.TestApp(app) @@ -49,7 +48,7 @@ def root(tmpdir): @pytest.fixture def priv(app): - b = bottle.Bottle() + b = bottle_wrapper.Bottle() b.mount("/priv/", app) return b @@ -732,7 +731,7 @@ def test_remove_pkg_notFound(self, root, testapp): def test_redirect_project_encodes_newlines(): """Ensure raw newlines are url encoded in the generated redirect.""" project = "\nSet-Cookie:malicious=1;" - request = bottle.Request( + request = bottle_wrapper.Request( {"HTTP_X_FORWARDED_PROTO": "/\nSet-Cookie:malicious=1;"} ) newpath = _app.get_bad_url_redirect_path(request, project) diff --git a/tests/test_environment.py b/tests/test_environment.py new file mode 100644 index 00000000..bdb275ea --- /dev/null +++ b/tests/test_environment.py @@ -0,0 +1,34 @@ +from importlib import reload + +import pytest + +import pypiserver.environment as environment + + +def test_default_bottle_memfile_is_none(monkeypatch): + monkeypatch.delenv( + "PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES", + raising=False, + ) + + reload(environment) + + assert ( + environment.Environment.PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES + is None + ), "expected default None value" + + +def test_override_bottle_memfile_is_set(monkeypatch): + value_100_mb = (2**20) * 100 + monkeypatch.setenv( + "PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES", + str(value_100_mb), + ) + + reload(environment) + + assert ( + environment.Environment.PYPISERVER_BOTTLE_MEMFILE_MAX_OVERRIDE_BYTES + == value_100_mb + ), "expected new 100 mb value" diff --git a/tests/test_init.py b/tests/test_init.py index befc3393..67de547b 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -99,7 +99,7 @@ def test_backwards_compat_kwargs_conversion( ), ) def test_backwards_compat_kwargs_duplicate_check( - kwargs: t.Dict[str, t.Any] + kwargs: t.Dict[str, t.Any], ) -> None: """Duplicate legacy and modern kwargs cause an error.""" with pytest.raises(ValueError) as err: diff --git a/tests/test_main.py b/tests/test_main.py index 5fcace74..da4aee50 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -7,10 +7,8 @@ import pytest -import pypiserver.bottle +import pypiserver.bottle_wrapper as bottle from pypiserver import __main__ -from pypiserver.bottle import Bottle - THIS_DIR = pathlib.Path(__file__).parent HTPASS_FILE = THIS_DIR / "../fixtures/htpasswd.a.a" @@ -18,7 +16,7 @@ class main_wrapper: - app: t.Optional[Bottle] + app: t.Optional[bottle.Bottle] run_kwargs: t.Optional[dict] update_args: t.Optional[tuple] update_kwargs: t.Optional[dict] @@ -51,7 +49,7 @@ def update(*args, **kwargs): main.update_args = args main.update_kwargs = kwargs - monkeypatch.setattr("pypiserver.bottle.run", run) + monkeypatch.setattr("pypiserver.bottle_wrapper.run", run) monkeypatch.setattr("pypiserver.manage.update_all_packages", update) return main @@ -262,7 +260,7 @@ def test_auto_servers() -> None: """Test auto servers.""" # A list of bottle ServerAdapters bottle_adapters = tuple( - a.__name__.lower() for a in pypiserver.bottle.AutoServer.adapters + a.__name__.lower() for a in bottle.AutoServer.adapters ) # We are going to expect that our AutoServer enum names must match those # at least closely enough to be recognizable. diff --git a/tox.ini b/tox.ini index e4f5fa31..b2e8c533 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37, py38, py39, py310, py311, py312, pypy3 +envlist = py38, py39, py310, py311, py312, pypy3 [testenv] deps=-r{toxinidir}/requirements/test.pip