diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3f5ec41..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,27 +0,0 @@ -os: Visual Studio 2015 - -platform: - - x64 - - x86 - -cache: - - sw -> win-ci.py - -environment: - matrix: - - PY: 36 - - -build_script: - - ps: | - If ($env:Platform -Match "x86") { - $env:VCVARS_PLATFORM="x86" - } Else { - $env:VCVARS_PLATFORM="amd64" - } - - call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" %VCVARS_PLATFORM% - - C:/Python36-x64/python.exe win-ci.py install_deps - - git clone --depth 1 "https://github.com/html5lib/html5lib-tests.git" test/html5lib-tests - -test_script: - - C:/Python36-x64/python.exe win-ci.py test diff --git a/.github/workflows/ci.py b/.github/workflows/ci.py new file mode 100644 index 0000000..86f8607 --- /dev/null +++ b/.github/workflows/ci.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +# License: Apache 2.0 Copyright: 2017, Kovid Goyal + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import shlex +import subprocess +import sys + + +is_macos = 'darwin' in sys.platform.lower() + + +def run(*a): + if len(a) == 1: + a = shlex.split(a[0]) + ret = subprocess.Popen(a).wait() + if ret != 0: + print('Running:', a, 'failed', file=sys.stderr) + raise SystemExit(ret) + + +def install_deps(): + if is_macos: + pass + else: + run('sudo apt-get update') + run('sudo apt-get install -y libxml2-dev libxslt-dev') + deps = 'chardet lxml beautifulsoup4'.split() + if sys.version_info.major == 2: + deps.append('BeautifulSoup') + run(sys.executable, '-m', 'pip', 'install', '--no-binary', 'lxml', *deps) + run(sys.executable, '-c', 'from lxml import etree; print(etree)') + + +def main(): + which = sys.argv[-1] + if hasattr(sys, 'getwindowsversion'): + run(sys.executable, os.path.join(os.path.dirname(__file__), 'win-ci.py'), which) + return + if which == 'install': + install_deps() + elif which == 'test': + builder = os.environ['BUILDER'] + run(sys.executable, builder, 'test') + if builder == 'build.py': + run(sys.executable, builder, 'leak') + else: + raise SystemExit('Unknown action:', which) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bc88958 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,49 @@ +name: CI +on: [push, pull_request] +env: + CI: 'true' + LC_ALL: en_US.UTF-8 + LANG: en_US.UTF-8 + +jobs: + test: + name: Test on ${{ matrix.os }} (python=${{ matrix.pyver }} cc=${{ matrix.cc }} builder=${{ matrix.builder }}) + runs-on: ${{ matrix.os }} + env: + CC: ${{ matrix.cc }} + BUILDER: ${{ matrix.builder }} + strategy: + matrix: + include: + - { pyver: 2.7, builder: build.py, os: ubuntu-latest, cc: gcc } + - { pyver: 2.7, builder: build.py, os: ubuntu-latest, cc: clang } + - { pyver: 3.6, builder: build.py, os: ubuntu-latest, cc: gcc } + - { pyver: 3.6, builder: build.py, os: ubuntu-latest, cc: clang } + - { pyver: 3.8, builder: setup.py, os: ubuntu-latest, cc: gcc } + + - { pyver: 3.8, builder: setup.py, os: macos-latest, cc: clang } + + - { pyver: 3.8, builder: setup.py, os: windows-latest, cc: cl } + + steps: + - name: Checkout source code + uses: actions/checkout@master + with: + fetch-depth: 10 + + - name: Set up Python ${{ matrix.pyver }} + uses: actions/setup-python@master + with: + python-version: ${{ matrix.pyver }} + + - name: Install dependencies + run: + python .github/workflows/ci.py install + + - name: Download html5lib tests + run: + git clone --depth 1 https://github.com/html5lib/html5lib-tests.git test/html5lib-tests + + - name: Run tests + run: + python .github/workflows/ci.py test diff --git a/win-ci.py b/.github/workflows/win-ci.py similarity index 75% rename from win-ci.py rename to .github/workflows/win-ci.py index 4a86b1b..bbc896b 100644 --- a/win-ci.py +++ b/.github/workflows/win-ci.py @@ -19,14 +19,12 @@ ZLIB = "http://zlib.net/zlib-{}.tar.xz".format("1.2.11") LIBXML2 = "ftp://xmlsoft.org/libxml2/libxml2-{}.tar.gz".format('2.9.4') LIBXSLT = "ftp://xmlsoft.org/libxml2/libxslt-{}.tar.gz".format('1.1.28') -LXML = "https://pypi.python.org/packages/20/b3/9f245de14b7696e2d2a386c0b09032a2ff6625270761d6543827e667d8de/lxml-3.8.0.tar.gz" # noqa +LXML = "https://files.pythonhosted.org/packages/c5/2f/a0d8aa3eee6d53d5723d89e1fc32eee11e76801b424e30b55c7aa6302b01/lxml-4.6.1.tar.gz" # noqa SW = os.path.abspath('sw') -if 'PY' in os.environ and 'Platform' in os.environ: - PYTHON = os.path.expandvars('C:\\Python%PY%-%Platform%\\python.exe').replace('-x86', '') -else: - PYTHON = sys.executable +PYTHON = os.path.abspath(sys.executable) os.environ['SW'] = SW -os.environ['PYTHONPATH'] = os.path.expandvars('%SW%\\python\\Lib\\site-packages;%PYTHONPATH%') +os.environ['PYTHONPATH'] = os.path.join(SW, r'python\Lib\site-packages') +plat = 'amd64' if sys.maxsize > 2**32 else 'x86' def printf(*a, **k): @@ -84,6 +82,55 @@ def run(*args, env=None, cwd=None): raise SystemExit(p.returncode) +def distutils_vcvars(): + from distutils.msvc9compiler import find_vcvarsall, get_build_version + return find_vcvarsall(get_build_version()) + + +def remove_dups(variable): + old_list = variable.split(os.pathsep) + new_list = [] + for i in old_list: + if i not in new_list: + new_list.append(i) + return os.pathsep.join(new_list) + + +def query_process(cmd): + if plat == 'amd64' and 'PROGRAMFILES(x86)' not in os.environ: + os.environ['PROGRAMFILES(x86)'] = os.environ['PROGRAMFILES'] + ' (x86)' + result = {} + popen = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + try: + stdout, stderr = popen.communicate() + if popen.wait() != 0: + raise RuntimeError(stderr.decode("mbcs")) + + stdout = stdout.decode("mbcs") + for line in stdout.splitlines(): + if '=' not in line: + continue + line = line.strip() + key, value = line.split('=', 1) + key = key.lower() + if key == 'path': + if value.endswith(os.pathsep): + value = value[:-1] + value = remove_dups(value) + result[key] = value + + finally: + popen.stdout.close() + popen.stderr.close() + return result + + +def query_vcvarsall(): + vcvarsall = distutils_vcvars() + return query_process('"%s" %s & set' % (vcvarsall, plat)) + + def download_and_extract(url): raw = io.BytesIO(download_file(url)) with tarfile.open(fileobj=raw, mode='r:*') as f: @@ -210,9 +257,13 @@ def lxml(): 'setup.py build_ext -I {0}/include;{0}/include/libxml2 -L {0}/lib'.format( SW.replace(os.sep, '/')).split())) run(PYTHON, 'setup.py', 'install', '--prefix', os.path.join(SW, 'python')) + package = glob.glob(os.path.join(SW, 'python', 'lib', 'site-packages', 'lxml-*.egg', 'lxml'))[0] + os.rename(package, os.path.join(SW, 'python', 'lib', 'site-packages', 'lxml')) def install_deps(): + env = query_vcvarsall() + os.environ.update(env) print(PYTHON) for x in 'build lib bin include python/Lib/site-packages'.split(): ensure_dir(os.path.join(SW, x)) @@ -227,25 +278,26 @@ def install_deps(): try: download_and_extract(globals()[name.upper()]) globals()[name]() - except: + except Exception: os.chdir(base) shutil.rmtree(name) raise def build(): - p = os.environ['PATH'] - p = os.path.join(SW, 'bin') + os.pathsep + p - env = dict( + env = query_vcvarsall() + os.environ.update(env) + os.environ.update(dict( LIBXML_INCLUDE_DIRS=r'{0}\include;{0}\include\libxml2'.format(SW), LIBXML_LIB_DIRS=r'{0}\lib'.format(SW), - PATH=p - ) - run(PYTHON, 'setup.py', 'test', env=env) + HTML5_PARSER_DLL_DIR=os.path.join(SW, 'bin'), + )) + print('Using PYTHONPATH:', os.environ['PYTHONPATH']) + run(PYTHON, 'setup.py', 'test') def main(): - if sys.argv[-1] == 'install_deps': + if sys.argv[-1] == 'install': install_deps() else: build() diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 155dd49..0000000 --- a/.travis.yml +++ /dev/null @@ -1,75 +0,0 @@ -env: - global: - - PYTHONHASHSEED=random - -matrix: - include: - - os: linux - language: python - python: 2.7 - env: BUILDER=build.py CC=gcc PYTHON=python - group: beta - dist: trusty - sudo: false - addons: - apt: - packages: - - libxml2-dev - - os: linux - language: python - python: 2.7 - env: BUILDER=build.py CC=clang PYTHON=python LSAN_OPTIONS=verbosity=1:log_threads=1 - group: beta - dist: trusty - # See https://github.com/travis-ci/travis-ci/issues/9033 - sudo: required - addons: - apt: - packages: - - libxml2-dev - - os: linux - language: python - python: 2.7 - env: BUILDER=setup.py PYTHON=python - group: beta - dist: trusty - sudo: false - addons: - apt: - packages: - - libxml2-dev - - os: linux - language: python - python: 3.6 - env: BUILDER=setup.py PYTHON=python - group: beta - dist: trusty - sudo: false - addons: - apt: - packages: - - libxml2-dev - - os: osx - language: generic - env: BUILDER=setup.py PYTHON=python3 - -install: | - set -e - if [[ "$TRAVIS_OS_NAME" == 'osx' ]]; then - brew update; - brew upgrade python; - python3 --version - pip3 install --no-binary lxml chardet lxml beautifulsoup4 - else - PLIB=$(ldd `which python` | grep libpython | cut -d ' ' -f 3) - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:`dirname $PLIB` - pip install --no-binary lxml chardet lxml beautifulsoup4 - if [[ $TRAVIS_PYTHON_VERSION == 2.* ]]; then pip install BeautifulSoup; fi - fi - $PYTHON -c "from lxml import etree; print(etree)" - git clone --depth 1 "https://github.com/html5lib/html5lib-tests.git" test/html5lib-tests - set +e - -script: - - $PYTHON $BUILDER test - - if [[ $BUILDER == "build.py" ]]; then $PYTHON $BUILDER leak; fi diff --git a/build.py b/build.py index 7d639a2..55834b6 100755 --- a/build.py +++ b/build.py @@ -25,7 +25,7 @@ _plat = sys.platform.lower() isosx = 'darwin' in _plat iswindows = hasattr(sys, 'getwindowsversion') -is_travis = os.environ.get('TRAVIS') == 'true' +is_ci = os.environ.get('CI') == 'true' Env = namedtuple('Env', 'cc cflags ldflags linker debug cc_name cc_ver') PKGCONFIG = os.environ.get('PKGCONFIG_EXE', 'pkg-config') with open(os.path.join(base, 'src/python-wrapper.c'), 'rb') as f: @@ -209,7 +209,7 @@ def build_obj(src, env, headers): TEST_EXE = os.path.join(build_dir, 'test') MEMLEAK_EXE = os.path.join(build_dir, 'mem-leak-check') -if is_travis: +if is_ci: TEST_EXE = os.path.join(os.path.dirname(os.path.abspath(sys.executable)), 'test-html5-parser') SRC_DIRS = 'src gumbo'.split() MOD_EXT = '.so' diff --git a/run_tests.py b/run_tests.py index 54f8bcd..0d50c6c 100644 --- a/run_tests.py +++ b/run_tests.py @@ -10,6 +10,12 @@ import sys import unittest +if 'HTML5_PARSER_DLL_DIR' in os.environ: + sys.save_dll_dir = os.add_dll_directory(os.environ['HTML5_PARSER_DLL_DIR']) + print('Added DLL directory', sys.save_dll_dir, 'with contents:', + os.listdir(os.environ['HTML5_PARSER_DLL_DIR'])) + print('Current sys.path:', sys.path) + self_path = os.path.abspath(__file__) base = os.path.dirname(self_path) html5lib_tests_path = os.path.join(base, 'test', 'html5lib-tests')