Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e99d3a5

Browse files
zoobazooba
authored
[3.6] bpo-30450: Improved logic for obtaining dependencies (#2751)
Adds alternate download approach for nuget.exe Fall back to git.exe if no Python is found. (#2739) Also check whether git.exe is on PATH if it will be used. Add support for HOST_PYTHON variable. Clear internal environment variables used in find_python.bat Use HOST_PYTHON as the actual Python if it is recent enough. Adds HOST_PYTHON variable to AppVeyor configuration
1 parent 241c4a2 commit e99d3a5

8 files changed

Lines changed: 145 additions & 47 deletions

File tree

‎.github/appveyor.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ build_script:
99
- cmd: PCbuild\build.bat -e
1010
test_script:
1111
- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0
12+
environment:
13+
HOST_PYTHON: C:\Python36\python.exe
1214

1315
# Only trigger AppVeyor if actual code or its configuration changes
1416
only_commits:

‎Doc/make.bat‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,19 @@ pushd %~dp0
55

66
set this=%~n0
77

8-
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
8+
call ..\PCBuild\find_python.bat %PYTHON%
9+
if "%SPHINXBUILD%" EQU "" if "%PYTHON%" NEQ "" (
10+
set SPHINXBUILD=%PYTHON%\..\Scripts\sphinx-build.exe
11+
rem Cannot use %SPHINXBUILD% in the same block where we set it
12+
if not exist "%PYTHON%\..\Scripts\sphinx-build.exe" (
13+
echo Installing sphinx with %PYTHON%
14+
"%PYTHON%" -m pip install sphinx
15+
if errorlevel 1 exit /B
16+
)
17+
)
18+
919
if "%PYTHON%" EQU "" set PYTHON=py
20+
if "%SPHINXBUILD%" EQU "" set SPHINXBUILD=sphinx-build
1021

1122
if "%1" NEQ "htmlhelp" goto :skiphhcsearch
1223
if exist "%HTMLHELP%" goto :skiphhcsearch

‎PCbuild/build.bat‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options]
55
echo.
66
echo.Build CPython from the command line. Requires the appropriate
77
echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt).
8-
echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is
9-
echo.given.
108
echo.
119
echo.After the flags recognized by this script, up to 9 arguments to be passed
1210
echo.directly to MSBuild may be passed. If the argument contains an '=', the

‎PCbuild/find_python.bat‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
@rem
2+
@rem Searches for python.exe and may download a private copy from nuget.
3+
@rem
4+
@rem This file is supposed to modify the state of the caller (specifically
5+
@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid
6+
@rem changing any other persistent state.
7+
@rem
8+
9+
@rem No arguments provided means do full search
10+
@if '%1' EQU '' goto :begin_search
11+
12+
@rem One argument may be the full path. Use a goto so we don't try to
13+
@rem parse the next if statement - incorrect quoting in the multi-arg
14+
@rem case can cause us to break immediately.
15+
@if '%2' EQU '' goto :one_arg
16+
17+
@rem Entire command line may represent the full path if quoting failed.
18+
@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found
19+
@goto :begin_search
20+
21+
:one_arg
22+
@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found
23+
24+
:begin_search
25+
@set PYTHON=
26+
27+
@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR%
28+
@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals)
29+
30+
@rem If we have Python in externals, use that one
31+
@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found
32+
33+
@rem If HOST_PYTHON is recent enough, use that
34+
@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found
35+
36+
@rem If py.exe finds a recent enough version, use that one
37+
@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found
38+
39+
@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%"
40+
@set _Py_NUGET=%NUGET%
41+
@set _Py_NUGET_URL=%NUGET_URL%
42+
@set _Py_HOST_PYTHON=%HOST_PYTHON%
43+
@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py
44+
@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe)
45+
@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl)
46+
@if NOT exist "%_Py_NUGET%" (
47+
@echo Downloading nuget...
48+
@rem NB: Must use single quotes around NUGET here, NOT double!
49+
@rem Otherwise, a space in the path would break things
50+
@rem If it fails, retry with any available copy of Python
51+
@powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%'
52+
@if errorlevel 1 (
53+
@%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%"
54+
)
55+
)
56+
@echo Installing Python via nuget...
57+
@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%"
58+
@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
59+
@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found
60+
61+
62+
@set _Py_Python_Source=
63+
@set _Py_EXTERNALS_DIR=
64+
@set _Py_NUGET=
65+
@set _Py_NUGET_URL=
66+
@set _Py_HOST_PYTHON=
67+
@exit /b 1
68+
69+
:found
70+
@echo Using %PYTHON% (%_Py_Python_Source%)
71+
@set _Py_Python_Source=
72+
@set _Py_EXTERNALS_DIR=
73+
@set _Py_NUGET=
74+
@set _Py_NUGET_URL=
75+
@set _Py_HOST_PYTHON=

‎PCbuild/get_externals.bat‎

Lines changed: 13 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,14 @@ rem Simple script to fetch source for external libraries
44

55
if "%PCBUILD%"=="" (set PCBUILD=%~dp0)
66
if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals)
7-
if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe)
8-
if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl)
97

108
set DO_FETCH=true
119
set DO_CLEAN=false
1210

1311
:CheckOpts
1412
if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts
1513
if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts
16-
if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
14+
if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts
1715
if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
1816
if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
1917
if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
@@ -33,22 +31,10 @@ if "%DO_FETCH%"=="false" goto end
3331

3432
if "%ORG%"=="" (set ORG=python)
3533

36-
if "%PYTHON_FOR_BUILD%"=="" (
37-
echo Checking for installed python...
38-
py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
39-
)
40-
if "%PYTHON_FOR_BUILD%"=="" (
41-
if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
42-
if NOT exist "%NUGET%" (
43-
echo Downloading nuget...
44-
rem NB: Must use single quotes around NUGET here, NOT double!
45-
rem Otherwise, a space in the path would break things
46-
powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
47-
)
48-
echo Installing Python via nuget...
49-
"%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
50-
rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
51-
set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
34+
call "%PCBUILD%find_python.bat" "%PYTHON%"
35+
36+
if "%PYTHON%"=="" (
37+
where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1
5238
)
5339

5440
echo.Fetching external libraries...
@@ -65,9 +51,12 @@ set libraries=%libraries% xz-5.2.2
6551
for %%e in (%libraries%) do (
6652
if exist "%EXTERNALS_DIR%\%%e" (
6753
echo.%%e already exists, skipping.
54+
) else if "%PYTHON%"=="" (
55+
echo.Fetching %%e with git...
56+
git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e"
6857
) else (
6958
echo.Fetching %%e...
70-
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e
59+
%PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e
7160
)
7261
)
7362

@@ -80,9 +69,12 @@ if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06
8069
for %%b in (%binaries%) do (
8170
if exist "%EXTERNALS_DIR%\%%b" (
8271
echo.%%b already exists, skipping.
72+
) else if "%PYTHON%"=="" (
73+
echo.Fetching %%b with git...
74+
git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b"
8375
) else (
8476
echo.Fetching %%b...
85-
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b
77+
%PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b
8678
)
8779
)
8880

‎PCbuild/urlretrieve.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Simple Python script to download a file. Used as a fallback
2+
# when other more reliable methods fail.
3+
#
4+
from __future__ import print_function
5+
import sys
6+
7+
try:
8+
from requests import get
9+
except ImportError:
10+
try:
11+
from urllib.request import urlretrieve
12+
USING = "urllib.request.urlretrieve"
13+
except ImportError:
14+
try:
15+
from urllib import urlretrieve
16+
USING = "urllib.retrieve"
17+
except ImportError:
18+
print("Python at", sys.executable, "is not suitable",
19+
"for downloading files.", file=sys.stderr)
20+
sys.exit(2)
21+
else:
22+
USING = "requests.get"
23+
24+
def urlretrieve(url, filename):
25+
r = get(url, stream=True)
26+
r.raise_for_status()
27+
with open(filename, 'wb') as f:
28+
for chunk in r.iter_content(chunk_size=1024):
29+
f.write(chunk)
30+
return filename
31+
32+
if __name__ == '__main__':
33+
if len(sys.argv) != 3:
34+
print("Usage: urlretrieve.py [url] [filename]", file=sys.stderr)
35+
sys.exit(1)
36+
URL = sys.argv[1]
37+
FILENAME = sys.argv[2]
38+
print("Downloading from", URL, "to", FILENAME, "using", USING)
39+
urlretrieve(URL, FILENAME)

‎Tools/msi/buildrelease.bat‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,6 @@ if ERRORLEVEL 1 (echo Cannot locate MSBuild.exe on PATH or as MSBUILD variable &
7676
if "%SKIPBUILD%" EQU "1" goto skipdoc
7777
if "%SKIPDOC%" EQU "1" goto skipdoc
7878

79-
if not defined PYTHON where py -q || echo Cannot find py on path and PYTHON is not set. && exit /B 1
80-
if not defined SPHINXBUILD where sphinx-build -q || echo Cannot find sphinx-build on path and SPHINXBUILD is not set. && exit /B 1
81-
8279
call "%D%..\..\doc\make.bat" htmlhelp
8380
if errorlevel 1 goto :eof
8481
:skipdoc

‎Tools/msi/get_externals.bat‎

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ set DO_FETCH=true
1212
set DO_CLEAN=false
1313

1414
:CheckOpts
15-
if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts
15+
if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts
1616
if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts
1717
if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts
1818
if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts
@@ -32,23 +32,7 @@ if "%DO_FETCH%"=="false" goto end
3232

3333
if "%ORG%"=="" (set ORG=python)
3434

35-
if "%PYTHON_FOR_BUILD%"=="" (
36-
echo Checking for installed python...
37-
py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6)
38-
)
39-
if "%PYTHON_FOR_BUILD%"=="" (
40-
if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%"
41-
if NOT exist "%NUGET%" (
42-
echo Downloading nuget...
43-
rem NB: Must use single quotes around NUGET here, NOT double!
44-
rem Otherwise, a space in the path would break things
45-
powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%'
46-
)
47-
echo Installing Python via nuget...
48-
"%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%"
49-
rem Quote it here; it's not quoted later because "py -3.6" wouldn't work
50-
set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe"
51-
)
35+
call "%PCBUILD%\find_python.bat" "%PYTHON%"
5236

5337
echo.Fetching external libraries...
5438

@@ -59,7 +43,7 @@ for %%e in (%libraries%) do (
5943
echo.%%e already exists, skipping.
6044
) else (
6145
echo.Fetching %%e...
62-
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
46+
%PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -O %ORG% %%e
6347
)
6448
)
6549

@@ -79,7 +63,7 @@ for %%b in (%binaries%) do (
7963
echo.%%b already exists, skipping.
8064
) else (
8165
echo.Fetching %%b...
82-
%PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
66+
%PYTHON% "%PCBUILD%get_external.py" -e "%EXTERNALS_DIR%" -b -O %ORG% %%b
8367
)
8468
)
8569

0 commit comments

Comments
 (0)