From 689ab7af9d45f2743fef2948379e25af0182df44 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 25 Sep 2019 15:58:10 +1000 Subject: [PATCH 001/430] 4.23 compat: disable HoloLens by default --- .../ue4-minimal/windows/Dockerfile | 4 ++++ .../ue4-minimal/windows/patch-build-graph.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 07374545..5a47b26a 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -12,6 +12,10 @@ RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version COPY patch-filters-xml.py C:\patch-filters-xml.py RUN python C:\patch-filters-xml.py C:\UnrealEngine\Engine\Build\InstalledEngineFilters.xml +# Patch out problematic entries in InstalledEngineBuild.xml introduced in UE4.23.0 +COPY patch-build-graph.py C:\patch-build-graph.py +RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineBuild.xml + # Create an Installed Build of the Engine # (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) ARG EXCLUDE_DEBUG diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py new file mode 100644 index 00000000..2d8d2053 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import os, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Prevent HoloLens support from being enabled by default under Windows in 4.23.0 onwards +buildXml = sys.argv[1] +code = readFile(buildXml) +code = code.replace( + 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', + 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"' +) +writeFile(buildXml, code) From ff1faef1036a71beeef9fd8ee41676224dd6aa09 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Oct 2019 18:16:50 +1000 Subject: [PATCH 002/430] Fix #53 --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 4 ++-- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index fc62d1e9..49c73c1b 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -12,14 +12,14 @@ RUN mkdir C:\GatheredDlls # Install 7-Zip, curl, and Python using Chocolatey # (Note that these need to be separate RUN directives for `choco` to work) RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" -RUN choco install -y 7zip curl python +RUN choco install -y 7zip curl && choco install -y python --version=3.7.5 # Copy the required DirectSound/DirectDraw and OpenGL DLL files from the host system (since these ship with Windows and don't have installers) COPY *.dll C:\GatheredDlls\ # Verify that the DLL files copied from the host can be loaded by the container OS ARG HOST_VERSION -RUN pip install pypiwin32 +RUN pip install pywin32 COPY copy.py verify-host-dlls.py C:\ RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ RUN python C:\verify-host-dlls.py %HOST_VERSION% C:\GatheredDlls diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 19cdb415..12a89f9a 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,6 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error -choco install -y curl python vcredist-all || goto :error +choco install -y curl vcredist-all || goto :error +choco install -y python --version=3.7.5 || goto :error @rem Reload our environment variables from the registry so the `git` command works call refreshenv From d2bf16b1974364a7d4a0bdba7593f96dcbbc5c26 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Oct 2019 18:33:45 +1000 Subject: [PATCH 003/430] Explicitly blacklist Windows version 1903 --- ue4docker/build.py | 8 ++++++++ ue4docker/infrastructure/WindowsUtils.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 90ca1eca..a8e25a42 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -82,6 +82,14 @@ def build(): logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) + # Verify that the host OS is not a release that is blacklisted due to critical bugs + if WindowsUtils.isBlacklistedWindowsVersion() == True: + logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) + logger.error('This version of Windows contains one or more critical bugs that', False) + logger.error('render it incapable of successfully building UE4 container images.', False) + logger.error('You will need to use an older or newer version of Windows.', False) + sys.exit(1) + # Verify that the user is not attempting to build images with a newer kernel version than the host OS if WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag): logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 1790275f..3088580d 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -10,11 +10,14 @@ class WindowsUtils(object): # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 17763 + _latestReleaseBuild = 18362 # The list of Windows Server Core base image tags that we support, in ascending version number order _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019'] + # The list of Windows Server and Windows 10 releases that are blacklisted due to critical bugs + _blacklistedReleases = ['1903'] + @staticmethod def _getVersionRegKey(subkey): ''' @@ -97,6 +100,13 @@ def getWindowsBuild(): build = WindowsUtils._getVersionRegKey('BuildLabEx').split('.')[1] return '{}.{}'.format(version, build) + @staticmethod + def isBlacklistedWindowsVersion(): + ''' + Determines if the Windows host system is a release with bugs that make it unsuitable for use + ''' + return WindowsUtils.getWindowsRelease() in WindowsUtils._blacklistedReleases + @staticmethod def isSupportedWindowsVersion(): ''' From 5bfdacc67cd1dc354f4fdc77f0ed64830e0bfdd8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Oct 2019 18:35:10 +1000 Subject: [PATCH 004/430] Bump version to 0.0.37 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index df5521bd..951a2115 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.36' +__version__ = '0.0.37' From 01a94b2a756254210b6a69eddf1055a5423bbe21 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 19 Oct 2019 10:52:25 +1000 Subject: [PATCH 005/430] Fix #54 --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 49c73c1b..02ee5c03 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -49,6 +49,9 @@ COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ # Add a sentinel label so we can easily identify all derived images, including intermediate images LABEL com.adamrehn.ue4-docker.sentinel="1" +# Enable long path support +RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f + # Install Chocolatey RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" From 0ea6c0025eb595e67335f6bb1b0ec18dbcddb010 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 19 Oct 2019 10:52:54 +1000 Subject: [PATCH 006/430] Bump version to 0.0.38 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 951a2115..2c18b358 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.37' +__version__ = '0.0.38' From 2cc9f1b4d6376e04375b15f5c7cef379be76b42d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 31 Oct 2019 16:47:30 +1000 Subject: [PATCH 007/430] Re-enable OpenGL for 4.23 under Linux --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 4 ++++ .../ue4-minimal/linux/enable-opengl.py | 19 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 7bbfc4f9..2d35ec76 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -7,6 +7,10 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder COPY set-changelist.py /tmp/set-changelist.py RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version +# Enable the OpenGL RHI for Engine versions where it is present but deprecated +COPY enable-opengl.py /tmp/enable-opengl.py +RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngine.ini + # Increase the output verbosity of the DDC generation step COPY verbose-ddc.py /tmp/verbose-ddc.py RUN python3 /tmp/verbose-ddc.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py b/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py new file mode 100644 index 00000000..87d4670f --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import os, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Enable the OpenGL RHI for Engine versions where it is present but deprecated +iniFile = sys.argv[1] +config = readFile(iniFile) +config = config.replace( + '; +TargetedRHIs=GLSL_430', + '+TargetedRHIs=GLSL_430' +) +writeFile(iniFile, config) From 91f4381aa9c030ad6431f4c7f4dc0510219dbdd6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 8 Nov 2019 23:35:08 +1000 Subject: [PATCH 008/430] Enable client and server targets by default in 4.23.0 onwards --- .../ue4-minimal/windows/patch-build-graph.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py index 2d8d2053..8b30764f 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py @@ -9,11 +9,25 @@ def writeFile(filename, data): with open(filename, 'wb') as f: f.write(data.encode('utf-8')) -# Prevent HoloLens support from being enabled by default under Windows in 4.23.0 onwards +# Read the build graph XML buildXml = sys.argv[1] code = readFile(buildXml) + +# Prevent HoloLens support from being enabled by default under Windows in 4.23.0 onwards code = code.replace( 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"' ) + +# Enable client and server targets by default in 4.23.0 onwards +code = code.replace( + 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' +) +code = code.replace( + 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' +) + +# Write the modified XML back to disk writeFile(buildXml, code) From 1af961e6ee413d32936fc16c2ae2c180a0ae811f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 8 Nov 2019 23:35:33 +1000 Subject: [PATCH 009/430] Bump version to 0.0.39 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 2c18b358..16270b50 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.38' +__version__ = '0.0.39' From 7023a56cdae5d1ea5b576c4bc654a8e826bb3a57 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 Nov 2019 12:59:41 +1000 Subject: [PATCH 010/430] Patch BaseEngine.ini to ensure Client and Server targets are detected correctly --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 4 ++++ .../dockerfiles/ue4-minimal/linux/fix-targets.py | 16 ++++++++++++++++ .../dockerfiles/ue4-minimal/windows/Dockerfile | 4 ++++ .../ue4-minimal/windows/fix-targets.py | 16 ++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py create mode 100644 ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 2d35ec76..02730227 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -29,6 +29,10 @@ COPY exclude-components.py /tmp/exclude-components.py RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && \ python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES +# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini +COPY fix-targets.py /tmp/fix-targets.py +RUN python3 /tmp/fix-targets.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Config/BaseEngine.ini + # Some versions of the Engine fail to include UnrealPak in the Installed Build, so copy it manually RUN cp ./Engine/Binaries/Linux/UnrealPak ./LocalBuilds/Engine/Linux/Engine/Binaries/Linux/UnrealPak diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py b/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py new file mode 100644 index 00000000..f573d403 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import os, re, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini +iniFile = sys.argv[1] +config = readFile(iniFile) +config = re.sub('PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', 'PlatformType="\\2", RequiredFile="\\1"', config) +writeFile(iniFile, config) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 5a47b26a..0620a34b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -25,6 +25,10 @@ COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && ` python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% +# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini +COPY fix-targets.py C:\fix-targets.py +RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\Config\BaseEngine.ini + # Copy our legacy toolchain installation script into the Installed Build # (This ensures we only need one COPY directive below) RUN C:\copy.py C:\legacy-toolchain-fix.py C:\UnrealEngine\LocalBuilds\Engine\Windows\ diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py b/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py new file mode 100644 index 00000000..f573d403 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +import os, re, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini +iniFile = sys.argv[1] +config = readFile(iniFile) +config = re.sub('PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', 'PlatformType="\\2", RequiredFile="\\1"', config) +writeFile(iniFile, config) From 69e80fdb872082a0639f4f4e942b71c5f9913455 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 Nov 2019 13:00:19 +1000 Subject: [PATCH 011/430] Bump version to 0.0.40 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 16270b50..d8195dfc 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.39' +__version__ = '0.0.40' From 2383ea210911b01ddaeb218a6a6c7376af1010b4 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 Nov 2019 01:57:00 +1000 Subject: [PATCH 012/430] Abort builds if the credential endpoint fails to start --- ue4docker/infrastructure/CredentialEndpoint.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/ue4docker/infrastructure/CredentialEndpoint.py index bb12e8fd..76c02ec1 100644 --- a/ue4docker/infrastructure/CredentialEndpoint.py +++ b/ue4docker/infrastructure/CredentialEndpoint.py @@ -33,12 +33,20 @@ def start(self): ''' Starts the HTTP endpoint as a child process ''' + + # Create a child process to run the credential endpoint self.endpoint = multiprocessing.Process( target = CredentialEndpoint._endpoint, args=(self.username, self.password, self.token) ) + + # Spawn the child process and give the endpoint time to start self.endpoint.start() time.sleep(2) + + # Verify that the endpoint started correctly + if self.endpoint.is_alive() == False: + raise RuntimeError('failed to start the credential endpoint') def stop(self): ''' From bcee77c0ab575b4e95fc2afb1c37794b69c6c8cf Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 Nov 2019 01:57:32 +1000 Subject: [PATCH 013/430] Bump version to 0.0.41 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index d8195dfc..4af9abb7 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.40' +__version__ = '0.0.41' From 55a464d1190b5311c12c7057e888a24a72145dd3 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 Nov 2019 02:06:34 +1000 Subject: [PATCH 014/430] Prevent git clone from hanging on blank credentials --- ue4docker/infrastructure/CredentialEndpoint.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/ue4docker/infrastructure/CredentialEndpoint.py index 76c02ec1..312ae3a8 100644 --- a/ue4docker/infrastructure/CredentialEndpoint.py +++ b/ue4docker/infrastructure/CredentialEndpoint.py @@ -8,8 +8,10 @@ def __init__(self, username, password): ''' Creates an endpoint manager for the supplied credentials ''' - self.username = username - self.password = password + + # Make sure neither our username or password are blank, since that can cause `git clone` to hang indefinitely + self.username = username if username is not None and len(username) > 0 else ' ' + self.password = password if password is not None and len(password) > 0 else ' ' self.endpoint = None # Generate a security token to require when requesting credentials From 83ca3db6c5d2322f0bd8e62b8a0542fbf7b74d99 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 Nov 2019 02:07:18 +1000 Subject: [PATCH 015/430] Bump version to 0.0.42 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 4af9abb7..8c0b2695 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.41' +__version__ = '0.0.42' From 2cd1876dec3970ad0f6ae1d94489fef881445f80 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 10 Dec 2019 11:23:05 +1000 Subject: [PATCH 016/430] Add support for CUDA 10.2 base image --- ue4docker/infrastructure/BuildConfiguration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 67aa2975..654454f9 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -16,7 +16,8 @@ 'cudagl': { '9.2': 'nvidia/cudagl:9.2-devel-ubuntu18.04', '10.0': 'nvidia/cudagl:10.0-devel-ubuntu18.04', - '10.1': 'nvidia/cudagl:10.1-devel-ubuntu18.04' + '10.1': 'nvidia/cudagl:10.1-devel-ubuntu18.04', + '10.2': 'nvidia/cudagl:10.2-devel-ubuntu18.04' } } From 6b7e654ae53a8a087d0452f60b63b598a148a02b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 10 Dec 2019 11:23:31 +1000 Subject: [PATCH 017/430] Bump version to 0.0.43 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8c0b2695..bd93a4a2 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.42' +__version__ = '0.0.43' From 4d28541e15500c19437dab7e2b3081856dfb7ba5 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 13 Dec 2019 16:45:49 +1000 Subject: [PATCH 018/430] Fix #60 --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 4 ++++ ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 48ec3229..932271b0 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -63,3 +63,7 @@ RUN echo "rm -rf /var/lib/apt/lists/*" >> ./root_commands.sh # so redirect the bundled clang to use the system linker instead COPY --chown=ue4:ue4 linker-fixup.py /tmp/linker-fixup.py RUN python3 /tmp/linker-fixup.py /home/ue4/UnrealEngine/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64 `which ld` + +# Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards +# (For details of what this is, see: ) +RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 0c4dfb34..1718153a 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -37,3 +37,7 @@ RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat # Run post-clone setup steps WORKDIR C:\UnrealEngine RUN Setup.bat + +# Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards +# (For details of what this is, see: ) +RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX From 6e45a185792630a5b23fe7a1b0dc48b716a36197 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 13 Dec 2019 16:46:31 +1000 Subject: [PATCH 019/430] Bump version to 0.0.44 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index bd93a4a2..abebd785 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.43' +__version__ = '0.0.44' From 3afde42fe900902451476a90697e78e0355590c8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 13 Dec 2019 23:02:11 +1000 Subject: [PATCH 020/430] Ignore errors deleting `XXX` dummy platform under Windows --- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 1718153a..cd97b3f1 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -40,4 +40,4 @@ RUN Setup.bat # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) -RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX +RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 From 8286aa79736028bd45953c73542ed3a874574930 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 13 Dec 2019 23:02:34 +1000 Subject: [PATCH 021/430] Bump version to 0.0.45 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index abebd785..369521b2 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.44' +__version__ = '0.0.45' From 715f302c40c3c561a5e667e3818c477962ad72d2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 30 Jan 2020 22:38:56 +1000 Subject: [PATCH 022/430] Add functionality for monitoring system resources during builds --- ue4docker/build.py | 11 ++- .../infrastructure/BuildConfiguration.py | 2 + ue4docker/infrastructure/ResourceMonitor.py | 76 +++++++++++++++++++ ue4docker/infrastructure/__init__.py | 1 + 4 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 ue4docker/infrastructure/ResourceMonitor.py diff --git a/ue4docker/build.py b/ue4docker/build.py index a8e25a42..97d4994e 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -158,6 +158,11 @@ def build(): password = _getPassword(config.args) print() + # If resource monitoring has been enabled, start the resource monitoring background thread + resourceMonitor = ResourceMonitor(logger, config.args.interval) + if config.args.monitor == True: + resourceMonitor.start() + # Start the HTTP credential endpoint as a child process and wait for it to start endpoint = CredentialEndpoint(username, password) endpoint.start() @@ -215,12 +220,16 @@ def build(): endTime = time.time() logger.action('Total execution time: {}'.format(humanfriendly.format_timespan(endTime - startTime))) + # Stop the resource monitoring background thread if it is running + resourceMonitor.stop() + # Stop the HTTP server endpoint.stop() - except Exception as e: + except (Exception, KeyboardInterrupt) as e: # One of the images failed to build logger.error('Error: {}'.format(e)) + resourceMonitor.stop() endpoint.stop() sys.exit(1) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 654454f9..0565ae5d 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -81,6 +81,8 @@ def addArguments(parser): parser.add_argument('-dlldir', default=None, help='Set the directory to copy required Windows DLLs from (default is the host System32 directory)') parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images') parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') + parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') + parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') def __init__(self, parser, argv): ''' diff --git a/ue4docker/infrastructure/ResourceMonitor.py b/ue4docker/infrastructure/ResourceMonitor.py new file mode 100644 index 00000000..5cdc65ce --- /dev/null +++ b/ue4docker/infrastructure/ResourceMonitor.py @@ -0,0 +1,76 @@ +import datetime, humanfriendly, psutil, shutil, threading, time +from .DockerUtils import DockerUtils + +class ResourceMonitor(threading.Thread): + + def __init__(self, logger, interval): + ''' + Creates a resource monitor with the specified configuration + ''' + super().__init__() + self._logger = logger + self._interval = interval + self._lock = threading.Lock() + self._shouldStop = False + + def stop(self): + ''' + Stops the resource monitor thread + ''' + + # Set the flag to instruct the resource monitor loop to stop + with self._lock: + self._shouldStop = True + + # Wait for the resource monitor thread to complete + if self.is_alive() == True: + self.join() + + def run(self): + ''' + The resource monitor loop itself + ''' + + # Determine which filesystem the Docker daemon uses for storing its data directory + dockerInfo = DockerUtils.info() + rootDir = dockerInfo['DockerRootDir'] + + # Sample the CPU usage using an interval of 1 second the first time to prime the system + # (See: ) + psutil.cpu_percent(1.0) + + # Loop until asked to stop + while True: + + # Check that the thread has not been asked to stop + with self._lock: + if self._shouldStop == True: + return + + # Format the timestamp for the current time in ISO 8601 format (albeit without the "T" separator) + isoTime = datetime.datetime.now().replace(microsecond=0).isoformat(' ') + + # We format data sizes using binary units (KiB, MiB, GiB, etc.) + formatSize = lambda size: humanfriendly.format_size(size, binary=True, keep_width=True) + + # Format the current quantity of available disk space on the Docker data directory's filesystem + diskSpace = formatSize(shutil.disk_usage(rootDir).free) + + # Format the current quantity of available system memory + physicalMemory = formatSize(psutil.virtual_memory().free) + virtualMemory = formatSize(psutil.swap_memory().free) + + # Format the current CPU usage levels + cpu = psutil.cpu_percent() + + # Report the current levels of our available resources + self._logger.info('[{}] [Available disk: {}] [Available memory: {} physical, {} virtual] [CPU usage: {:.2f}%]'.format( + isoTime, + diskSpace, + physicalMemory, + virtualMemory, + cpu + ), False) + + # Sleep until the next sampling interval + time.sleep(self._interval) diff --git a/ue4docker/infrastructure/__init__.py b/ue4docker/infrastructure/__init__.py index 600abb9f..7ffdb9c5 100644 --- a/ue4docker/infrastructure/__init__.py +++ b/ue4docker/infrastructure/__init__.py @@ -10,5 +10,6 @@ from .NetworkUtils import NetworkUtils from .PackageUtils import PackageUtils from .PrettyPrinting import PrettyPrinting +from .ResourceMonitor import ResourceMonitor from .SubprocessUtils import SubprocessUtils from .WindowsUtils import WindowsUtils From f8320ddcf4c153ff05c08ec4dd5cbe360bf08b79 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 30 Jan 2020 22:39:16 +1000 Subject: [PATCH 023/430] Bump version to 0.0.46 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 369521b2..4948c025 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.45' +__version__ = '0.0.46' From 0d699950cf2959737e2802a06d8868bcaa23d52b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 31 Jan 2020 14:45:44 +1000 Subject: [PATCH 024/430] Add `ue4-docker diagnostics` command --- setup.py | 8 +- ue4docker/diagnostics/__init__.py | 2 + ue4docker/diagnostics/base.py | 19 +++++ ue4docker/diagnostics/diagnostic_8gig.py | 78 +++++++++++++++++++ ue4docker/diagnostics/diagnostic_all.py | 38 +++++++++ ue4docker/diagnostics_cmd.py | 61 +++++++++++++++ .../diagnostics/8gig/linux/Dockerfile | 7 ++ .../diagnostics/8gig/windows/Dockerfile | 24 ++++++ ue4docker/infrastructure/DockerUtils.py | 4 +- ue4docker/main.py | 5 ++ 10 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 ue4docker/diagnostics/__init__.py create mode 100644 ue4docker/diagnostics/base.py create mode 100644 ue4docker/diagnostics/diagnostic_8gig.py create mode 100644 ue4docker/diagnostics/diagnostic_all.py create mode 100644 ue4docker/diagnostics_cmd.py create mode 100644 ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile create mode 100644 ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile diff --git a/setup.py b/setup.py index 483b4a74..bb69bfbe 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,12 @@ author='Adam Rehn', author_email='adam@adamrehn.com', license='MIT', - packages=['ue4docker', 'ue4docker.exports', 'ue4docker.infrastructure'], + packages=[ + 'ue4docker', + 'ue4docker.diagnostics', + 'ue4docker.exports', + 'ue4docker.infrastructure' + ], zip_safe=False, python_requires = '>=3.6', install_requires = [ @@ -46,6 +51,7 @@ package_data = { 'ue4docker': [ 'dockerfiles/*/*/.dockerignore', + 'dockerfiles/diagnostics/*/*/*', 'dockerfiles/*/*/*' ] }, diff --git a/ue4docker/diagnostics/__init__.py b/ue4docker/diagnostics/__init__.py new file mode 100644 index 00000000..521a5fe6 --- /dev/null +++ b/ue4docker/diagnostics/__init__.py @@ -0,0 +1,2 @@ +from .diagnostic_all import allDiagnostics +from .diagnostic_8gig import diagnostic8Gig diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py new file mode 100644 index 00000000..4b3ed074 --- /dev/null +++ b/ue4docker/diagnostics/base.py @@ -0,0 +1,19 @@ +class DiagnosticBase(object): + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + raise NotImplementedError + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + raise NotImplementedError + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + raise NotImplementedError diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py new file mode 100644 index 00000000..5d4c085e --- /dev/null +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -0,0 +1,78 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase + +import os, platform, subprocess +from os.path import abspath, dirname, join + +class diagnostic8Gig(DiagnosticBase): + + # The tag we use for built images + IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:8gig' + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + return 'Check for Docker 8GiB filesystem layer bug' + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + return '\n'.join([ + 'This diagnostic determines if the Docker daemon suffers from the 8GiB filesystem', + 'layer bug reported here: https://github.com/moby/moby/issues/37581', + '', + 'This bug was fixed in Docker CE 18.09.0, but still exists in some versions of', + 'Docker CE under Windows 10 and Docker EE under Windows Server.', + '', + 'You can force the use of a Linux image under Windows hosts by specifying the', + '--linux flag, which can be useful when testing LCOW support.' + ]) + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = 'linux' if '--linux' in args or platform.system().lower() != 'windows' else 'windows' + + # Under Windows host systems, determine the appropriate container image base tag + buildArgs = [ + '-m', '4GiB', + '--build-arg', + 'BASETAG=' + WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) + ] if containerPlatform == 'windows' else [] + + # Attempt to build the Dockerfile + contextDir = join(dirname(dirname(abspath(__file__))), 'dockerfiles', 'diagnostics', '8gig', containerPlatform) + try: + logger.action('[8gig] Attempting to build an image with an 8GiB filesystem layer...', False) + command = ['docker', 'build', '-t', diagnostic8Gig.IMAGE_TAG, contextDir] + buildArgs + self._printAndRun(logger, command, check=True) + built = True + except: + logger.error('[8gig] Build failed!') + built = False + + # Remove any built images, including intermediate images + logger.action('[8gig] Cleaning up...', False) + if built == True: + self._printAndRun(logger, ['docker', 'rmi', diagnostic8Gig.IMAGE_TAG]) + self._printAndRun(logger, ['docker', 'system', 'prune', '-f']) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action('[8gig] Diagnostic succeeded! The Docker daemon can build images with 8GiB filesystem layers.\n') + else: + logger.error('[8gig] Diagnostic failed! The Docker daemon cannot build images with 8GiB filesystem layers.\n', True) + + return built + + def _printAndRun(self, logger, command, check=False): + ''' + Prints a command and then executes it + ''' + logger.info('[8gig] Run: {}'.format(command), False) + subprocess.run(command, check=check) diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py new file mode 100644 index 00000000..dcc53da9 --- /dev/null +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -0,0 +1,38 @@ +from .base import DiagnosticBase +from .diagnostic_8gig import diagnostic8Gig + +class allDiagnostics(DiagnosticBase): + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + return 'Run all available diagnostics' + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + return 'This diagnostic runs all available diagnostics in sequence.' + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + + # Run all available diagnostics in turn, storing the results + results = [] + diagnostics = [diagnostic8Gig()] + for index, diagnostic in enumerate(diagnostics): + + # Run the diagnostic and report its result + logger.info('[all] Running individual diagnostic: "{}"'.format(diagnostic.getName()), True) + results.append(diagnostic.run(logger)) + logger.info('[all] Individual diagnostic result: {}'.format('passed' if results[-1] == True else 'failed'), False) + + # Print a newline after the last diagnostic has run + if index == len(diagnostics) - 1: + print() + + # Only report success if all diagnostics succeeded + return False not in results diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py new file mode 100644 index 00000000..9d961b02 --- /dev/null +++ b/ue4docker/diagnostics_cmd.py @@ -0,0 +1,61 @@ +from .infrastructure import Logger, PrettyPrinting +from .diagnostics import * +import sys + +def diagnostics(): + + # The diagnostics that can be run + DIAGNOSTICS = { + 'all': allDiagnostics(), + '8gig': diagnostic8Gig() + } + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix='[{} diagnostics] '.format(sys.argv[0])) + + # Parse the supplied command-line arguments + stripped = list([arg for arg in sys.argv if arg.strip('-') not in ['h', 'help']]) + args = { + 'help': len(sys.argv) > len(stripped), + 'diagnostic': stripped[1] if len(stripped) > 1 else None, + } + + # If a diagnostic name has been specified, verify that it is valid + if args['diagnostic'] is not None and args['diagnostic'] not in DIAGNOSTICS: + logger.error('Error: unrecognised diagnostic "{}".'.format(args['diagnostic']), False) + sys.exit(1) + + # Determine if we are running a diagnostic + if args['help'] == False and args['diagnostic'] is not None: + + # Run the diagnostic + diagnostic = DIAGNOSTICS[args['diagnostic']] + logger.action('Running diagnostic: "{}"'.format(diagnostic.getName()), False) + passed = diagnostic.run(logger, stripped[2:]) + + # Print the result + if passed == True: + logger.action('Diagnostic result: passed', False) + else: + logger.error('Diagnostic result: failed', False) + + # Determine if we are displaying the help for a specific diagnostic + elif args['help'] == True and args['diagnostic'] is not None: + + # Display the help for the diagnostic + diagnostic = DIAGNOSTICS[args['diagnostic']] + print('{} diagnostics {}'.format(sys.argv[0], args['diagnostic'])) + print(diagnostic.getName() + '\n') + print(diagnostic.getDescription()) + + else: + + # Print usage syntax + print('Usage: {} diagnostics DIAGNOSTIC\n'.format(sys.argv[0])) + print('Runs diagnostics to detect issues with the host system configuration\n') + print('Available diagnostics:') + PrettyPrinting.printColumns([ + (diagnostic, DIAGNOSTICS[diagnostic].getName()) + for diagnostic in DIAGNOSTICS + ]) + print('\nRun `{} diagnostics DIAGNOSTIC --help` for more information on a diagnostic.'.format(sys.argv[0])) diff --git a/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile b/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile new file mode 100644 index 00000000..fb5f463d --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# The BusyBox version of `head` doesn't support the syntax "8G", so we specify 8GiB in bytes +RUN head -c 8589934592 file diff --git a/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile b/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile new file mode 100644 index 00000000..f295dba1 --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile @@ -0,0 +1,24 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} +SHELL ["cmd", "/S", "/C"] + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Write an 8GiB file in blocks of 1GiB to avoid allocating too much memory in one hit +RUN powershell "` + $ErrorActionPreference = 'Stop';` + $writer = [System.IO.File]::OpenWrite('file');` + ` + $random = new-object Random;` + $blockSize = 1073741824;` + $bytes = new-object byte[] $blockSize;` + for ($i=0; $i -lt 8; $i++)` + {` + $random.NextBytes($bytes);` + $writer.Write($bytes, 0, $blockSize);` + }` + ` + $writer.Close();` + " diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 9f3e61c4..04f89f26 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -112,14 +112,14 @@ def maxsize(): return 20.0 @staticmethod - def listImages(tagFilter = None, filters = {}): + def listImages(tagFilter = None, filters = {}, all=False): ''' Retrieves the details for each image matching the specified filters ''' # Retrieve the list of images matching the specified filters client = docker.from_env() - images = client.images.list(filters=filters) + images = client.images.list(filters=filters, all=all) # Apply our tag filter if one was specified if tagFilter is not None: diff --git a/ue4docker/main.py b/ue4docker/main.py index 3b7b5ef8..62072ce8 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -1,6 +1,7 @@ from .infrastructure import DarwinUtils, DockerUtils, Logger, PrettyPrinting, WindowsUtils from .build import build from .clean import clean +from .diagnostics_cmd import diagnostics from .export import export from .info import info from .setup_cmd import setup @@ -41,6 +42,10 @@ def main(): 'function': clean, 'description': 'Cleans built container images' }, + 'diagnostics': { + 'function': diagnostics, + 'description': 'Runs diagnostics to detect issues with the host system configuration' + }, 'export': { 'function': export, 'description': 'Exports components from built container images to the host system' From 56ebd11d6ac13572daba7159d911dd308c97cf4d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 31 Jan 2020 14:46:29 +1000 Subject: [PATCH 025/430] Bump version to 0.0.47 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 4948c025..97e658fe 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.46' +__version__ = '0.0.47' From db2579d57665e57587e41a5df3c885b5bc2cac1c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 2 Feb 2020 11:32:24 +1000 Subject: [PATCH 026/430] Improve error messages for Docker Desktop platform mismatches --- ue4docker/build.py | 11 +++++++++++ ue4docker/diagnostics/diagnostic_8gig.py | 13 ++++++++++++- ue4docker/info.py | 7 +++++-- ue4docker/infrastructure/ResourceMonitor.py | 7 +++++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 97d4994e..55e55c8c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -50,6 +50,17 @@ def build(): logger.error('Error: could not detect Docker version. Please ensure Docker is installed.') sys.exit(1) + # Verify that we aren't trying to build Windows containers under Windows 10 when in Linux container mode (or vice versa) + dockerPlatform = DockerUtils.info()['OSType'].lower() + if config.containerPlatform == 'windows' and dockerPlatform == 'linux': + logger.error('Error: cannot build Windows containers when Docker Desktop is in Linux container', False) + logger.error('mode. Use the --linux flag if you want to build Linux containers instead.', False) + sys.exit(1) + elif config.containerPlatform == 'linux' and dockerPlatform == 'windows': + logger.error('Error: cannot build Linux containers when Docker Desktop is in Windows container', False) + logger.error('mode. Remove the --linux flag if you want to build Windows containers instead.', False) + sys.exit(1) + # Create an auto-deleting temporary directory to hold our build context with tempfile.TemporaryDirectory() as tempDir: diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index 5d4c085e..e136da37 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -27,7 +27,7 @@ def getDescription(self): 'Docker CE under Windows 10 and Docker EE under Windows Server.', '', 'You can force the use of a Linux image under Windows hosts by specifying the', - '--linux flag, which can be useful when testing LCOW support.' + '--linux flag, which can be useful when testing Docker Desktop or LCOW support.' ]) def run(self, logger, args=[]): @@ -38,6 +38,17 @@ def run(self, logger, args=[]): # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) containerPlatform = 'linux' if '--linux' in args or platform.system().lower() != 'windows' else 'windows' + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + dockerPlatform = DockerUtils.info()['OSType'].lower() + if containerPlatform == 'windows' and dockerPlatform == 'linux': + logger.error('[8gig] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.', False) + logger.error('[8gig] Use the --linux flag if you want to test Linux containers instead.', False) + return False + elif containerPlatform == 'linux' and dockerPlatform == 'windows': + logger.error('[8gig] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.', False) + logger.error('[8gig] Remove the --linux flag if you want to test Windows containers instead.', False) + return False + # Under Windows host systems, determine the appropriate container image base tag buildArgs = [ '-m', '4GiB', diff --git a/ue4docker/info.py b/ue4docker/info.py index c43f3b63..2db9935f 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -26,8 +26,11 @@ def info(): maxSize = DockerUtils.maxsize() rootDir = dockerInfo['DockerRootDir'] + # If we are communicating with a Linux Docker daemon under Windows or macOS then we can't query the available disk space + canQueryDisk = dockerInfo['OSType'].lower() == platform.system().lower() + # Gather our information about the host system - diskSpace = shutil.disk_usage(rootDir).free + diskSpace = _formatSize(shutil.disk_usage(rootDir).free) if canQueryDisk == True else 'Unknown (typically means the Docker daemon is running in a Moby VM, e.g. Docker Desktop)' memPhysical = psutil.virtual_memory().total memVirtual = psutil.swap_memory().total cpuPhysical = psutil.cpu_count(False) @@ -47,7 +50,7 @@ def info(): ('Docker daemon version', dockerInfo['ServerVersion']), ('NVIDIA Docker supported', 'Yes' if nvidiaDocker == True else 'No'), ('Maximum image size', '{:.0f}GB'.format(maxSize) if maxSize != -1 else 'No limit detected'), - ('Available disk space', _formatSize(diskSpace)), + ('Available disk space', diskSpace), ('Total system memory', '{} physical, {} virtual'.format(_formatSize(memPhysical), _formatSize(memVirtual))), ('Number of processors', '{} physical, {} logical'.format(cpuPhysical, cpuLogical)) ] diff --git a/ue4docker/infrastructure/ResourceMonitor.py b/ue4docker/infrastructure/ResourceMonitor.py index 5cdc65ce..bb225ff4 100644 --- a/ue4docker/infrastructure/ResourceMonitor.py +++ b/ue4docker/infrastructure/ResourceMonitor.py @@ -1,4 +1,4 @@ -import datetime, humanfriendly, psutil, shutil, threading, time +import datetime, humanfriendly, os, psutil, shutil, threading, time from .DockerUtils import DockerUtils class ResourceMonitor(threading.Thread): @@ -35,6 +35,9 @@ def run(self): dockerInfo = DockerUtils.info() rootDir = dockerInfo['DockerRootDir'] + # If we cannot access the Docker data directory (e.g. when the daemon is in a Moby VM), don't report disk space + reportDisk = os.path.exists(rootDir) + # Sample the CPU usage using an interval of 1 second the first time to prime the system # (See: ) psutil.cpu_percent(1.0) @@ -54,7 +57,7 @@ def run(self): formatSize = lambda size: humanfriendly.format_size(size, binary=True, keep_width=True) # Format the current quantity of available disk space on the Docker data directory's filesystem - diskSpace = formatSize(shutil.disk_usage(rootDir).free) + diskSpace = formatSize(shutil.disk_usage(rootDir).free) if reportDisk == True else 'Unknown' # Format the current quantity of available system memory physicalMemory = formatSize(psutil.virtual_memory().free) From ab66ed27f954d8a2a0e453fcf76a7324b6df2bd2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 2 Feb 2020 12:27:41 +1000 Subject: [PATCH 027/430] Add a diagnostic for Windows 1903/1909 storage-opt issues --- ue4docker/diagnostics/__init__.py | 1 + ue4docker/diagnostics/base.py | 12 +++++ ue4docker/diagnostics/diagnostic_8gig.py | 15 ++---- ue4docker/diagnostics/diagnostic_all.py | 3 +- ue4docker/diagnostics/diagnostic_maxsize.py | 57 +++++++++++++++++++++ ue4docker/diagnostics_cmd.py | 3 +- 6 files changed, 78 insertions(+), 13 deletions(-) create mode 100644 ue4docker/diagnostics/diagnostic_maxsize.py diff --git a/ue4docker/diagnostics/__init__.py b/ue4docker/diagnostics/__init__.py index 521a5fe6..1348857b 100644 --- a/ue4docker/diagnostics/__init__.py +++ b/ue4docker/diagnostics/__init__.py @@ -1,2 +1,3 @@ from .diagnostic_all import allDiagnostics from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_maxsize import diagnosticMaxSize diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py index 4b3ed074..4589717a 100644 --- a/ue4docker/diagnostics/base.py +++ b/ue4docker/diagnostics/base.py @@ -1,3 +1,5 @@ +import subprocess + class DiagnosticBase(object): def getName(self): @@ -17,3 +19,13 @@ def run(self, logger, args=[]): Runs the diagnostic ''' raise NotImplementedError + + + # Helper functionality for derived classes + + def _printAndRun(self, logger, prefix, command, check=False): + ''' + Prints a command and then executes it + ''' + logger.info(prefix + 'Run: {}'.format(command), False) + subprocess.run(command, check=check) diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index e136da37..a5df055a 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -1,7 +1,7 @@ from ..infrastructure import DockerUtils, WindowsUtils from .base import DiagnosticBase -import os, platform, subprocess +import os, platform from os.path import abspath, dirname, join class diagnostic8Gig(DiagnosticBase): @@ -61,7 +61,7 @@ def run(self, logger, args=[]): try: logger.action('[8gig] Attempting to build an image with an 8GiB filesystem layer...', False) command = ['docker', 'build', '-t', diagnostic8Gig.IMAGE_TAG, contextDir] + buildArgs - self._printAndRun(logger, command, check=True) + self._printAndRun(logger, '[8gig] ', command, check=True) built = True except: logger.error('[8gig] Build failed!') @@ -70,8 +70,8 @@ def run(self, logger, args=[]): # Remove any built images, including intermediate images logger.action('[8gig] Cleaning up...', False) if built == True: - self._printAndRun(logger, ['docker', 'rmi', diagnostic8Gig.IMAGE_TAG]) - self._printAndRun(logger, ['docker', 'system', 'prune', '-f']) + self._printAndRun(logger, '[8gig] ', ['docker', 'rmi', diagnostic8Gig.IMAGE_TAG]) + self._printAndRun(logger, '[8gig] ', ['docker', 'system', 'prune', '-f']) # Inform the user of the outcome of the diagnostic if built == True: @@ -80,10 +80,3 @@ def run(self, logger, args=[]): logger.error('[8gig] Diagnostic failed! The Docker daemon cannot build images with 8GiB filesystem layers.\n', True) return built - - def _printAndRun(self, logger, command, check=False): - ''' - Prints a command and then executes it - ''' - logger.info('[8gig] Run: {}'.format(command), False) - subprocess.run(command, check=check) diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index dcc53da9..4cf603a8 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -1,5 +1,6 @@ from .base import DiagnosticBase from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_maxsize import diagnosticMaxSize class allDiagnostics(DiagnosticBase): @@ -22,7 +23,7 @@ def run(self, logger, args=[]): # Run all available diagnostics in turn, storing the results results = [] - diagnostics = [diagnostic8Gig()] + diagnostics = [diagnostic8Gig(), diagnosticMaxSize()] for index, diagnostic in enumerate(diagnostics): # Run the diagnostic and report its result diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py new file mode 100644 index 00000000..9dcfc14b --- /dev/null +++ b/ue4docker/diagnostics/diagnostic_maxsize.py @@ -0,0 +1,57 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase +import platform + +class diagnosticMaxSize(DiagnosticBase): + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + return 'Check for Windows storage-opt bug' + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + return '\n'.join([ + 'This diagnostic determines if the Windows host OS suffers from the bug that', + 'prevents users from increasing the maximum allowed image size using Docker\'s', + '`storage-opt` configuration key, as reported here:', + 'https://github.com/docker/for-win/issues/4100', + '', + 'This bug is present in Windows Server and Windows 10 versions 1903 and 1909,', + 'but a workaround was introduced in Docker CE version 19.03.6.', + ]) + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + + # Verify that we are running under Windows and are in Windows container mode if using Docker Desktop + dockerPlatform = DockerUtils.info()['OSType'].lower() + hostPlatform = platform.system().lower() + if hostPlatform != 'windows': + logger.info('[maxsize] This diagnostic only applies to Windows host systems.', False) + return True + elif dockerPlatform != 'windows': + logger.error('[maxsize] Error: Docker Desktop is currently in Linux container mode.', False) + logger.error('[maxsize] Please switch to Windows container mode to run this diagnostic.', False) + return False + + # Verify that we are running Windows Server or Windows 10 version 1903 or newer + if WindowsUtils.getWindowsVersion()['patch'] < 18362: + logger.info('[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.', False) + return True + + # Attempt to run a Windows Nanoserver 1903 container with the `storage-mode` configuration options set + # (The bug doesn't seem to be triggered when using older Windows images, presumably because they use an older host kernel) + try: + command = ['docker', 'run', '--rm', '--storage-opt', 'size=200GB', 'mcr.microsoft.com/windows/nanoserver:1903', 'cmd', 'exit'] + self._printAndRun(logger, '[maxsize] ', command, check=True) + logger.action('[maxsize] Diagnostic succeeded! The host platform can specify maximum image sizes using Docker\'s `storage-opt` configuration key.\n') + return True + except: + logger.error('[maxsize] Diagnostic failed! The host platform cannot specify maximum image sizes using Docker\'s `storage-opt` configuration key.\n', True) + return False diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index 9d961b02..f1498e51 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -7,7 +7,8 @@ def diagnostics(): # The diagnostics that can be run DIAGNOSTICS = { 'all': allDiagnostics(), - '8gig': diagnostic8Gig() + '8gig': diagnostic8Gig(), + 'maxsize': diagnosticMaxSize() } # Create our logger to generate coloured output on stderr From 28f0e2dab480f91990faa3f2604492a81f260e5d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 2 Feb 2020 12:32:28 +1000 Subject: [PATCH 028/430] Blacklist Windows 1909 and add option to ignore blacklist for testing --- ue4docker/build.py | 2 +- ue4docker/infrastructure/BuildConfiguration.py | 2 ++ ue4docker/infrastructure/WindowsUtils.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 55e55c8c..8a01c3c9 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -94,7 +94,7 @@ def build(): logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) # Verify that the host OS is not a release that is blacklisted due to critical bugs - if WindowsUtils.isBlacklistedWindowsVersion() == True: + if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) logger.error('This version of Windows contains one or more critical bugs that', False) logger.error('render it incapable of successfully building UE4 container images.', False) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 0565ae5d..75009f39 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -83,6 +83,7 @@ def addArguments(parser): parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') + parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') def __init__(self, parser, argv): ''' @@ -139,6 +140,7 @@ def __init__(self, parser, argv): self.excludedComponents = set(self.args.exclude) self.baseImage = None self.prereqsTag = None + self.ignoreBlacklist = self.args.ignore_blacklist # Generate our flags for keeping or excluding components self.exclusionFlags = [ diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 3088580d..3eb33d2f 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -16,7 +16,7 @@ class WindowsUtils(object): _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019'] # The list of Windows Server and Windows 10 releases that are blacklisted due to critical bugs - _blacklistedReleases = ['1903'] + _blacklistedReleases = ['1903', '1909'] @staticmethod def _getVersionRegKey(subkey): From 033cbb2d90094236b4c4cecd53ab3fe82c1ac1a8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 2 Feb 2020 12:39:48 +1000 Subject: [PATCH 029/430] Bump version to 0.0.48 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 97e658fe..d4810c9c 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.47' +__version__ = '0.0.48' From 3901804f7daf7ecd7daab956599a57e706ed027b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 10 Feb 2020 13:21:21 +1000 Subject: [PATCH 030/430] Patch UBT to fix BinaryOutputDir errors during JSON output --- .../dockerfiles/ue4-source/linux/Dockerfile | 4 +++ .../dockerfiles/ue4-source/linux/patch-ubt.py | 25 +++++++++++++++++++ .../dockerfiles/ue4-source/windows/Dockerfile | 4 +++ .../ue4-source/windows/patch-ubt.py | 25 +++++++++++++++++++ 4 files changed, 58 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py create mode 100644 ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 932271b0..be61652a 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -67,3 +67,7 @@ RUN python3 /tmp/linker-fixup.py /home/ue4/UnrealEngine/Engine/Extras/ThirdParty # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX + +# Apply our bugfix patches to UnrealBuildTool (UBT) +COPY --chown=ue4:ue4 patch-ubt.py /tmp/patch-ubt.py +RUN python3 /tmp/patch-ubt.py /home/ue4/UnrealEngine/Engine/Source/Programs/UnrealBuildTool diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py b/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py new file mode 100644 index 00000000..2d6c8053 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import os, re, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +def patchFile(filename, search, replace): + contents = readFile(filename) + patched = contents.replace(search, replace) + writeFile(filename, patched) + +# Apply our bugfixes to UnrealBuildTool (UBT) + +# Ensure modules always have a binary output directory value set when exporting JSON +# (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) +patchFile( + os.path.join(sys.argv[1], 'Configuration', 'UEBuildTarget.cs'), + 'Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);', + 'Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);' +) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index cd97b3f1..f30619b7 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -41,3 +41,7 @@ RUN Setup.bat # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 + +# Apply our bugfix patches to UnrealBuildTool (UBT) +COPY patch-ubt.py C:\patch-ubt.py +RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildTool diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py b/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py new file mode 100644 index 00000000..2d6c8053 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import os, re, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +def patchFile(filename, search, replace): + contents = readFile(filename) + patched = contents.replace(search, replace) + writeFile(filename, patched) + +# Apply our bugfixes to UnrealBuildTool (UBT) + +# Ensure modules always have a binary output directory value set when exporting JSON +# (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) +patchFile( + os.path.join(sys.argv[1], 'Configuration', 'UEBuildTarget.cs'), + 'Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);', + 'Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);' +) From f1705938b39664775eda783e90143511e369f0eb Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 10 Feb 2020 17:10:40 +1000 Subject: [PATCH 031/430] Bump version to 0.0.49 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index d4810c9c..471b6f00 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.48' +__version__ = '0.0.49' From 8809d162148398ea103c4cefd408aa8cc254ca3f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 10 Feb 2020 21:52:54 +1000 Subject: [PATCH 032/430] Fix #66 --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 8f0d36c9..d3b7933d 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli USER root RUN pip3 install setuptools wheel -RUN pip3 install ue4cli conan-ue4cli +RUN pip3 install ue4cli 'conan-ue4cli>=0.0.17' USER ue4 # Extract the third-party library details from UBT @@ -23,7 +23,7 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install setuptools wheel -RUN pip3 install ue4cli conan-ue4cli ue4-ci-helpers +RUN pip3 install ue4cli 'conan-ue4cli>=0.0.17' ue4-ci-helpers USER ue4 # Copy the Conan configuration settings and package cache from the builder image diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index 44cee90b..6c369e7e 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli conan-ue4cli --no-warn-script-location +RUN pip install ue4cli "conan-ue4cli>=0.0.17" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -19,7 +19,7 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli conan-ue4cli ue4-ci-helpers --no-warn-script-location +RUN pip install ue4cli "conan-ue4cli>=0.0.17" ue4-ci-helpers --no-warn-script-location # Copy the Conan configuration settings and package cache from the builder image COPY --from=builder C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan From 8c15f1b6aec9029e202922837e6bcec72a371e71 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 10 Feb 2020 21:53:11 +1000 Subject: [PATCH 033/430] Bump version to 0.0.50 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 471b6f00..7291e4d6 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.49' +__version__ = '0.0.50' From 3f5b10e987418b07212b53ba3b1b4c991f28c6ad Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 12 Feb 2020 16:50:56 +1000 Subject: [PATCH 034/430] Add CEF dependencies to ue4-full under Linux --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index d3b7933d..164579ea 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -44,12 +44,19 @@ USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf -# Enable X11 support +# Enable X11 support (including the libraries required by CEF) USER root RUN apt-get install -y --no-install-recommends \ + libasound2 \ + libatk1.0-0 \ + libcairo2 \ libfontconfig1 \ libfreetype6 \ libglu1 \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ libsm6 \ libxcomposite1 \ libxcursor1 \ From 1e46c6c0be5a0bbecfcc3617b0b46d1cabe21314 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 12 Feb 2020 17:14:17 +1000 Subject: [PATCH 035/430] Bump version to 0.0.51 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 7291e4d6..efd792f8 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.50' +__version__ = '0.0.51' From af521fe3ff2174beb9b6edcb40224d576a58c70a Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 12 Feb 2020 23:08:31 +1000 Subject: [PATCH 036/430] Add xvfb to ue4-full under Linux --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 164579ea..9a70ee29 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -44,7 +44,7 @@ USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf -# Enable X11 support (including the libraries required by CEF) +# Enable X11 support (including the libraries required by CEF) and xvfb so we can create a dummy display if needed USER root RUN apt-get install -y --no-install-recommends \ libasound2 \ @@ -68,5 +68,6 @@ RUN apt-get install -y --no-install-recommends \ x11-xkb-utils \ xauth \ xfonts-base \ - xkb-data + xkb-data \ + xvfb USER ue4 From 01455ac67aa829fa76c953fcffc46a173581353b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 12 Feb 2020 23:09:10 +1000 Subject: [PATCH 037/430] Bump version to 0.0.52 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index efd792f8..0298312f 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.51' +__version__ = '0.0.52' From 7e7ee6b1deb67b3424f324b7b7a7ec230b387e9b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 20 Feb 2020 22:10:01 +1000 Subject: [PATCH 038/430] Fix #70 --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 6 +++--- .../{verbose-ddc.py => patch-build-graph.py} | 16 +++++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) rename ue4docker/dockerfiles/ue4-minimal/linux/{verbose-ddc.py => patch-build-graph.py} (53%) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 02730227..ee84ff59 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -11,9 +11,9 @@ RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.ver COPY enable-opengl.py /tmp/enable-opengl.py RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngine.ini -# Increase the output verbosity of the DDC generation step -COPY verbose-ddc.py /tmp/verbose-ddc.py -RUN python3 /tmp/verbose-ddc.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml +# Patch the default settings in InstalledEngineBuild.xml and increase the output verbosity of the DDC generation step +COPY patch-build-graph.py /tmp/patch-build-graph.py +RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py similarity index 53% rename from ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py rename to ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index 622087d5..ceee8218 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -9,11 +9,25 @@ def writeFile(filename, data): with open(filename, 'wb') as f: f.write(data.encode('utf-8')) -# Add verbose output flags to the `BuildDerivedDataCache` command +# Read the build graph XML buildXml = sys.argv[1] code = readFile(buildXml) + +# Add verbose output flags to the `BuildDerivedDataCache` command code = code.replace( 'Command Name="BuildDerivedDataCache" Arguments="', 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ' ) + +# Enable client and server targets by default in 4.23.0 onwards +code = code.replace( + 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' +) +code = code.replace( + 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' +) + +# Write the modified XML back to disk writeFile(buildXml, code) From d19b39948109cc81d70506f63291d449a82ce3a9 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 20 Feb 2020 22:11:15 +1000 Subject: [PATCH 039/430] Bump version to 0.0.53 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 0298312f..5f0b2415 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.52' +__version__ = '0.0.53' From a9428f0447fa78aae4428375bac6661b34c93297 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 22 Feb 2020 17:59:18 +1000 Subject: [PATCH 040/430] Disable Linux server builds for 4.24.0 to 4.24.2 --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 2 +- .../ue4-minimal/linux/patch-build-graph.py | 26 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index ee84ff59..b43b942d 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -13,7 +13,7 @@ RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngin # Patch the default settings in InstalledEngineBuild.xml and increase the output verbosity of the DDC generation step COPY patch-build-graph.py /tmp/patch-build-graph.py -RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml +RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index ceee8218..20338f0a 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, sys +import json, os, sys def readFile(filename): with open(filename, 'rb') as f: @@ -13,21 +13,27 @@ def writeFile(filename, data): buildXml = sys.argv[1] code = readFile(buildXml) +# Read the UE4 version information +versionFile = sys.argv[2] +versionData = json.loads(readFile(versionFile)) + # Add verbose output flags to the `BuildDerivedDataCache` command code = code.replace( 'Command Name="BuildDerivedDataCache" Arguments="', 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ' ) -# Enable client and server targets by default in 4.23.0 onwards -code = code.replace( - 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' -) -code = code.replace( - 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' -) +# Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail +# (See for details of the bug and its fix) +if versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3: + code = code.replace( + 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' + ) + code = code.replace( + 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' + ) # Write the modified XML back to disk writeFile(buildXml, code) From f2fb37f2f3ebcf131d5c2049699cd10f90e6e8fe Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 22 Feb 2020 18:01:10 +1000 Subject: [PATCH 041/430] Bump version to 0.0.54 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 5f0b2415..fa516113 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.53' +__version__ = '0.0.54' From 60cd190bd8d6db8bb49ca6c5d899e1d786db23dc Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 28 Feb 2020 17:09:04 +1000 Subject: [PATCH 042/430] Disable building for AArch64 by default under Linux --- .../dockerfiles/ue4-minimal/linux/patch-build-graph.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index 20338f0a..c6478b04 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -23,6 +23,12 @@ def writeFile(filename, data): 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ' ) +# Disable building for AArch64 by default (enabled by default since 4.24.0) +code = code.replace( + 'Property Name="DefaultWithLinuxAArch64" Value="true"', + 'Property Name="DefaultWithLinuxAArch64" Value="false"' +) + # Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail # (See for details of the bug and its fix) if versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3: From 6886693f86e8800863ca4fc97516cafad99b08c8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 28 Feb 2020 17:09:23 +1000 Subject: [PATCH 043/430] Bump version to 0.0.55 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index fa516113..f8e88b3e 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.54' +__version__ = '0.0.55' From f53bc1a2762d0ad6e5bfcd597e9e417593e1771b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 2 Mar 2020 11:57:48 +1000 Subject: [PATCH 044/430] Explicitly set ue4cli configuration directory in ue4-full image --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 8 ++++++-- ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 9a70ee29..ab803f38 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli USER root RUN pip3 install setuptools wheel -RUN pip3 install ue4cli 'conan-ue4cli>=0.0.17' +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.17' USER ue4 # Extract the third-party library details from UBT @@ -23,9 +23,13 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install setuptools wheel -RUN pip3 install ue4cli 'conan-ue4cli>=0.0.17' ue4-ci-helpers +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.17' ue4-ci-helpers USER ue4 +# Explicitly set the configuration directory for ue4cli +# (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) +ENV UE4CLI_CONFIG_DIR /home/ue4/.config/ue4cli + # Copy the Conan configuration settings and package cache from the builder image COPY --from=builder --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index 6c369e7e..ff86c220 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli "conan-ue4cli>=0.0.17" --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.17" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -19,7 +19,11 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli "conan-ue4cli>=0.0.17" ue4-ci-helpers --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.17" ue4-ci-helpers --no-warn-script-location + +# Explicitly set the configuration directory for ue4cli +# (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) +ENV UE4CLI_CONFIG_DIR C:\Users\ContainerAdministrator\AppData\Roaming\ue4cli # Copy the Conan configuration settings and package cache from the builder image COPY --from=builder C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan From c9769984bf369f03c145c4f10d0bc6b1c594930b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 2 Mar 2020 11:58:06 +1000 Subject: [PATCH 045/430] Bump version to 0.0.56 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index f8e88b3e..b2c95617 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.55' +__version__ = '0.0.56' From 627a06239e6dee63fb810bdbbdde6b98b54fc92f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 3 Mar 2020 20:56:49 +1000 Subject: [PATCH 046/430] Bump minimum conan-ue4cli version to 0.0.20 --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index ab803f38..575c50da 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli USER root RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.17' +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.20' USER ue4 # Extract the third-party library details from UBT @@ -23,7 +23,7 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.17' ue4-ci-helpers +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.20' ue4-ci-helpers USER ue4 # Explicitly set the configuration directory for ue4cli diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index ff86c220..e57070d2 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.17" --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.20" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -19,7 +19,7 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.17" ue4-ci-helpers --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.20" ue4-ci-helpers --no-warn-script-location # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) From 0104a8da57016b6e506d470d1780aa5a022defac Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 3 Mar 2020 20:57:12 +1000 Subject: [PATCH 047/430] Bump version to 0.0.57 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index b2c95617..f31e7249 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.56' +__version__ = '0.0.57' From 9a3d9b265a0eb930d4ee726ccb86a57e3f74abe6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 4 Mar 2020 08:52:48 +1000 Subject: [PATCH 048/430] Bump conan-ue4cli minimum version to 0.0.21 --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 575c50da..20f9662d 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli USER root RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.20' +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.21' USER ue4 # Extract the third-party library details from UBT @@ -23,7 +23,7 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.20' ue4-ci-helpers +RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.21' ue4-ci-helpers USER ue4 # Explicitly set the configuration directory for ue4cli diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index e57070d2..f5b74e9b 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -6,7 +6,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.20" --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.21" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -19,7 +19,7 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.20" ue4-ci-helpers --no-warn-script-location +RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.21" ue4-ci-helpers --no-warn-script-location # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) From 886f0998d737272ac1971e38cbbdac4117b9a3ec Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 4 Mar 2020 08:53:12 +1000 Subject: [PATCH 049/430] Bump version to 0.0.58 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index f31e7249..03de9806 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.57' +__version__ = '0.0.58' From 07ef0cb81d5c58cdcd3e1c4a32182b07fb585ed5 Mon Sep 17 00:00:00 2001 From: Paolo Galeone Date: Thu, 12 Mar 2020 09:24:49 +0100 Subject: [PATCH 050/430] Use dotnet SDK 4.8 In this way, we can create the images on Windows server updated with the February 2020 path. Fixes https://github.com/adamrehn/ue4-docker/issues/73 --- ue4docker/infrastructure/BuildConfiguration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 75009f39..a38fe376 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -177,7 +177,7 @@ def _generateWindowsConfig(self): # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag - self.baseImage = 'mcr.microsoft.com/dotnet/framework/sdk:4.7.2-windowsservercore-' + self.basetag + self.baseImage = 'mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-' + self.basetag self.prereqsTag = self.basetag # Verify that any user-specified base tag is valid From 57356189dd25e6b37652dc6e69804670a38b197a Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 15 Mar 2020 13:37:14 +1000 Subject: [PATCH 051/430] Blacklist EOL Windows container image tags --- ue4docker/build.py | 8 ++++++++ ue4docker/infrastructure/WindowsUtils.py | 26 ++++++++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 8a01c3c9..eba41097 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -93,6 +93,14 @@ def build(): logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) + # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) + if WindowsUtils.isEndOfLifeWindowsVersion(config.basetag) == True: + logger.error('Error: detected EOL base OS image tag: {}'.format(config.basetag), False) + logger.error('This version of Windows has reached End Of Life (EOL), which means', False) + logger.error('Microsoft no longer supports or maintains container base images for it.', False) + logger.error('You will need to use a base image tag for a supported version of Windows.', False) + sys.exit(1) + # Verify that the host OS is not a release that is blacklisted due to critical bugs if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 3eb33d2f..de8c60f5 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -12,12 +12,15 @@ class WindowsUtils(object): # The latest Windows build version we recognise as a non-Insider build _latestReleaseBuild = 18362 - # The list of Windows Server Core base image tags that we support, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019'] + # The list of Windows Server Core base image tags that we recognise, in ascending version number order + _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909'] - # The list of Windows Server and Windows 10 releases that are blacklisted due to critical bugs + # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs _blacklistedReleases = ['1903', '1909'] + # The list of Windows Server Core container image releases that are unsupported due to having reached EOL + _eolReleases = ['1709'] + @staticmethod def _getVersionRegKey(subkey): ''' @@ -101,11 +104,22 @@ def getWindowsBuild(): return '{}.{}'.format(version, build) @staticmethod - def isBlacklistedWindowsVersion(): + def isBlacklistedWindowsVersion(release=None): + ''' + Determines if the specified Windows release is one with bugs that make it unsuitable for use + (defaults to checking the host OS release if one is not specified) + ''' + release = WindowsUtils.getWindowsRelease() if release is None else release + return release in WindowsUtils._blacklistedReleases + + @staticmethod + def isEndOfLifeWindowsVersion(release=None): ''' - Determines if the Windows host system is a release with bugs that make it unsuitable for use + Determines if the specified Windows release is one that has reached End Of Life (EOL) + (defaults to checking the host OS release if one is not specified) ''' - return WindowsUtils.getWindowsRelease() in WindowsUtils._blacklistedReleases + release = WindowsUtils.getWindowsRelease() if release is None else release + return release in WindowsUtils._eolReleases @staticmethod def isSupportedWindowsVersion(): From 574c031a61a84f67d7e514886077bcec6532b418 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 15 Mar 2020 13:41:18 +1000 Subject: [PATCH 052/430] Bump version to 0.0.59 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 03de9806..2f038b5c 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.58' +__version__ = '0.0.59' From 31d81a3ba33e4a32741fa9c523f3a6a540a50238 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 21 Mar 2020 13:52:22 +1000 Subject: [PATCH 053/430] Delete temporary DDC files when creating Installed Builds --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 1 + ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 1 + 2 files changed, 2 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index b43b942d..049b2aea 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -27,6 +27,7 @@ ARG EXCLUDE_TEMPLATES WORKDIR /home/ue4/UnrealEngine COPY exclude-components.py /tmp/exclude-components.py RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && \ + rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC && \ python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 0620a34b..b09d8c42 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -23,6 +23,7 @@ ARG EXCLUDE_TEMPLATES WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && ` + del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL && ` python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini From f8f7040c586f3841dbadca0e3faf9483bddce88b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 22 Mar 2020 14:04:12 +1000 Subject: [PATCH 054/430] Bump version to 0.0.60 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 2f038b5c..0dba5635 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.59' +__version__ = '0.0.60' From f72e792dab09f4579be8ed1e245b4dea2e2596a6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 18 Apr 2020 12:15:12 +1000 Subject: [PATCH 055/430] Add link to details of blacklisted Windows versions --- ue4docker/build.py | 4 ++++ ue4docker/infrastructure/WindowsUtils.py | 1 + 2 files changed, 5 insertions(+) diff --git a/ue4docker/build.py b/ue4docker/build.py index eba41097..6c566260 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -104,9 +104,13 @@ def build(): # Verify that the host OS is not a release that is blacklisted due to critical bugs if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) + logger.error('', False) logger.error('This version of Windows contains one or more critical bugs that', False) logger.error('render it incapable of successfully building UE4 container images.', False) logger.error('You will need to use an older or newer version of Windows.', False) + logger.error('', False) + logger.error('For more information, see:', False) + logger.error('https://unrealcontainers.com/docs/concepts/windows-containers', False) sys.exit(1) # Verify that the user is not attempting to build images with a newer kernel version than the host OS diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index de8c60f5..5e796df3 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -16,6 +16,7 @@ class WindowsUtils(object): _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909'] # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs + # (See: ) _blacklistedReleases = ['1903', '1909'] # The list of Windows Server Core container image releases that are unsupported due to having reached EOL From 63c860f93713b31518d56b6c14b3c286777f9c4f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 18 Apr 2020 15:47:37 +1000 Subject: [PATCH 056/430] Add Vulkan support to the Linux version of ue4-full --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 20f9662d..51b569ce 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -48,8 +48,20 @@ USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf +# Enable Vulkan support +RUN apt-get install -y --no-install-recommends libvulkan1 && \ + VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \ + mkdir -p /etc/vulkan/icd.d/ && \ + echo \ + "{\ + \"file_format_version\" : \"1.0.0\",\ + \"ICD\": {\ + \"library_path\": \"libGLX_nvidia.so.0\",\ + \"api_version\" : \"${VULKAN_API_VERSION}\"\ + }\ + }" > /etc/vulkan/icd.d/nvidia_icd.json + # Enable X11 support (including the libraries required by CEF) and xvfb so we can create a dummy display if needed -USER root RUN apt-get install -y --no-install-recommends \ libasound2 \ libatk1.0-0 \ From cc1b5e97897d270aa2d9d6e6f475816302274429 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 18 Apr 2020 15:47:58 +1000 Subject: [PATCH 057/430] Bump version to 0.0.61 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 0dba5635..7f07b041 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.60' +__version__ = '0.0.61' From c64d3092442b119bb7d799a4254b72b54d14679c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 20 Apr 2020 14:11:09 +1000 Subject: [PATCH 058/430] Add the ability to exclude DDC generation --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 3 ++- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 3 ++- ue4docker/infrastructure/BuildConfiguration.py | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 049b2aea..8ab49ef0 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -22,11 +22,12 @@ RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -Clean # Create an Installed Build of the Engine # (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) +ARG BUILD_DDC ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES WORKDIR /home/ue4/UnrealEngine COPY exclude-components.py /tmp/exclude-components.py -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && \ +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC && \ python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index b09d8c42..c8462f5e 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -18,11 +18,12 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB # Create an Installed Build of the Engine # (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) +ARG BUILD_DDC ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% && ` del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL && ` python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index a38fe376..39453638 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -33,6 +33,9 @@ class ExcludedComponent(object): The different components that we support excluding from the built images ''' + # Engine Derived Data Cache (DDC) + DDC = 'ddc' + # Engine debug symbols Debug = 'debug' @@ -47,6 +50,7 @@ def description(component): ''' return { + ExcludedComponent.DDC: 'Derived Data Cache (DDC)', ExcludedComponent.Debug: 'Debug symbols', ExcludedComponent.Templates: 'Template projects and samples' @@ -70,7 +74,7 @@ def addArguments(parser): parser.add_argument('--no-full', action='store_true', help='Don\'t build the ue4-full image') parser.add_argument('--no-cache', action='store_true', help='Disable Docker build cache') parser.add_argument('--random-memory', action='store_true', help='Use a random memory limit for Windows containers') - parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') + parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.DDC, ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') parser.add_argument('-password', default=None, help='Specify the password to use when cloning the git repository') @@ -144,6 +148,8 @@ def __init__(self, parser, argv): # Generate our flags for keeping or excluding components self.exclusionFlags = [ + + '--build-arg', 'BUILD_DDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true'), '--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0), '--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0) ] From ab61e3a46efcce31051ea8f24fd21a611a97c72d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 21 Apr 2020 18:35:11 +1000 Subject: [PATCH 059/430] Fix #77 --- ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py index be1ec31e..7420baae 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py @@ -27,7 +27,7 @@ def readFile(filename): for symbolFile in symbolFiles: log('Removing debug symbol file {}...'.format(symbolFile)) try: - os.unlink(symbolFile, 0) + os.unlink(symbolFile) except: log(' Warning: failed to remove debug symbol file {}.'.format(symbolFile)) From cdd5c49f2fbef1fea9c89e6e65b31f79608c1bd4 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 21 Apr 2020 18:36:54 +1000 Subject: [PATCH 060/430] Bump version to 0.0.62 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 7f07b041..24837cfc 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.61' +__version__ = '0.0.62' From 049dd3122ca6b07e88b70ad6509c41053fa5d9d7 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 15:39:25 +1000 Subject: [PATCH 061/430] Add `ue4 test` command to test built images --- setup.py | 4 +- ue4docker/__init__.py | 1 + ue4docker/main.py | 5 +++ ue4docker/test.py | 59 ++++++++++++++++++++++++++++ ue4docker/tests/build-and-package.py | 19 +++++++++ 5 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 ue4docker/test.py create mode 100644 ue4docker/tests/build-and-package.py diff --git a/setup.py b/setup.py index bb69bfbe..2ed64bcf 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,7 @@ python_requires = '>=3.6', install_requires = [ 'colorama', + 'container-utils', 'docker>=3.0.0', 'flask', 'humanfriendly', @@ -52,7 +53,8 @@ 'ue4docker': [ 'dockerfiles/*/*/.dockerignore', 'dockerfiles/diagnostics/*/*/*', - 'dockerfiles/*/*/*' + 'dockerfiles/*/*/*', + 'tests/*.py' ] }, entry_points = { diff --git a/ue4docker/__init__.py b/ue4docker/__init__.py index cbf7415d..6824568e 100644 --- a/ue4docker/__init__.py +++ b/ue4docker/__init__.py @@ -4,5 +4,6 @@ from .info import info from .main import main from .setup_cmd import setup +from .test import test from .version_cmd import version from .version import __version__ diff --git a/ue4docker/main.py b/ue4docker/main.py index 62072ce8..0f12fa4c 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -5,6 +5,7 @@ from .export import export from .info import info from .setup_cmd import setup +from .test import test from .version_cmd import version import logging, os, platform, sys @@ -58,6 +59,10 @@ def main(): 'function': setup, 'description': 'Automatically configures the host system where possible' }, + 'test': { + 'function': test, + 'description': 'Runs tests to verify the correctness of built container images' + }, 'version': { 'function': version, 'description': 'Prints the ue4-docker version number' diff --git a/ue4docker/test.py b/ue4docker/test.py new file mode 100644 index 00000000..d15be740 --- /dev/null +++ b/ue4docker/test.py @@ -0,0 +1,59 @@ +from .infrastructure import DockerUtils, GlobalConfiguration, Logger +from container_utils import ContainerUtils +import docker, os, sys + +def test(): + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix='[{} test] '.format(sys.argv[0])) + + # Create our Docker API client + client = docker.from_env() + + # Check that an image tag has been specified + if len(sys.argv) > 1 and sys.argv[1].strip('-') not in ['h', 'help']: + + # Verify that the specified container image exists + tag = sys.argv[1] + image = GlobalConfiguration.resolveTag('ue4-full:{}'.format(tag) if ':' not in tag else tag) + if DockerUtils.exists(image) == False: + logger.error('Error: the specified container image "{}" does not exist.'.format(image)) + sys.exit(1) + + # Start a container to run our tests in, automatically stopping and removing the container when we finish + logger.action('Starting a container using the "{}" image...'.format(image), False) + container = ContainerUtils.start_for_exec(client, image) + with ContainerUtils.automatically_stop(container): + + # Create the workspace directory in the container + workspaceDir = ContainerUtils.workspace_dir(container) + ContainerUtils.exec(container, ['mkdir', workspaceDir]) + + # Copy our test scripts into the container + testDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests') + ContainerUtils.copy_from_host(container, testDir, workspaceDir) + + # Create a harness to invoke individual tests + containerPath = ContainerUtils.path(container) + pythonCommand = 'python' if ContainerUtils.container_platform(container) == 'windows' else 'python3' + def runTest(script): + logger.action('Running test "{}"...'.format(script), False) + try: + ContainerUtils.exec(container, [pythonCommand, containerPath.join(workspaceDir, script)], workdir=workspaceDir) + logger.action('Passed test "{}"'.format(script), False) + except RuntimeError as e: + logger.error('Error: test "{}" failed!'.format(script)) + raise e from None + + # Run each of our tests in turn + runTest('build-and-package.py') + + # If we've reached this point then all of the tests passed + logger.action('All tests passed.', False) + + else: + + # Print usage syntax + print('Usage: {} test TAG'.format(sys.argv[0])) + print('Runs tests to verify the correctness of built container images\n') + print('TAG should specify the tag of the ue4-full image to test.') diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py new file mode 100644 index 00000000..0ecaec9a --- /dev/null +++ b/ue4docker/tests/build-and-package.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import os, shutil, subprocess, ue4cli + +# Runs a command, raising an error if it returns a nonzero exit code +def run(command, **kwargs): + return subprocess.run(command, check=True, **kwargs) + +# Retrieve the short version string for the Engine +manager = ue4cli.UnrealManagerFactory.create() +version = manager.getEngineVersion('short') + +# Clone a simple C++ project and verify that we can build and package it +repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) +projectDir = os.path.join(os.getcwd(), 'BasicCxx') +run(['git', 'clone', '--depth=1', repo, projectDir]) +run(['ue4', 'package', 'Shipping'], cwd=projectDir) + +# Perform cleanup +shutil.rmtree(projectDir) From 62e92d9aad859d34be16d3f1a5fd0efc4f982dac Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 15:51:34 +1000 Subject: [PATCH 062/430] Use shell to run mkdir command inside container in `ue4 test` --- ue4docker/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/test.py b/ue4docker/test.py index d15be740..3be65c66 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -27,7 +27,7 @@ def test(): # Create the workspace directory in the container workspaceDir = ContainerUtils.workspace_dir(container) - ContainerUtils.exec(container, ['mkdir', workspaceDir]) + ContainerUtils.exec(container, ContainerUtils.shell_prefix(container) + ['mkdir ' + workspaceDir]) # Copy our test scripts into the container testDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests') From 722f7f214e4dd4769d371dbb32438fac68709905 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 15:58:46 +1000 Subject: [PATCH 063/430] Retry cleanups in tests under Windows --- ue4docker/tests/build-and-package.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index 0ecaec9a..72e5ca74 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -1,10 +1,21 @@ #!/usr/bin/env python3 -import os, shutil, subprocess, ue4cli +import os, shutil, subprocess, time, ue4cli + # Runs a command, raising an error if it returns a nonzero exit code def run(command, **kwargs): return subprocess.run(command, check=True, **kwargs) +# Repeatedly calls a function until it succeeds or the max number of retries has been reached +def repeat(func, maxRetries=5, sleepTime=0.5): + for i in range(0, maxRetries): + try: + func() + break + except: + time.sleep(sleepTime) + + # Retrieve the short version string for the Engine manager = ue4cli.UnrealManagerFactory.create() version = manager.getEngineVersion('short') @@ -16,4 +27,5 @@ def run(command, **kwargs): run(['ue4', 'package', 'Shipping'], cwd=projectDir) # Perform cleanup -shutil.rmtree(projectDir) +# (This can sometimes fail arbitrarily under Windows, so retry a few times if it does) +repeat(lambda: shutil.rmtree(projectDir)) From 4ac21a8ff09e4aa8abe7963ee1bd49d7cc81ef49 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 16:15:01 +1000 Subject: [PATCH 064/430] Bump version to 0.0.63 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 24837cfc..731ce3e3 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.62' +__version__ = '0.0.63' From 3ec7ba75f9ab69e900e40f9c510107e6024e9c48 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 17:42:14 +1000 Subject: [PATCH 065/430] Add test for consuming dependencies with conan-ue4cli --- ue4docker/test.py | 1 + ue4docker/tests/build-and-package.py | 1 + ue4docker/tests/consume-external-deps.py | 66 ++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 ue4docker/tests/consume-external-deps.py diff --git a/ue4docker/test.py b/ue4docker/test.py index 3be65c66..670ac089 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -47,6 +47,7 @@ def runTest(script): # Run each of our tests in turn runTest('build-and-package.py') + runTest('consume-external-deps.py') # If we've reached this point then all of the tests passed logger.action('All tests passed.', False) diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index 72e5ca74..4fec6aff 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -4,6 +4,7 @@ # Runs a command, raising an error if it returns a nonzero exit code def run(command, **kwargs): + print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) return subprocess.run(command, check=True, **kwargs) # Repeatedly calls a function until it succeeds or the max number of retries has been reached diff --git a/ue4docker/tests/consume-external-deps.py b/ue4docker/tests/consume-external-deps.py new file mode 100644 index 00000000..57d3e804 --- /dev/null +++ b/ue4docker/tests/consume-external-deps.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import os, shutil, subprocess, time, ue4cli + + +# Reads data from a file +def read(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +# Repeatedly calls a function until it succeeds or the max number of retries has been reached +def repeat(func, maxRetries=5, sleepTime=0.5): + for i in range(0, maxRetries): + try: + func() + break + except: + time.sleep(sleepTime) + +# Runs a command, raising an error if it returns a nonzero exit code +def run(command, **kwargs): + print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) + return subprocess.run(command, check=True, **kwargs) + +# Writes data to a file +def write(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + + +# Retrieve the short version string for the Engine +manager = ue4cli.UnrealManagerFactory.create() +version = manager.getEngineVersion('short') + +# Clone a simple C++ project +repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) +projectDir = os.path.join(os.getcwd(), 'BasicCxx') +run(['git', 'clone', '--depth=1', repo, projectDir]) + +# Generate a code module to wrap our external dependencies +sourceDir = os.path.join(projectDir, 'Source') +run(['ue4', 'conan', 'boilerplate', 'WrapperModule'], cwd=sourceDir) + +# Add the wrapper module as a dependency of the project's main source code module +rulesFile = os.path.join(sourceDir, 'BasicCxx', 'BasicCxx.Build.cs') +rules = read(rulesFile) +rules = rules.replace('PublicDependencyModuleNames.AddRange(new string[] {', 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ') +write(rulesFile, rules) + +# Add some dependencies to the module's conanfile.py +moduleDir = os.path.join(sourceDir, 'WrapperModule') +conanfile = os.path.join(moduleDir, 'conanfile.py') +deps = read(conanfile) +deps = deps.replace('pass', 'self._requireUnreal("zlib/ue4@adamrehn/{}")') +write(conanfile, deps) + +# Verify that we can build the project with dynamically located dependencies +run(['ue4', 'build'], cwd=projectDir) +run(['ue4', 'clean'], cwd=projectDir) + +# Verify that we can build the project with precomputed dependency data +run(['ue4', 'conan', 'precompute', 'host'], cwd=moduleDir) +run(['ue4', 'build'], cwd=projectDir) + +# Perform cleanup +# (This can sometimes fail arbitrarily under Windows, so retry a few times if it does) +repeat(lambda: shutil.rmtree(projectDir)) From 351d82126117b3cf6a22c3369e25abb5a89f279c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 17:56:28 +1000 Subject: [PATCH 066/430] Run individual tests in temp directories to avoid Windows permission issues --- ue4docker/tests/build-and-package.py | 28 +++------ ue4docker/tests/consume-external-deps.py | 76 ++++++++++-------------- 2 files changed, 42 insertions(+), 62 deletions(-) diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index 4fec6aff..d0a00ba2 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, shutil, subprocess, time, ue4cli +import os, subprocess, tempfile, ue4cli # Runs a command, raising an error if it returns a nonzero exit code @@ -7,26 +7,16 @@ def run(command, **kwargs): print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) return subprocess.run(command, check=True, **kwargs) -# Repeatedly calls a function until it succeeds or the max number of retries has been reached -def repeat(func, maxRetries=5, sleepTime=0.5): - for i in range(0, maxRetries): - try: - func() - break - except: - time.sleep(sleepTime) - # Retrieve the short version string for the Engine manager = ue4cli.UnrealManagerFactory.create() version = manager.getEngineVersion('short') -# Clone a simple C++ project and verify that we can build and package it -repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) -projectDir = os.path.join(os.getcwd(), 'BasicCxx') -run(['git', 'clone', '--depth=1', repo, projectDir]) -run(['ue4', 'package', 'Shipping'], cwd=projectDir) - -# Perform cleanup -# (This can sometimes fail arbitrarily under Windows, so retry a few times if it does) -repeat(lambda: shutil.rmtree(projectDir)) +# Create an auto-deleting temporary directory to work in +with tempfile.TemporaryDirectory() as tempDir: + + # Clone a simple C++ project and verify that we can build and package it + repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) + projectDir = os.path.join(tempDir, 'BasicCxx') + run(['git', 'clone', '--depth=1', repo, projectDir]) + run(['ue4', 'package', 'Shipping'], cwd=projectDir) diff --git a/ue4docker/tests/consume-external-deps.py b/ue4docker/tests/consume-external-deps.py index 57d3e804..df25b766 100644 --- a/ue4docker/tests/consume-external-deps.py +++ b/ue4docker/tests/consume-external-deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, shutil, subprocess, time, ue4cli +import os, subprocess, tempfile, ue4cli # Reads data from a file @@ -7,15 +7,6 @@ def read(filename): with open(filename, 'rb') as f: return f.read().decode('utf-8') -# Repeatedly calls a function until it succeeds or the max number of retries has been reached -def repeat(func, maxRetries=5, sleepTime=0.5): - for i in range(0, maxRetries): - try: - func() - break - except: - time.sleep(sleepTime) - # Runs a command, raising an error if it returns a nonzero exit code def run(command, **kwargs): print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) @@ -31,36 +22,35 @@ def write(filename, data): manager = ue4cli.UnrealManagerFactory.create() version = manager.getEngineVersion('short') -# Clone a simple C++ project -repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) -projectDir = os.path.join(os.getcwd(), 'BasicCxx') -run(['git', 'clone', '--depth=1', repo, projectDir]) - -# Generate a code module to wrap our external dependencies -sourceDir = os.path.join(projectDir, 'Source') -run(['ue4', 'conan', 'boilerplate', 'WrapperModule'], cwd=sourceDir) - -# Add the wrapper module as a dependency of the project's main source code module -rulesFile = os.path.join(sourceDir, 'BasicCxx', 'BasicCxx.Build.cs') -rules = read(rulesFile) -rules = rules.replace('PublicDependencyModuleNames.AddRange(new string[] {', 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ') -write(rulesFile, rules) - -# Add some dependencies to the module's conanfile.py -moduleDir = os.path.join(sourceDir, 'WrapperModule') -conanfile = os.path.join(moduleDir, 'conanfile.py') -deps = read(conanfile) -deps = deps.replace('pass', 'self._requireUnreal("zlib/ue4@adamrehn/{}")') -write(conanfile, deps) - -# Verify that we can build the project with dynamically located dependencies -run(['ue4', 'build'], cwd=projectDir) -run(['ue4', 'clean'], cwd=projectDir) - -# Verify that we can build the project with precomputed dependency data -run(['ue4', 'conan', 'precompute', 'host'], cwd=moduleDir) -run(['ue4', 'build'], cwd=projectDir) - -# Perform cleanup -# (This can sometimes fail arbitrarily under Windows, so retry a few times if it does) -repeat(lambda: shutil.rmtree(projectDir)) +# Create an auto-deleting temporary directory to work in +with tempfile.TemporaryDirectory() as tempDir: + + # Clone a simple C++ project + repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) + projectDir = os.path.join(tempDir, 'BasicCxx') + run(['git', 'clone', '--depth=1', repo, projectDir]) + + # Generate a code module to wrap our external dependencies + sourceDir = os.path.join(projectDir, 'Source') + run(['ue4', 'conan', 'boilerplate', 'WrapperModule'], cwd=sourceDir) + + # Add the wrapper module as a dependency of the project's main source code module + rulesFile = os.path.join(sourceDir, 'BasicCxx', 'BasicCxx.Build.cs') + rules = read(rulesFile) + rules = rules.replace('PublicDependencyModuleNames.AddRange(new string[] {', 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ') + write(rulesFile, rules) + + # Add some dependencies to the module's conanfile.py + moduleDir = os.path.join(sourceDir, 'WrapperModule') + conanfile = os.path.join(moduleDir, 'conanfile.py') + deps = read(conanfile) + deps = deps.replace('pass', 'self._requireUnreal("zlib/ue4@adamrehn/{}")') + write(conanfile, deps) + + # Verify that we can build the project with dynamically located dependencies + run(['ue4', 'build'], cwd=projectDir) + run(['ue4', 'clean'], cwd=projectDir) + + # Verify that we can build the project with precomputed dependency data + run(['ue4', 'conan', 'precompute', 'host'], cwd=moduleDir) + run(['ue4', 'build'], cwd=projectDir) From 6bfcf3ac63b84f34c9f04d8590c7efdd1f47981e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 18:17:13 +1000 Subject: [PATCH 067/430] Forcibly delete .git dirs in tests under Windows --- ue4docker/tests/build-and-package.py | 6 +++++- ue4docker/tests/consume-external-deps.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index d0a00ba2..257237e8 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, subprocess, tempfile, ue4cli +import os, platform, subprocess, tempfile, ue4cli # Runs a command, raising an error if it returns a nonzero exit code @@ -20,3 +20,7 @@ def run(command, **kwargs): projectDir = os.path.join(tempDir, 'BasicCxx') run(['git', 'clone', '--depth=1', repo, projectDir]) run(['ue4', 'package', 'Shipping'], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == 'Windows': + run(['del', '/f', '/s', '/q', os.path.join(projectDir, '.git')], shell=True) diff --git a/ue4docker/tests/consume-external-deps.py b/ue4docker/tests/consume-external-deps.py index df25b766..6ff50b6b 100644 --- a/ue4docker/tests/consume-external-deps.py +++ b/ue4docker/tests/consume-external-deps.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, subprocess, tempfile, ue4cli +import os, platform, subprocess, tempfile, ue4cli # Reads data from a file @@ -54,3 +54,7 @@ def write(filename, data): # Verify that we can build the project with precomputed dependency data run(['ue4', 'conan', 'precompute', 'host'], cwd=moduleDir) run(['ue4', 'build'], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == 'Windows': + run(['del', '/f', '/s', '/q', os.path.join(projectDir, '.git')], shell=True) From 0f8b814aee7455b2f6f4a423b4003b67b7cbfbff Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 May 2020 18:23:38 +1000 Subject: [PATCH 068/430] Bump version to 0.0.64 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 731ce3e3..f30a819f 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.63' +__version__ = '0.0.64' From fa83a6eb924e35592f2d6f58a04acf1760244c90 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 May 2020 18:34:31 +1000 Subject: [PATCH 069/430] Fix #81 --- ue4docker/infrastructure/WindowsUtils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 5e796df3..b1577cc9 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -1,4 +1,6 @@ +from .DockerUtils import DockerUtils from .PackageUtils import PackageUtils +from pkg_resources import parse_version import os, platform if platform.system() == 'Windows': @@ -110,8 +112,9 @@ def isBlacklistedWindowsVersion(release=None): Determines if the specified Windows release is one with bugs that make it unsuitable for use (defaults to checking the host OS release if one is not specified) ''' + dockerVersion = parse_version(DockerUtils.version()['Version']) release = WindowsUtils.getWindowsRelease() if release is None else release - return release in WindowsUtils._blacklistedReleases + return release in WindowsUtils._blacklistedReleases and dockerVersion < parse_version('19.03.6') @staticmethod def isEndOfLifeWindowsVersion(release=None): From fea948d8ea05fb5a3fb385f03c7ae1c85ed881dd Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 May 2020 18:34:53 +1000 Subject: [PATCH 070/430] Bump version to 0.0.65 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index f30a819f..8f7bdcae 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.64' +__version__ = '0.0.65' From d8684d5b16cce6e35318b4fb7366ece3e83df9fe Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 May 2020 18:57:56 +1000 Subject: [PATCH 071/430] Add support for Windows 1903 and 1909 base image tags --- ue4docker/infrastructure/WindowsUtils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index b1577cc9..cdaa56f0 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -161,7 +161,9 @@ def getReleaseBaseTag(release): return { '1709': '1709', '1803': '1803', - '1809': 'ltsc2019' + '1809': 'ltsc2019', + '1903': '1903', + '1909': '1909' }.get(release, 'ltsc2016') @staticmethod From 526baac4f9f4c57b12221b966b6cc53fcc116e38 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 May 2020 19:08:32 +1000 Subject: [PATCH 072/430] Download Vulkan runtime under Windows instead of the entire SDK --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 02ee5c03..755c6aca 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -34,8 +34,8 @@ RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\GatheredDlls\ # Gather the Vulkan runtime library -RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/1.1.73.0/windows/VulkanSDK-1.1.73.0-Installer.exe?Human=true" --output %TEMP%\VulkanSDK.exe -RUN 7z e %TEMP%\VulkanSDK.exe -oC:\GatheredDlls -y RunTimeInstaller\x64\vulkan-1.dll +RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip +RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" # Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) RUN choco install -y windbg From 271ab2457085bdb05601f87e0e1d8b6cf5193594 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 11 May 2020 19:13:23 +1000 Subject: [PATCH 073/430] Bump version to 0.0.66 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8f7bdcae..39fa7e3f 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.65' +__version__ = '0.0.66' From e12f596c8e60e843b0d310e924ae9355225cf5bc Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 13 May 2020 18:43:41 +1000 Subject: [PATCH 074/430] Add options to customise ue4cli and conan-ue4cli package versions --- ue4docker/build.py | 17 +++++++++++++++- .../dockerfiles/ue4-full/linux/Dockerfile | 10 ++++++++-- .../dockerfiles/ue4-full/windows/Dockerfile | 10 ++++++++-- .../infrastructure/BuildConfiguration.py | 20 +++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 6c566260..81823a18 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -82,6 +82,12 @@ def build(): logger.info('Repository: ' + config.repository, False) logger.info('Branch/tag: ' + config.branch + '\n', False) + # Determine if we are using a custom version for ue4cli or conan-ue4cli + if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: + logger.info('CUSTOM PACKAGE VERSIONS:', False) + logger.info('ue4cli: {}'.format(config.ue4cliVersion if config.ue4cliVersion is not None else 'default'), False) + logger.info('conan-ue4cli: {}\n'.format(config.conanUe4cliVersion if config.conanUe4cliVersion is not None else 'default'), False) + # Determine if we are building Windows or Linux containers if config.containerPlatform == 'windows': @@ -235,7 +241,16 @@ def build(): # Build the full UE4 CI image, unless requested otherwise by the user buildUe4Full = buildUe4Minimal == True and config.noFull == False if buildUe4Full == True: - builder.build('ue4-full', mainTags, config.platformArgs + ue4BuildArgs) + + # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them + infrastructureFlags = [] + if config.ue4cliVersion is not None: + infrastructureFlags.extend(['--build-arg', 'UE4CLI_VERSION={}'.format(config.ue4cliVersion)]) + if config.conanUe4cliVersion is not None: + infrastructureFlags.extend(['--build-arg', 'CONAN_UE4CLI_VERSION={}'.format(config.conanUe4cliVersion)]) + + # Build the image + builder.build('ue4-full', mainTags, config.platformArgs + ue4BuildArgs + infrastructureFlags) else: logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 51b569ce..e9540c16 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -1,12 +1,16 @@ ARG NAMESPACE ARG TAG ARG PREREQS_TAG +ARG UE4CLI_VERSION="ue4cli>=0.0.41" +ARG CONAN_UE4CLI_VERSION="ue4cli>=0.0.41" FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION # Install ue4cli and conan-ue4cli USER root RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.21' +RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" USER ue4 # Extract the third-party library details from UBT @@ -15,6 +19,8 @@ RUN ue4 conan generate # Copy the generated Conan packages into a new image with our Installed Build FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION # Clone the UE4Capture repository RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture @@ -23,7 +29,7 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install setuptools wheel -RUN pip3 install 'ue4cli>=0.0.41' 'conan-ue4cli>=0.0.21' ue4-ci-helpers +RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers USER ue4 # Explicitly set the configuration directory for ue4cli diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index f5b74e9b..7bda7ed6 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -2,11 +2,15 @@ ARG NAMESPACE ARG TAG ARG PREREQS_TAG +ARG UE4CLI_VERSION="ue4cli>=0.0.41" +ARG CONAN_UE4CLI_VERSION="ue4cli>=0.0.41" FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.21" --no-warn-script-location +RUN pip install "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -16,10 +20,12 @@ RUN ue4 conan generate && ue4 conan generate --remove-only # Copy the generated Conan packages into a new image with our Installed Build FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "ue4cli>=0.0.41" "conan-ue4cli>=0.0.21" ue4-ci-helpers --no-warn-script-location +RUN pip install "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue4-ci-helpers --no-warn-script-location # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 39453638..6daea6c6 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -85,6 +85,8 @@ def addArguments(parser): parser.add_argument('-dlldir', default=None, help='Set the directory to copy required Windows DLLs from (default is the host System32 directory)') parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images') parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') + parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image') + parser.add_argument('-conan-ue4cli', default=None, help='Override the default version of conan-ue4cli installed in the ue4-full image') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') @@ -146,6 +148,10 @@ def __init__(self, parser, argv): self.prereqsTag = None self.ignoreBlacklist = self.args.ignore_blacklist + # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them + self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) + self.conanUe4cliVersion = self._processPackageVersion('conan-ue4cli', self.args.conan_ue4cli) + # Generate our flags for keeping or excluding components self.exclusionFlags = [ @@ -253,3 +259,17 @@ def _generateLinuxConfig(self): else: self.baseImage = LINUX_BASE_IMAGES['opengl'] self.prereqsTag = 'opengl' + + def _processPackageVersion(self, package, version): + + # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified + # (e.g. package==X.X.X, package>=X.X.X, git+https://url/for/package/repo.git, etc.) + if version is None or '/' in version or version.lower().startswith(package): + return version + + # If a version specifier (e.g. ==X.X.X, >=X.X.X, etc.) was specified, prefix it with the package name + if '=' in version: + return package + version + + # If a raw version number was specified, prefix the package name and a strict equality specifier + return '{}=={}'.format(package, version) From bc4375139780dcb6153ce3866621c3b89adccbe1 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 14 May 2020 00:43:27 +1000 Subject: [PATCH 075/430] Force process isolation mode when running tests under Windows --- ue4docker/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/test.py b/ue4docker/test.py index 670ac089..d2cf20ee 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -1,6 +1,6 @@ from .infrastructure import DockerUtils, GlobalConfiguration, Logger from container_utils import ContainerUtils -import docker, os, sys +import docker, os, platform, sys def test(): @@ -22,7 +22,7 @@ def test(): # Start a container to run our tests in, automatically stopping and removing the container when we finish logger.action('Starting a container using the "{}" image...'.format(image), False) - container = ContainerUtils.start_for_exec(client, image) + container = ContainerUtils.start_for_exec(client, image, isolation = 'process' if platform.system().lower() == 'windows' else None) with ContainerUtils.automatically_stop(container): # Create the workspace directory in the container From 9d88a3926f65038dcf925d379a334ed69894669c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 14 May 2020 00:48:25 +1000 Subject: [PATCH 076/430] Bump version to 0.0.67 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 39fa7e3f..3bed87b5 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.66' +__version__ = '0.0.67' From 5889e9726c78ae68e704fb8d7038c93f2e1c0b8e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 May 2020 12:50:54 +1000 Subject: [PATCH 077/430] Improve Windows platform check in `ue4-docker test` command --- ue4docker/test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ue4docker/test.py b/ue4docker/test.py index d2cf20ee..8da5ab88 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -1,5 +1,5 @@ from .infrastructure import DockerUtils, GlobalConfiguration, Logger -from container_utils import ContainerUtils +from container_utils import ContainerUtils, ImageUtils import docker, os, platform, sys def test(): @@ -20,9 +20,13 @@ def test(): logger.error('Error: the specified container image "{}" does not exist.'.format(image)) sys.exit(1) + # Use process isolation mode when testing Windows containers, since running Hyper-V containers don't currently support manipulating the filesystem + platform = ImageUtils.image_platform(client, image) + isolation = 'process' if platform == 'windows' else None + # Start a container to run our tests in, automatically stopping and removing the container when we finish logger.action('Starting a container using the "{}" image...'.format(image), False) - container = ContainerUtils.start_for_exec(client, image, isolation = 'process' if platform.system().lower() == 'windows' else None) + container = ContainerUtils.start_for_exec(client, image, isolation=isolation) with ContainerUtils.automatically_stop(container): # Create the workspace directory in the container From 499e64ea410a576eb50a4feb4f64742799fd616e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 May 2020 12:52:48 +1000 Subject: [PATCH 078/430] Fix typo in conan-ue4cli default version and use latest ue4cli and conan-ue4cli --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index e9540c16..b1da322c 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -1,8 +1,8 @@ ARG NAMESPACE ARG TAG ARG PREREQS_TAG -ARG UE4CLI_VERSION="ue4cli>=0.0.41" -ARG CONAN_UE4CLI_VERSION="ue4cli>=0.0.41" +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index 7bda7ed6..798d4995 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -2,8 +2,8 @@ ARG NAMESPACE ARG TAG ARG PREREQS_TAG -ARG UE4CLI_VERSION="ue4cli>=0.0.41" -ARG CONAN_UE4CLI_VERSION="ue4cli>=0.0.41" +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION From e03614e285f039af484493eb0fa0fed02616e2a1 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 15 May 2020 12:53:18 +1000 Subject: [PATCH 079/430] Bump version to 0.0.68 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 3bed87b5..fb21e294 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.67' +__version__ = '0.0.68' From 5b8b0cece76950a4929c450d6e44bde8ad99481e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 8 Jul 2020 18:12:43 +1000 Subject: [PATCH 080/430] Add support for Windows version 2004 --- ue4docker/infrastructure/WindowsUtils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index cdaa56f0..250b7637 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -12,10 +12,10 @@ class WindowsUtils(object): # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 18362 + _latestReleaseBuild = 19041 # The list of Windows Server Core base image tags that we recognise, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909'] + _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004'] # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) @@ -163,7 +163,8 @@ def getReleaseBaseTag(release): '1803': '1803', '1809': 'ltsc2019', '1903': '1903', - '1909': '1909' + '1909': '1909', + '2004': '2004' }.get(release, 'ltsc2016') @staticmethod From 0db3548cb0c10ae4b9dd6d2e01ca13f2a90952ea Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 9 Jul 2020 12:53:15 +1000 Subject: [PATCH 081/430] Add flags to control Windows container options in the 8gig diagnostic --- ue4docker/diagnostics/diagnostic_8gig.py | 48 ++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index a5df055a..6e565d30 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -1,7 +1,7 @@ from ..infrastructure import DockerUtils, WindowsUtils from .base import DiagnosticBase -import os, platform +import argparse, os, platform from os.path import abspath, dirname, join class diagnostic8Gig(DiagnosticBase): @@ -9,6 +9,14 @@ class diagnostic8Gig(DiagnosticBase): # The tag we use for built images IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:8gig' + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 8gig') + self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") + self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + def getName(self): ''' Returns the human-readable name of the diagnostic @@ -26,8 +34,7 @@ def getDescription(self): 'This bug was fixed in Docker CE 18.09.0, but still exists in some versions of', 'Docker CE under Windows 10 and Docker EE under Windows Server.', '', - 'You can force the use of a Linux image under Windows hosts by specifying the', - '--linux flag, which can be useful when testing Docker Desktop or LCOW support.' + self._parser.format_help() ]) def run(self, logger, args=[]): @@ -35,11 +42,15 @@ def run(self, logger, args=[]): Runs the diagnostic ''' + # Parse our supplied arguments + args = self._parser.parse_args(args) + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) - containerPlatform = 'linux' if '--linux' in args or platform.system().lower() != 'windows' else 'windows' + containerPlatform = 'linux' if args.linux == True or platform.system().lower() != 'windows' else 'windows' # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) - dockerPlatform = DockerUtils.info()['OSType'].lower() + dockerInfo = DockerUtils.info() + dockerPlatform = dockerInfo['OSType'].lower() if containerPlatform == 'windows' and dockerPlatform == 'linux': logger.error('[8gig] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.', False) logger.error('[8gig] Use the --linux flag if you want to test Linux containers instead.', False) @@ -49,12 +60,27 @@ def run(self, logger, args=[]): logger.error('[8gig] Remove the --linux flag if you want to test Windows containers instead.', False) return False - # Under Windows host systems, determine the appropriate container image base tag - buildArgs = [ - '-m', '4GiB', - '--build-arg', - 'BASETAG=' + WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) - ] if containerPlatform == 'windows' else [] + # Set our build arguments when testing Windows containers + buildArgs = [] + if containerPlatform == 'windows': + + # Determine the appropriate container image base tag for the host system release unless the user specified a base tag + defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) + baseTag = args.basetag if args.basetag is not None else defaultBaseTag + buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] + + # Use the default isolation mode unless requested otherwise + isolation = args.isolation if args.isolation is not None else dockerInfo['Isolation'] + buildArgs += ['--isolation={}'.format(isolation)] + + # If the user specified process isolation mode and a different base tag to the host system then warn them + if isolation == 'process' and baseTag != defaultBaseTag: + logger.info('[8gig] Warning: attempting to use different Windows container/host versions', False) + logger.info('[8gig] when running in process isolation mode, this will usually break!', False) + + # Set a sensible memory limit when using Hyper-V isolation mode + if isolation == 'hyperv': + buildArgs += ['-m', '4GiB'] # Attempt to build the Dockerfile contextDir = join(dirname(dirname(abspath(__file__))), 'dockerfiles', 'diagnostics', '8gig', containerPlatform) From 5fd6cb8215643737ec279f99fc9463935ee5234c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 9 Jul 2020 12:54:58 +1000 Subject: [PATCH 082/430] Bump version to 0.0.69 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index fb21e294..f125efa4 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.68' +__version__ = '0.0.69' From d80d6e127fdca451016b246cf0bf6327411118da Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 13 Jul 2020 18:52:52 +1000 Subject: [PATCH 083/430] Update documentation links --- .github/CONTRIBUTING.md | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index d29555d6..7d9c7899 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -12,7 +12,7 @@ Guidelines for contributing to ue4-docker When creating an issue to report a bug, please follow the provided issue template. Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. -If you are creating an issue for a feature request, you can disregard the default issue template. However, please make sure you have thoroughly read [the documentation](https://adamrehn.com/docs/ue4-docker/) to ensure the requested feature does not already exist before creating an issue. +If you are creating an issue for a feature request, you can disregard the default issue template. However, please make sure you have thoroughly read [the documentation](https://docs.adamrehn.com/ue4-docker/) to ensure the requested feature does not already exist before creating an issue. ## Creating pull requests diff --git a/README.md b/README.md index d900361a..b863b4ef 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@

Unreal Engine and Docker Logos

Unreal Engine 4 Docker Containers

-

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

+

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

 

-**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://adamrehn.com/docs/ue4-docker/) to view details specific to using the ue4-docker project.** +**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://docs.adamrehn.com/ue4-docker/) to view details specific to using the ue4-docker project.** The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. @@ -17,7 +17,7 @@ Key features include: Resources: -- **Documentation:** +- **Documentation:** - **GitHub repository:** - **Package on PyPI:** - **Related articles:** From eaca5460b3751ddeafe7c5990ddbe13c4f25f78e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 3 Aug 2020 23:10:47 +1000 Subject: [PATCH 084/430] Print error details when verifying host DLLs under Windows --- .../ue4-build-prerequisites/windows/verify-host-dlls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py index a565f7c3..5d95d11c 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py @@ -34,9 +34,11 @@ def getOsVersion(): dllName = os.path.basename(dll) try: dllVersion = getDllVersion(dll) - except: + except Exception as err: print('\nError: could not read the version string from the DLL file "{}".'.format(dllName), file=sys.stderr) print('Please ensure the DLLs copied from the host are valid DLL files.', file=sys.stderr) + print('\nError details:', file=sys.stderr) + print(err, file=sys.stderr) sys.exit(1) # Print the DLL details From afdbb73bfedf04966a3c403c078b3cb6cc944050 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 4 Aug 2020 11:46:08 +1000 Subject: [PATCH 085/430] More error reporting when verifying host DLLs under Windows --- .../ue4-build-prerequisites/windows/verify-host-dlls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py index 5d95d11c..f58c84a2 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py @@ -49,8 +49,10 @@ def getOsVersion(): try: handle = win32api.LoadLibrary(dll) win32api.FreeLibrary(handle) - except: + except Exception as err: print('\nError: the container OS cannot load the DLL file "{}".'.format(dllName), file=sys.stderr) print('This typically indicates that the DLL is from a newer version of Windows.', file=sys.stderr) print('Please ensure the DLLs copied from the host match the container OS version.', file=sys.stderr) + print('\nError details:', file=sys.stderr) + print(err, file=sys.stderr) sys.exit(1) From 8c5a9f53f39cb90824ea2db055daec7d03ca5295 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 4 Aug 2020 12:18:28 +1000 Subject: [PATCH 086/430] Copy DLLs from Sysnative instead of System32 under 32-bit Python --- ue4docker/infrastructure/BuildConfiguration.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 6daea6c6..44b39565 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -180,7 +180,9 @@ def describeExcludedComponents(self): def _generateWindowsConfig(self): # Store the path to the directory containing our required Windows DLL files - self.defaultDllDir = os.path.join(os.environ['SystemRoot'], 'System32') + hostSysnative = os.path.join(os.environ['SystemRoot'], 'Sysnative') + hostSystem32 = os.path.join(os.environ['SystemRoot'], 'System32') + self.defaultDllDir = hostSysnative if os.path.exists(hostSysnative) else hostSystem32 self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir # Determine base tag for the Windows release of the host system From 440ba910b89cc93af13ad05795de1634d8654d05 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 4 Aug 2020 12:38:23 +1000 Subject: [PATCH 087/430] Bump version to 0.0.70 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index f125efa4..1ec0e79a 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.69' +__version__ = '0.0.70' From 1d54c1a8e5737281aaa175307282e5665e9a5756 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 24 Aug 2020 19:02:28 +1000 Subject: [PATCH 088/430] Add log output to diagnose flag parsing issues --- ue4docker/build.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 81823a18..8da0a3a7 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -75,12 +75,20 @@ def build(): # Resolve our main set of tags for the generated images mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] - # Determine if we are building a custom version of UE4 + # Print the command-line invocation that triggered this build, masking any supplied passwords + args = ['*******' if config.args.password is not None and arg == config.args.password else arg for arg in sys.argv] + logger.info('COMMAND-LINE INVOCATION:', False) + logger.info(str(args), False) + + # Print the details of the Unreal Engine version being built + logger.info('UNREAL ENGINE VERSION SETTINGS:') + logger.info('Custom build: {}'.format('Yes' if config.custom == True else 'No'), False) if config.custom == True: - logger.info('CUSTOM ENGINE BUILD:', False) - logger.info('Custom name: ' + config.release, False) - logger.info('Repository: ' + config.repository, False) - logger.info('Branch/tag: ' + config.branch + '\n', False) + logger.info('Custom name: ' + config.release, False) + else: + logger.info('Release: ' + config.release, False) + logger.info('Repository: ' + config.repository, False) + logger.info('Branch/tag: ' + config.branch + '\n', False) # Determine if we are using a custom version for ue4cli or conan-ue4cli if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: From f9c931c4ced787e5168c8ff0bc7938390257e26c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 24 Aug 2020 19:02:51 +1000 Subject: [PATCH 089/430] Bump version to 0.0.71 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 1ec0e79a..047193d7 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.70' +__version__ = '0.0.71' From 55eaf4c635e55dcf57bc0c405fd225d1bbef7acd Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 24 Aug 2020 20:15:58 +1000 Subject: [PATCH 090/430] Use alternate Chocolatey package for windbg under Windows --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 755c6aca..adc11331 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -38,7 +38,7 @@ RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulka RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" # Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) -RUN choco install -y windbg +RUN choco install -y windows-sdk-10-version-2004-windbg RUN C:\copy.py "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\pdbcopy.exe" C:\GatheredDlls\ # Copy our gathered DLLs (and pdbcopy.exe) into a clean image to reduce image size From c53622d9ab3a016816321a65a9541af08293f3f9 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 31 Oct 2020 17:03:29 +1000 Subject: [PATCH 091/430] Bump version to 0.0.72 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 047193d7..9ff94a6b 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.71' +__version__ = '0.0.72' From ab97e5475017598a26fdca7f2d6449e244185323 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 12:42:17 +1000 Subject: [PATCH 092/430] Fix #113 --- ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 3a5254e6..75e1366b 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -15,6 +15,7 @@ RUN \ curl \ git \ python3 \ + python3-dev \ python3-pip \ shared-mime-info \ tzdata \ From e5049a8027002d2af7bb0802a7d3773afc704b52 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 13:00:45 +1000 Subject: [PATCH 093/430] Add proxy server question to issue template --- .github/ISSUE_TEMPLATE.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 72a6f46a..7a264e17 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,4 +4,8 @@ Output of the `ue4-docker info` command: FULL OUTPUT OF ue4-docker info GOES HERE ``` -(The rest of the issue description goes here. If you're reporting a problem with building the container images, be sure to include the full output of the `ue4-docker build` command, along with the command line parameters used to invoke the build. If you're making a feature request, you can remove the template contents entirely, since the `ue4-docker info` output is only needed for helping diagnose and reproduce bugs.) +Additional details: + +- Are you accessing the network through a proxy server? **Yes/No** + +(The rest of the issue description goes here. If you're reporting a problem with building container images, be sure to include the full output of the `ue4-docker build` command, including the initial output lines that display the command line parameters used to invoke the build. If you're making a feature request, you can remove the template contents entirely, since the `ue4-docker info` output and related information is only needed for helping diagnose and reproduce bugs.) From a29ae6c9bed468475feb384b7e4178ed66809daf Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 15:15:18 +1000 Subject: [PATCH 094/430] Bump required size limit for Windows containers to 400GiB --- ue4docker/infrastructure/WindowsUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 250b7637..31bbd028 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -49,7 +49,7 @@ def requiredSizeLimit(): ''' Returns the minimum required image size limit (in GB) for Windows containers ''' - return 200.0 + return 400.0 @staticmethod def minimumRequiredVersion(): From 7a2fb52df9b14ddb292866f7b09ed76927a8ef63 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 16:59:21 +1000 Subject: [PATCH 095/430] Add support for Windows version 20H2 --- ue4docker/infrastructure/WindowsUtils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 31bbd028..720eec50 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -12,10 +12,10 @@ class WindowsUtils(object): # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 19041 + _latestReleaseBuild = 19042 # The list of Windows Server Core base image tags that we recognise, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004'] + _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004', '20H2'] # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) @@ -164,7 +164,9 @@ def getReleaseBaseTag(release): '1809': 'ltsc2019', '1903': '1903', '1909': '1909', - '2004': '2004' + '2004': '2004', + '2009': '20H2', + '20H2': '20H2' }.get(release, 'ltsc2016') @staticmethod From fddb9735ecabec29c626cf1ee00f786620e92b64 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 17:19:31 +1000 Subject: [PATCH 096/430] Incorporate changes from @TBBle adapted from #97 --- ue4docker/diagnostics/diagnostic_8gig.py | 14 +++++++---- .../diagnostics/8gig/windows/Dockerfile | 20 ++++------------ .../diagnostics/8gig/windows/test.ps1 | 24 +++++++++++++++++++ 3 files changed, 37 insertions(+), 21 deletions(-) create mode 100644 ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index 6e565d30..e4081fa3 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -14,6 +14,7 @@ def __init__(self): # Setup our argument parser so we can use its help message output in our description text self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 8gig') self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") + self._parser.add_argument('--random', action='store_true', help="Create a file filled with random bytes instead of zeroes under Windows") self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") @@ -28,11 +29,11 @@ def getDescription(self): Returns a description of what the diagnostic does ''' return '\n'.join([ - 'This diagnostic determines if the Docker daemon suffers from the 8GiB filesystem', - 'layer bug reported here: https://github.com/moby/moby/issues/37581', + 'This diagnostic determines if the Docker daemon suffers from one of the 8GiB filesystem', + 'layer bugs reported at https://github.com/moby/moby/issues/37581 (affects all platforms)', + 'or https://github.com/moby/moby/issues/40444 (affects Windows containers only)', '', - 'This bug was fixed in Docker CE 18.09.0, but still exists in some versions of', - 'Docker CE under Windows 10 and Docker EE under Windows Server.', + '#37581 was fixed in Docker CE 18.09.0 and #40444 was fixed in Docker CE 20.10.0', '', self._parser.format_help() ]) @@ -67,7 +68,10 @@ def run(self, logger, args=[]): # Determine the appropriate container image base tag for the host system release unless the user specified a base tag defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) baseTag = args.basetag if args.basetag is not None else defaultBaseTag - buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] + buildArgs = [ + '--build-arg', 'BASETAG={}'.format(baseTag), + '--build-arg', 'CREATE_RANDOM={}'.format('1' if args.random == True else '0'), + ] # Use the default isolation mode unless requested otherwise isolation = args.isolation if args.isolation is not None else dockerInfo['Isolation'] diff --git a/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile b/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile index f295dba1..760d9920 100644 --- a/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile +++ b/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile @@ -6,19 +6,7 @@ SHELL ["cmd", "/S", "/C"] # Add a sentinel label so we can easily identify intermediate images LABEL com.adamrehn.ue4-docker.sentinel="1" -# Write an 8GiB file in blocks of 1GiB to avoid allocating too much memory in one hit -RUN powershell "` - $ErrorActionPreference = 'Stop';` - $writer = [System.IO.File]::OpenWrite('file');` - ` - $random = new-object Random;` - $blockSize = 1073741824;` - $bytes = new-object byte[] $blockSize;` - for ($i=0; $i -lt 8; $i++)` - {` - $random.NextBytes($bytes);` - $writer.Write($bytes, 0, $blockSize);` - }` - ` - $writer.Close();` - " +# Atttempt to create an 8GiB filesystem layer +ARG CREATE_RANDOM +COPY test.ps1 C:\test.ps1 +RUN powershell -ExecutionPolicy Bypass -File C:\test.ps1 diff --git a/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 b/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 new file mode 100644 index 00000000..436f4eb2 --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 @@ -0,0 +1,24 @@ +# Determine if we are creating a file filled with zeroes or random bytes +if ($Env:CREATE_RANDOM -ne 1) +{ + Write-Host 'Attempting to create an 8GiB file containing zeroes...'; + fsutil.exe file createnew file 8589934592 +} +else +{ + Write-Host 'Attempting to create an 8GiB file containing random bytes...'; + $ErrorActionPreference = 'Stop'; + $writer = [System.IO.File]::OpenWrite('file'); + + # Write the file in blocks of 1GiB to avoid allocating too much memory in one hit + $random = new-object Random; + $blockSize = 1073741824; + $bytes = new-object byte[] $blockSize; + for ($i=0; $i -lt 8; $i++) + { + $random.NextBytes($bytes); + $writer.Write($bytes, 0, $blockSize); + } + + $writer.Close(); +} From 99ca1b0e5ac9c79830bd900139c018de86fbec42 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 11 Dec 2020 18:00:09 +1000 Subject: [PATCH 097/430] Patch 4.25.4 so it doesn't break --- .../dockerfiles/ue4-source/linux/Dockerfile | 4 ++ .../ue4-source/linux/patch-broken-releases.py | 41 +++++++++++++++++++ .../dockerfiles/ue4-source/windows/Dockerfile | 4 ++ .../windows/patch-broken-releases.py | 41 +++++++++++++++++++ 4 files changed, 90 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py create mode 100644 ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index be61652a..a259fdd7 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -71,3 +71,7 @@ RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX # Apply our bugfix patches to UnrealBuildTool (UBT) COPY --chown=ue4:ue4 patch-ubt.py /tmp/patch-ubt.py RUN python3 /tmp/patch-ubt.py /home/ue4/UnrealEngine/Engine/Source/Programs/UnrealBuildTool + +# Apply our bugfix patches to broken Engine releases such as 4.25.4 +COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py +RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py new file mode 100644 index 00000000..a3505719 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json, os, subprocess, sys +from os.path import join + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Determine if we are building UE 4.25.4 +engineRoot = sys.argv[1] +versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) +if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: + + # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them + # (See: ) + gitdepsFile = join(engineRoot, 'Engine', 'Build', 'Commit.gitdeps.xml') + gitdepsXml = readFile(gitdepsFile) + if '', + '\n ' + + '' + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '' + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '' + ) + + writeFile(gitdepsFile, gitdepsXml) + print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index f30619b7..49a07598 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -45,3 +45,7 @@ RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 # Apply our bugfix patches to UnrealBuildTool (UBT) COPY patch-ubt.py C:\patch-ubt.py RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildTool + +# Apply our bugfix patches to broken Engine releases such as 4.25.4 +COPY patch-broken-releases.py C:\patch-broken-releases.py +RUN python C:\patch-broken-releases.py C:\UnrealEngine\Engine diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py new file mode 100644 index 00000000..a3505719 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +import json, os, subprocess, sys +from os.path import join + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +# Determine if we are building UE 4.25.4 +engineRoot = sys.argv[1] +versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) +if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: + + # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them + # (See: ) + gitdepsFile = join(engineRoot, 'Engine', 'Build', 'Commit.gitdeps.xml') + gitdepsXml = readFile(gitdepsFile) + if '', + '\n ' + + '' + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '' + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '' + ) + + writeFile(gitdepsFile, gitdepsXml) + print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) From 280b23fa647a9c5f734b005a27cb46fd7e0d553c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 12 Dec 2020 15:08:22 +1000 Subject: [PATCH 098/430] Fix incorrect patch path in Windows ue4-source Dockerfile --- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 49a07598..8d31340d 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -48,4 +48,4 @@ RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildToo # Apply our bugfix patches to broken Engine releases such as 4.25.4 COPY patch-broken-releases.py C:\patch-broken-releases.py -RUN python C:\patch-broken-releases.py C:\UnrealEngine\Engine +RUN python C:\patch-broken-releases.py C:\UnrealEngine From 477e284e71797bde4804c88423124bef9531c335 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sun, 13 Dec 2020 16:33:16 +1000 Subject: [PATCH 099/430] Run 4.25.4 fix script before post-clone setup --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 9 +++++---- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index a259fdd7..66c9db98 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -47,6 +47,11 @@ COPY --chown=ue4:ue4 preinstall-mono.py /tmp/preinstall-mono.py RUN python3 /tmp/preinstall-mono.py USER ue4 +# Apply our bugfix patches to broken Engine releases such as 4.25.4 +# (Make sure we do this before the post-clone setup steps are run) +COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py +RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine + # Extract the list of post-clone setup commands that require root and add them to the script, # running everything else as the non-root user to avoid creating files owned by root RUN ./Setup.sh @@ -71,7 +76,3 @@ RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX # Apply our bugfix patches to UnrealBuildTool (UBT) COPY --chown=ue4:ue4 patch-ubt.py /tmp/patch-ubt.py RUN python3 /tmp/patch-ubt.py /home/ue4/UnrealEngine/Engine/Source/Programs/UnrealBuildTool - -# Apply our bugfix patches to broken Engine releases such as 4.25.4 -COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py -RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 8d31340d..9eabab13 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -34,6 +34,11 @@ RUN python C:\legacy-toolchain-fix.py COPY patch-setup-win.py C:\patch-setup-win.py RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat +# Apply our bugfix patches to broken Engine releases such as 4.25.4 +# (Make sure we do this before the post-clone setup steps are run) +COPY patch-broken-releases.py C:\patch-broken-releases.py +RUN python C:\patch-broken-releases.py C:\UnrealEngine + # Run post-clone setup steps WORKDIR C:\UnrealEngine RUN Setup.bat @@ -45,7 +50,3 @@ RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 # Apply our bugfix patches to UnrealBuildTool (UBT) COPY patch-ubt.py C:\patch-ubt.py RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildTool - -# Apply our bugfix patches to broken Engine releases such as 4.25.4 -COPY patch-broken-releases.py C:\patch-broken-releases.py -RUN python C:\patch-broken-releases.py C:\UnrealEngine From c883cea56d37e849d1cf94b6e4f3ec18c4391444 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 14 Dec 2020 02:45:39 +1000 Subject: [PATCH 100/430] Bump version to 0.0.73 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 9ff94a6b..24d47569 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.72' +__version__ = '0.0.73' From f0d17eee64eebf7f3097250ca79fdb230c43f403 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Dec 2020 15:16:46 +1000 Subject: [PATCH 101/430] Add network diagnostic to fix #105 --- ue4docker/diagnostics/__init__.py | 1 + ue4docker/diagnostics/base.py | 77 +++++++++++++++++++ ue4docker/diagnostics/diagnostic_8gig.py | 59 +++----------- ue4docker/diagnostics/diagnostic_all.py | 3 +- ue4docker/diagnostics/diagnostic_maxsize.py | 6 ++ ue4docker/diagnostics/diagnostic_network.py | 72 +++++++++++++++++ ue4docker/diagnostics_cmd.py | 3 +- .../diagnostics/network/linux/Dockerfile | 7 ++ .../diagnostics/network/windows/Dockerfile | 10 +++ 9 files changed, 189 insertions(+), 49 deletions(-) create mode 100644 ue4docker/diagnostics/diagnostic_network.py create mode 100644 ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile create mode 100644 ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile diff --git a/ue4docker/diagnostics/__init__.py b/ue4docker/diagnostics/__init__.py index 1348857b..8c86f7a4 100644 --- a/ue4docker/diagnostics/__init__.py +++ b/ue4docker/diagnostics/__init__.py @@ -1,3 +1,4 @@ from .diagnostic_all import allDiagnostics from .diagnostic_8gig import diagnostic8Gig from .diagnostic_maxsize import diagnosticMaxSize +from .diagnostic_network import diagnosticNetwork diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py index 4589717a..d974118f 100644 --- a/ue4docker/diagnostics/base.py +++ b/ue4docker/diagnostics/base.py @@ -1,3 +1,5 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from os.path import abspath, dirname, join import subprocess class DiagnosticBase(object): @@ -14,6 +16,12 @@ def getDescription(self): ''' raise NotImplementedError + def getPrefix(self): + ''' + Returns the short name of the diagnostic for use in log output + ''' + raise NotImplementedError + def run(self, logger, args=[]): ''' Runs the diagnostic @@ -29,3 +37,72 @@ def _printAndRun(self, logger, prefix, command, check=False): ''' logger.info(prefix + 'Run: {}'.format(command), False) subprocess.run(command, check=check) + + def _checkPlatformMistmatch(self, logger, containerPlatform): + ''' + Verifies that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + ''' + prefix = self.getPrefix() + dockerInfo = DockerUtils.info() + dockerPlatform = dockerInfo['OSType'].lower() + if containerPlatform == 'windows' and dockerPlatform == 'linux': + logger.error('[{}] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.'.format(prefix), False) + logger.error('[{}] Use the --linux flag if you want to test Linux containers instead.'.format(prefix), False) + raise RuntimeError + elif containerPlatform == 'linux' and dockerPlatform == 'windows': + logger.error('[{}] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.'.format(prefix), False) + logger.error('[{}] Remove the --linux flag if you want to test Windows containers instead.'.format(prefix), False) + raise RuntimeError + + def _generateWindowsBuildArgs(self, logger, basetagOverride=None, isolationOverride=None): + ''' + Generates the build arguments for testing Windows containers, with optional overrides for base tag and isolation mode + ''' + + # Determine the appropriate container image base tag for the host system release unless the user specified a base tag + buildArgs = [] + defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) + baseTag = basetagOverride if basetagOverride is not None else defaultBaseTag + buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] + + # Use the default isolation mode unless requested otherwise + dockerInfo = DockerUtils.info() + isolation = isolationOverride if isolationOverride is not None else dockerInfo['Isolation'] + buildArgs += ['--isolation={}'.format(isolation)] + + # If the user specified process isolation mode and a different base tag to the host system then warn them + prefix = self.getPrefix() + if isolation == 'process' and baseTag != defaultBaseTag: + logger.info('[{}] Warning: attempting to use different Windows container/host versions'.format(prefix), False) + logger.info('[{}] when running in process isolation mode, this will usually break!'.format(prefix), False) + + # Set a sensible memory limit when using Hyper-V isolation mode + if isolation == 'hyperv': + buildArgs += ['-m', '4GiB'] + + return buildArgs + + def _buildDockerfile(self, logger, containerPlatform, tag, buildArgs): + ''' + Attempts to build the diagnostic's Dockerfile for the specified container platform, with the specified parameters + ''' + + # Attempt to build the Dockerfile + prefix = self.getPrefix() + contextDir = join(dirname(dirname(abspath(__file__))), 'dockerfiles', 'diagnostics', prefix, containerPlatform) + try: + command = ['docker', 'build', '-t', tag, contextDir] + buildArgs + self._printAndRun(logger, '[{}] '.format(prefix), command, check=True) + built = True + except: + logger.error('[{}] Build failed!'.format(prefix)) + built = False + + # Remove any built images, including intermediate images + logger.action('[{}] Cleaning up...'.format(prefix), False) + if built == True: + self._printAndRun(logger, '[{}] '.format(prefix), ['docker', 'rmi', tag]) + self._printAndRun(logger, '[{}] '.format(prefix), ['docker', 'system', 'prune', '-f']) + + # Report the success or failure of the build + return built diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index e4081fa3..a7c86cc7 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -38,6 +38,12 @@ def getDescription(self): self._parser.format_help() ]) + def getPrefix(self): + ''' + Returns the short name of the diagnostic for use in log output + ''' + return '8gig' + def run(self, logger, args=[]): ''' Runs the diagnostic @@ -50,58 +56,17 @@ def run(self, logger, args=[]): containerPlatform = 'linux' if args.linux == True or platform.system().lower() != 'windows' else 'windows' # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) - dockerInfo = DockerUtils.info() - dockerPlatform = dockerInfo['OSType'].lower() - if containerPlatform == 'windows' and dockerPlatform == 'linux': - logger.error('[8gig] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.', False) - logger.error('[8gig] Use the --linux flag if you want to test Linux containers instead.', False) - return False - elif containerPlatform == 'linux' and dockerPlatform == 'windows': - logger.error('[8gig] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.', False) - logger.error('[8gig] Remove the --linux flag if you want to test Windows containers instead.', False) + try: + self._checkPlatformMistmatch(logger, containerPlatform) + except RuntimeError: return False # Set our build arguments when testing Windows containers - buildArgs = [] - if containerPlatform == 'windows': - - # Determine the appropriate container image base tag for the host system release unless the user specified a base tag - defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) - baseTag = args.basetag if args.basetag is not None else defaultBaseTag - buildArgs = [ - '--build-arg', 'BASETAG={}'.format(baseTag), - '--build-arg', 'CREATE_RANDOM={}'.format('1' if args.random == True else '0'), - ] - - # Use the default isolation mode unless requested otherwise - isolation = args.isolation if args.isolation is not None else dockerInfo['Isolation'] - buildArgs += ['--isolation={}'.format(isolation)] - - # If the user specified process isolation mode and a different base tag to the host system then warn them - if isolation == 'process' and baseTag != defaultBaseTag: - logger.info('[8gig] Warning: attempting to use different Windows container/host versions', False) - logger.info('[8gig] when running in process isolation mode, this will usually break!', False) - - # Set a sensible memory limit when using Hyper-V isolation mode - if isolation == 'hyperv': - buildArgs += ['-m', '4GiB'] + buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) if containerPlatform == 'windows' else [] # Attempt to build the Dockerfile - contextDir = join(dirname(dirname(abspath(__file__))), 'dockerfiles', 'diagnostics', '8gig', containerPlatform) - try: - logger.action('[8gig] Attempting to build an image with an 8GiB filesystem layer...', False) - command = ['docker', 'build', '-t', diagnostic8Gig.IMAGE_TAG, contextDir] + buildArgs - self._printAndRun(logger, '[8gig] ', command, check=True) - built = True - except: - logger.error('[8gig] Build failed!') - built = False - - # Remove any built images, including intermediate images - logger.action('[8gig] Cleaning up...', False) - if built == True: - self._printAndRun(logger, '[8gig] ', ['docker', 'rmi', diagnostic8Gig.IMAGE_TAG]) - self._printAndRun(logger, '[8gig] ', ['docker', 'system', 'prune', '-f']) + logger.action('[8gig] Attempting to build an image with an 8GiB filesystem layer...', False) + built = self._buildDockerfile(logger, containerPlatform, diagnostic8Gig.IMAGE_TAG, buildArgs) # Inform the user of the outcome of the diagnostic if built == True: diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index 4cf603a8..09996ee3 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -1,6 +1,7 @@ from .base import DiagnosticBase from .diagnostic_8gig import diagnostic8Gig from .diagnostic_maxsize import diagnosticMaxSize +from .diagnostic_network import diagnosticNetwork class allDiagnostics(DiagnosticBase): @@ -23,7 +24,7 @@ def run(self, logger, args=[]): # Run all available diagnostics in turn, storing the results results = [] - diagnostics = [diagnostic8Gig(), diagnosticMaxSize()] + diagnostics = [diagnostic8Gig(), diagnosticMaxSize(), diagnosticNetwork()] for index, diagnostic in enumerate(diagnostics): # Run the diagnostic and report its result diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py index 9dcfc14b..c7e30fe5 100644 --- a/ue4docker/diagnostics/diagnostic_maxsize.py +++ b/ue4docker/diagnostics/diagnostic_maxsize.py @@ -24,6 +24,12 @@ def getDescription(self): 'but a workaround was introduced in Docker CE version 19.03.6.', ]) + def getPrefix(self): + ''' + Returns the short name of the diagnostic for use in log output + ''' + return 'maxsize' + def run(self, logger, args=[]): ''' Runs the diagnostic diff --git a/ue4docker/diagnostics/diagnostic_network.py b/ue4docker/diagnostics/diagnostic_network.py new file mode 100644 index 00000000..3d65a075 --- /dev/null +++ b/ue4docker/diagnostics/diagnostic_network.py @@ -0,0 +1,72 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase +import argparse, platform + +class diagnosticNetwork(DiagnosticBase): + + # The tag we use for built images + IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:network' + + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics network') + self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") + self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + return 'Check that containers can access the internet correctly' + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + return '\n'.join([ + 'This diagnostic determines if running containers are able to access the internet,', + 'resolve DNS entries, and download remote files.', + '', + 'This is primarily useful in troubleshooting network connectivity and proxy issues.' + ]) + + def getPrefix(self): + ''' + Returns the short name of the diagnostic for use in log output + ''' + return 'network' + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = 'linux' if args.linux == True or platform.system().lower() != 'windows' else 'windows' + + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + try: + self._checkPlatformMistmatch(logger, containerPlatform) + except RuntimeError: + return False + + # Set our build arguments when testing Windows containers + buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) if containerPlatform == 'windows' else [] + + # Attempt to build the Dockerfile + logger.action('[network] Attempting to build an image that accesses network resources...', False) + built = self._buildDockerfile(logger, containerPlatform, diagnosticNetwork.IMAGE_TAG, buildArgs) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action('[network] Diagnostic succeeded! Running containers can access network resources without any issues.\n') + else: + logger.error('[network] Diagnostic failed! Running containers cannot access network resources. See the docs for troubleshooting tips:', True) + logger.error('[network] https://docs.adamrehn.com/ue4-docker/building-images/troubleshooting-build-issues#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n', False) + + return built diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index f1498e51..4749b716 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -8,7 +8,8 @@ def diagnostics(): DIAGNOSTICS = { 'all': allDiagnostics(), '8gig': diagnostic8Gig(), - 'maxsize': diagnosticMaxSize() + 'maxsize': diagnosticMaxSize(), + 'network': diagnosticNetwork() } # Create our logger to generate coloured output on stderr diff --git a/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile b/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile new file mode 100644 index 00000000..922a3a33 --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Attempt to download the Chocolatey installation script (not actually useful under Linux of course, but useful as a network test) +RUN wget 'https://chocolatey.org/install.ps1' diff --git a/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile b/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile new file mode 100644 index 00000000..9cda65ad --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile @@ -0,0 +1,10 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} +SHELL ["cmd", "/S", "/C"] + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Attempt to download the Chocolatey installation script +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "(New-Object System.Net.WebClient).DownloadFile('https://chocolatey.org/install.ps1', 'install.ps1')" From df72068dd744d5b5fb64b42c406acca9a1f33339 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Dec 2020 15:17:07 +1000 Subject: [PATCH 102/430] Bump version to 0.0.74 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 24d47569..cc8e1983 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.73' +__version__ = '0.0.74' From eaf8a8c0f48fca51b4e122aeecff551751ed62d2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 21 Dec 2020 11:39:57 +1000 Subject: [PATCH 103/430] Only print patched file contents in verbose output mode --- ue4docker/build.py | 3 ++- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 9 ++++++--- .../ue4-source/linux/patch-broken-releases.py | 7 ++++++- .../dockerfiles/ue4-source/linux/patch-setup-linux.py | 6 +++++- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 7 +++++-- .../ue4-source/windows/patch-broken-releases.py | 7 ++++++- .../dockerfiles/ue4-source/windows/patch-setup-win.py | 6 +++++- ue4docker/infrastructure/BuildConfiguration.py | 2 ++ ue4docker/main.py | 7 ++++++- 9 files changed, 43 insertions(+), 11 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 8da0a3a7..8236a77e 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -225,7 +225,8 @@ def build(): prereqConsumerArgs = ['--build-arg', 'PREREQS_TAG={}'.format(config.prereqsTag)] ue4SourceArgs = prereqConsumerArgs + [ '--build-arg', 'GIT_REPO={}'.format(config.repository), - '--build-arg', 'GIT_BRANCH={}'.format(config.branch) + '--build-arg', 'GIT_BRANCH={}'.format(config.branch), + '--build-arg', 'VERBOSE_OUTPUT={}'.format('1' if config.verbose == True else '0') ] builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + endpoint.args()) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 66c9db98..bfb8415d 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -1,6 +1,9 @@ ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 + # The git repository that we will clone ARG GIT_REPO="" @@ -30,8 +33,8 @@ USER ue4 # Patch out all instances of `sudo` in Setup.sh, plus any commands that refuse to run as root COPY --chown=ue4:ue4 patch-setup-linux.py /tmp/patch-setup-linux.py -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Setup.sh -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Engine/Build/BatchFiles/Linux/Setup.sh +RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Setup.sh $VERBOSE_OUTPUT +RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Engine/Build/BatchFiles/Linux/Setup.sh $VERBOSE_OUTPUT # Create a script to hold the list of post-clone setup commands that require root WORKDIR /home/ue4/UnrealEngine @@ -50,7 +53,7 @@ USER ue4 # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py -RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine +RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT # Extract the list of post-clone setup commands that require root and add them to the script, # running everything else as the non-root user to avoid creating files owned by root diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py index a3505719..9919defb 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -12,6 +12,7 @@ def writeFile(filename, data): # Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: @@ -38,4 +39,8 @@ def writeFile(filename, data): ) writeFile(gitdepsFile, gitdepsXml) - print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) + + if verboseOutput == True: + print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) + else: + print('PATCHED {}'.format(gitdepsFile), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py b/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py index c0368dce..ab26ce66 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py @@ -11,9 +11,13 @@ def writeFile(filename, data): # Extract all commands requiring `sudo` in Setup.sh and place them in root_commands.sh setupScript = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' code = readFile(setupScript) code = re.sub('(\\s)sudo ([^\\n]+)\\n', '\\1echo \\2 >> /home/ue4/UnrealEngine/root_commands.sh\\n', code) writeFile(setupScript, code) # Print the patched code to stderr for debug purposes -print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) +if verboseOutput == True: + print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) +else: + print('PATCHED {}'.format(setupScript), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 9eabab13..b4bc2b42 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -2,6 +2,9 @@ ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 + # The git repository that we will clone ARG GIT_REPO="" @@ -32,12 +35,12 @@ RUN python C:\legacy-toolchain-fix.py # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat COPY patch-setup-win.py C:\patch-setup-win.py -RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat +RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat %VERBOSE_OUTPUT% # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) COPY patch-broken-releases.py C:\patch-broken-releases.py -RUN python C:\patch-broken-releases.py C:\UnrealEngine +RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% # Run post-clone setup steps WORKDIR C:\UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index a3505719..9919defb 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -12,6 +12,7 @@ def writeFile(filename, data): # Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: @@ -38,4 +39,8 @@ def writeFile(filename, data): ) writeFile(gitdepsFile, gitdepsXml) - print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) + + if verboseOutput == True: + print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) + else: + print('PATCHED {}'.format(gitdepsFile), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py b/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py index ecb6b88f..2415a446 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py @@ -12,6 +12,7 @@ def writeFile(filename, data): # Comment out the call to the UE4 prereqs installer in Setup.bat PREREQ_CALL = 'start /wait Engine\\Extras\\Redist\\en-us\\UE4PrereqSetup_x64.exe' setupScript = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' code = readFile(setupScript) code = code.replace('echo Installing prerequisites...', 'echo (Skipping installation of prerequisites)') code = code.replace(PREREQ_CALL, '@rem ' + PREREQ_CALL) @@ -26,4 +27,7 @@ def writeFile(filename, data): writeFile(setupScript, code) # Print the patched code to stderr for debug purposes -print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) +if verboseOutput == True: + print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) +else: + print('PATCHED {}'.format(setupScript), file=sys.stderr) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 44b39565..82e3cd6c 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -90,6 +90,7 @@ def addArguments(parser): parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') + parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output during builds (useful for debugging)') def __init__(self, parser, argv): ''' @@ -147,6 +148,7 @@ def __init__(self, parser, argv): self.baseImage = None self.prereqsTag = None self.ignoreBlacklist = self.args.ignore_blacklist + self.verbose = self.args.verbose # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) diff --git a/ue4docker/main.py b/ue4docker/main.py index 0f12fa4c..7b42e5de 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -18,8 +18,13 @@ def main(): # Configure verbose logging if the user requested it # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) if '-v' in sys.argv or '--verbose' in sys.argv: - sys.argv = list([arg for arg in sys.argv if arg not in ['-v', '--verbose']]) + + # Enable verbose logging logging.getLogger().setLevel(logging.DEBUG) + + # Filter out the verbose flag to avoid breaking commands that don't support it + if not (len(sys.argv) > 1 and sys.argv[1] in ['build']): + sys.argv = list([arg for arg in sys.argv if arg not in ['-v', '--verbose']]) # Verify that Docker is installed if DockerUtils.installed() == False: From 9c0429c5d5e1b648c6fc4197361f129ab3ddd2aa Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 21 Dec 2020 12:56:31 +1000 Subject: [PATCH 104/430] Experimental: remove .git directory to speed up builds --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 3 +++ ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 8ab49ef0..d00a45f4 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -3,6 +3,9 @@ ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN rm -rf /home/ue4/UnrealEngine/.git + # Set the changelist number in Build.version to ensure our Build ID is generated correctly COPY set-changelist.py /tmp/set-changelist.py RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index c8462f5e..0903a723 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -4,6 +4,9 @@ ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN del /s /q C:\UnrealEngine\.git + # Set the changelist number in Build.version to ensure our Build ID is generated correctly COPY set-changelist.py C:\set-changelist.py RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version From 3b64eeb3fb249e40ffc7889e838267de87f3276d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 22 Dec 2020 11:01:52 +1000 Subject: [PATCH 105/430] Bump version to 0.0.75 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index cc8e1983..ba7c809f 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.74' +__version__ = '0.0.75' From a13207f7a1986ca549ba7d3bbb8a701d9a8d6dcc Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 28 Dec 2020 20:49:50 +1000 Subject: [PATCH 106/430] Use the Windows SDK v1809 Chocolatey package for WinDbg --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index adc11331..5e105dfd 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -38,7 +38,7 @@ RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulka RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" # Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) -RUN choco install -y windows-sdk-10-version-2004-windbg +RUN choco install -y windows-sdk-10-version-1809-windbg RUN C:\copy.py "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\pdbcopy.exe" C:\GatheredDlls\ # Copy our gathered DLLs (and pdbcopy.exe) into a clean image to reduce image size From 4e467bc96de211ae7490587ed12cc9e661ac4b6f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 29 Dec 2020 23:42:33 +1000 Subject: [PATCH 107/430] Bump version to 0.0.76 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index ba7c809f..700e0db6 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.75' +__version__ = '0.0.76' From ef0c05f6d2b4536ef2e6aab4ed5712a34b1de20d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 29 Jan 2021 16:45:23 +1000 Subject: [PATCH 108/430] Don't assume semver format for macOS version strings --- setup.py | 1 + ue4docker/infrastructure/DarwinUtils.py | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index 2ed64bcf..0651f0f8 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'docker>=3.0.0', 'flask', 'humanfriendly', + 'packaging>=19.1', 'psutil', 'requests', 'semver>=2.7.9', diff --git a/ue4docker/infrastructure/DarwinUtils.py b/ue4docker/infrastructure/DarwinUtils.py index b61e0e71..7c054693 100644 --- a/ue4docker/infrastructure/DarwinUtils.py +++ b/ue4docker/infrastructure/DarwinUtils.py @@ -1,9 +1,6 @@ -from .PackageUtils import PackageUtils +from packaging import version import os, platform -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - class DarwinUtils(object): @staticmethod @@ -38,4 +35,4 @@ def isSupportedMacOsVersion(): ''' Verifies that the macOS host system meets our minimum version requirements ''' - return semver.compare(DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion()) >= 0 + return version.parse(DarwinUtils.getMacOsVersion()) >= version.parse(DarwinUtils.minimumRequiredVersion()) From f5a883b7e5e26f3d243bbc946f03f5eb9e716e9f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 30 Jan 2021 19:21:07 +1000 Subject: [PATCH 109/430] Bump version to 0.0.77 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 700e0db6..953095ee 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.76' +__version__ = '0.0.77' From d1f2f95be54b7a438f7f994f353c2036728debbf Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 10:47:31 +1000 Subject: [PATCH 110/430] Add -layout option to the build command --- ue4docker/build.py | 15 ++++++++++++++- ue4docker/infrastructure/BuildConfiguration.py | 4 +++- ue4docker/infrastructure/ImageBuilder.py | 18 +++++++++++++++--- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 8236a77e..b6899509 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -70,7 +70,7 @@ def build(): shutil.copytree(contextOrig, contextRoot) # Create the builder instance to build the Docker images - builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun) + builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir) # Resolve our main set of tags for the generated images mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] @@ -180,6 +180,13 @@ def build(): username = '' password = '' + elif config.layoutDir is not None: + + # Don't bother prompting the user for any credentials when we're just copying the Dockerfiles to a directory + logger.info('Copying generated Dockerfiles to: {}'.format(config.layoutDir), False) + username = '' + password = '' + elif builder.willBuild('ue4-source', mainTags) == False: # Don't bother prompting the user for any credentials if we're not building the ue4-source image @@ -209,6 +216,12 @@ def build(): # Keep track of our starting time startTime = time.time() + # If we're copying Dockerfiles to an output directory then make sure it exists and is empty + if config.layoutDir is not None: + if os.path.exists(config.layoutDir): + shutil.rmtree(config.layoutDir) + os.makedirs(config.layoutDir) + # Compute the build options for the UE4 build prerequisites image # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 82e3cd6c..c81535d0 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -77,7 +77,7 @@ def addArguments(parser): parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.DDC, ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') - parser.add_argument('-password', default=None, help='Specify the password to use when cloning the git repository') + parser.add_argument('-password', default=None, help='Specify the password or access token to use when cloning the git repository') parser.add_argument('-repo', default=None, help='Set the custom git repository to clone when "custom" is specified as the release value') parser.add_argument('-branch', default=None, help='Set the custom branch/tag to clone when "custom" is specified as the release value') parser.add_argument('-isolation', default=None, help='Set the isolation mode to use for Windows containers (process or hyperv)') @@ -87,6 +87,7 @@ def addArguments(parser): parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image') parser.add_argument('-conan-ue4cli', default=None, help='Override the default version of conan-ue4cli installed in the ue4-full image') + parser.add_argument('-layout', default=None, help='Copy generated Dockerfiles to the specified directory and don\'t build the images') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') @@ -149,6 +150,7 @@ def __init__(self, parser, argv): self.prereqsTag = None self.ignoreBlacklist = self.args.ignore_blacklist self.verbose = self.args.verbose + self.layoutDir = self.args.layout # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 55140662..3e20e4ab 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,10 +1,10 @@ from .DockerUtils import DockerUtils from .GlobalConfiguration import GlobalConfiguration -import humanfriendly, os, subprocess, time +import humanfriendly, os, shutil, subprocess, time class ImageBuilder(object): - def __init__(self, root, platform, logger, rebuild=False, dryRun=False): + def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None): ''' Creates an ImageBuilder for the specified build parameters ''' @@ -13,6 +13,7 @@ def __init__(self, root, platform, logger, rebuild=False, dryRun=False): self.logger = logger self.rebuild = rebuild self.dryRun = dryRun + self.layoutDir = layoutDir def build(self, name, tags, args): ''' @@ -33,6 +34,7 @@ def build(self, name, tags, args): imageTags = self._formatTags(name, tags) self._processImage( imageTags[0], + name, DockerUtils.build(imageTags, self.context(name), args), 'build', 'built' @@ -50,6 +52,7 @@ def pull(self, image): ''' self._processImage( image, + None, DockerUtils.pull(image), 'pull', 'pulled' @@ -74,7 +77,7 @@ def _willProcess(self, image): ''' return self.rebuild == True or DockerUtils.exists(image) == False - def _processImage(self, image, command, actionPresentTense, actionPastTense): + def _processImage(self, image, name, command, actionPresentTense, actionPastTense): ''' Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) ''' @@ -91,6 +94,15 @@ def _processImage(self, image, command, actionPresentTense, actionPastTense): self.logger.action('Completed dry run for image "{}".'.format(image), newline=False) return + # Determine if we're just copying the Dockerfile to an output directory + if self.layoutDir is not None: + source = self.context(name) + dest = os.path.join(self.layoutDir, os.path.basename(name)) + self.logger.action('Copying "{}" to "{}"...'.format(source, dest), newline=False) + shutil.copytree(source, dest) + self.logger.action('Copied Dockerfile for image "{}".'.format(image), newline=False) + return + # Attempt to process the image using the supplied command startTime = time.time() exitCode = subprocess.call(command) From 379935ca06c3b9908d6405179df38f547fd67be4 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 10:50:44 +1000 Subject: [PATCH 111/430] Disable platform mismatch check when -layout is specified --- ue4docker/build.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index b6899509..241abea0 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -51,15 +51,17 @@ def build(): sys.exit(1) # Verify that we aren't trying to build Windows containers under Windows 10 when in Linux container mode (or vice versa) - dockerPlatform = DockerUtils.info()['OSType'].lower() - if config.containerPlatform == 'windows' and dockerPlatform == 'linux': - logger.error('Error: cannot build Windows containers when Docker Desktop is in Linux container', False) - logger.error('mode. Use the --linux flag if you want to build Linux containers instead.', False) - sys.exit(1) - elif config.containerPlatform == 'linux' and dockerPlatform == 'windows': - logger.error('Error: cannot build Linux containers when Docker Desktop is in Windows container', False) - logger.error('mode. Remove the --linux flag if you want to build Windows containers instead.', False) - sys.exit(1) + # (Note that we don't bother performing this check when we're just copying Dockerfiles to an output directory) + if config.layoutDir is None: + dockerPlatform = DockerUtils.info()['OSType'].lower() + if config.containerPlatform == 'windows' and dockerPlatform == 'linux': + logger.error('Error: cannot build Windows containers when Docker Desktop is in Linux container', False) + logger.error('mode. Use the --linux flag if you want to build Linux containers instead.', False) + sys.exit(1) + elif config.containerPlatform == 'linux' and dockerPlatform == 'windows': + logger.error('Error: cannot build Linux containers when Docker Desktop is in Windows container', False) + logger.error('mode. Remove the --linux flag if you want to build Windows containers instead.', False) + sys.exit(1) # Create an auto-deleting temporary directory to hold our build context with tempfile.TemporaryDirectory() as tempDir: From 78637ea9ecf3dd2893e1972f471de6233e9785fd Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 11:13:22 +1000 Subject: [PATCH 112/430] Add Jinja templating support to make Dockerfile generation more flexible --- setup.py | 1 + ue4docker/build.py | 9 ++++++++- ue4docker/infrastructure/BuildConfiguration.py | 10 ++++++++++ ue4docker/infrastructure/ImageBuilder.py | 13 +++++++++++-- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0651f0f8..e7dad001 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ 'docker>=3.0.0', 'flask', 'humanfriendly', + 'Jinja2>=2.11.3', 'packaging>=19.1', 'psutil', 'requests', diff --git a/ue4docker/build.py b/ue4docker/build.py index 241abea0..9c26c205 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -72,7 +72,7 @@ def build(): shutil.copytree(contextOrig, contextRoot) # Create the builder instance to build the Docker images - builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir) + builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir, config.opts) # Resolve our main set of tags for the generated images mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] @@ -98,6 +98,13 @@ def build(): logger.info('ue4cli: {}'.format(config.ue4cliVersion if config.ue4cliVersion is not None else 'default'), False) logger.info('conan-ue4cli: {}\n'.format(config.conanUe4cliVersion if config.conanUe4cliVersion is not None else 'default'), False) + # Report any advanced configuration options that were specified + if len(config.opts) > 0: + logger.info('ADVANCED CONFIGURATION OPTIONS:', False) + for key, value in config.opts.items(): + logger.info('{}: {}'.format(key, value), False) + print('', file=sys.stderr, flush=True) + # Determine if we are building Windows or Linux containers if config.containerPlatform == 'windows': diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index c81535d0..738b75f6 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -75,6 +75,7 @@ def addArguments(parser): parser.add_argument('--no-cache', action='store_true', help='Disable Docker build cache') parser.add_argument('--random-memory', action='store_true', help='Use a random memory limit for Windows containers') parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.DDC, ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') + parser.add_argument('--opt', action='append', default=[], help='Set an advanced configuration option (can be specified multiple times to specify multiple options)') parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') parser.add_argument('-password', default=None, help='Specify the password or access token to use when cloning the git repository') @@ -156,6 +157,15 @@ def __init__(self, parser, argv): self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) self.conanUe4cliVersion = self._processPackageVersion('conan-ue4cli', self.args.conan_ue4cli) + # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) + self.opts = {} + for o in self.args.opt: + if '=' in o: + key, value = o.split('=', 1) + self.opts[key] = value + else: + self.opts[o] = True + # Generate our flags for keeping or excluding components self.exclusionFlags = [ diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 3e20e4ab..77af5cd1 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,10 +1,12 @@ from .DockerUtils import DockerUtils +from .FilesystemUtils import FilesystemUtils from .GlobalConfiguration import GlobalConfiguration import humanfriendly, os, shutil, subprocess, time +from jinja2 import Environment, Template class ImageBuilder(object): - def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None): + def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None, templateContext=None): ''' Creates an ImageBuilder for the specified build parameters ''' @@ -14,14 +16,21 @@ def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDi self.rebuild = rebuild self.dryRun = dryRun self.layoutDir = layoutDir + self.templateContext = templateContext if templateContext is not None else {} def build(self, name, tags, args): ''' Builds the specified image if it doesn't exist or if we're forcing a rebuild ''' - # Inject our filesystem layer commit message after each RUN directive in the Dockerfile + # Create a Jinja template environment and render the Dockerfile template + environment = Environment(autoescape=False) dockerfile = os.path.join(self.context(name), 'Dockerfile') + templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) + rendered = templateInstance.render(self.templateContext) + FilesystemUtils.writeFile(dockerfile, rendered) + + # Inject our filesystem layer commit message after each RUN directive in the Dockerfile DockerUtils.injectPostRunMessage(dockerfile, self.platform, [ '', 'RUN directive complete. Docker will now commit the filesystem layer to disk.', From 18d5fa661c8959b8ad12c016acfec429ec1f32c9 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 11:31:45 +1000 Subject: [PATCH 113/430] Add template flag to disable legacy (4.19) patches --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 2 ++ .../dockerfiles/ue4-source/windows/Dockerfile | 2 ++ ue4docker/infrastructure/BuildConfiguration.py | 14 ++++++++++++-- ue4docker/infrastructure/ImageBuilder.py | 2 +- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index bfb8415d..c43a2449 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -43,12 +43,14 @@ RUN echo "set -x" >> ./root_commands.sh RUN echo "apt-get update" >> ./root_commands.sh RUN chmod a+x ./root_commands.sh +{% if not disable_legacy_patches %} # Preinstall mono for 4.19 so the GitDependencies.exe call triggered by Setup.sh works correctly # (this is a no-op for newer Engine versions) USER root COPY --chown=ue4:ue4 preinstall-mono.py /tmp/preinstall-mono.py RUN python3 /tmp/preinstall-mono.py USER ue4 +{% endif %} # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index b4bc2b42..84e56d54 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -27,10 +27,12 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat WORKDIR C:\ RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine +{% if not disable_legacy_patches %} # Install legacy toolchain components if we're building UE4.19 # (This call is a no-op under newer Engine versions) COPY legacy-toolchain-fix.py C:\legacy-toolchain-fix.py RUN python C:\legacy-toolchain-fix.py +{% endif %} # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 738b75f6..11b538e3 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -162,9 +162,9 @@ def __init__(self, parser, argv): for o in self.args.opt: if '=' in o: key, value = o.split('=', 1) - self.opts[key] = value + self.opts[key.replace('-', '_')] = self._processTemplateValue(value) else: - self.opts[o] = True + self.opts[o.replace('-', '_')] = True # Generate our flags for keeping or excluding components self.exclusionFlags = [ @@ -289,3 +289,13 @@ def _processPackageVersion(self, package, version): # If a raw version number was specified, prefix the package name and a strict equality specifier return '{}=={}'.format(package, version) + + def _processTemplateValue(self, value): + + # If the value is a boolean (either raw or represented by zero or one) then parse it + if value.lower() in ['true', '1']: + return True + elif value.lower() in ['false', '0']: + return False + + return value diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 77af5cd1..5b7301f9 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -24,7 +24,7 @@ def build(self, name, tags, args): ''' # Create a Jinja template environment and render the Dockerfile template - environment = Environment(autoescape=False) + environment = Environment(autoescape=False, trim_blocks=True, lstrip_blocks=True) dockerfile = os.path.join(self.context(name), 'Dockerfile') templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) rendered = templateInstance.render(self.templateContext) From 709318c324fbf0a8eb0cd37648b9904a3f90eaaa Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 15:50:40 +1000 Subject: [PATCH 114/430] Add support for BuildKit build secrets under Linux --- ue4docker/build.py | 11 +++++- .../dockerfiles/ue4-source/linux/Dockerfile | 26 ++++++++++--- ...r.sh => git-credential-helper-endpoint.sh} | 0 .../linux/git-credential-helper-secrets.sh | 6 +++ ue4docker/infrastructure/DockerUtils.py | 8 ++++ ue4docker/infrastructure/ImageBuilder.py | 39 +++++++++++++------ 6 files changed, 73 insertions(+), 17 deletions(-) rename ue4docker/dockerfiles/ue4-source/linux/{git-credential-helper.sh => git-credential-helper-endpoint.sh} (100%) create mode 100644 ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh diff --git a/ue4docker/build.py b/ue4docker/build.py index 9c26c205..ea42a6cf 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -243,14 +243,23 @@ def build(): else: builder.build('adamrehn/ue4-build-prerequisites', [config.prereqsTag], config.platformArgs + prereqsArgs) + # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets + secrets = {} + if config.opts.get('use_build_secrets', False) == True: + secrets = { + 'username': username, + 'password': password + } + # Build the UE4 source image prereqConsumerArgs = ['--build-arg', 'PREREQS_TAG={}'.format(config.prereqsTag)] + credentialArgs = [] if len(secrets) > 0 else endpoint.args() ue4SourceArgs = prereqConsumerArgs + [ '--build-arg', 'GIT_REPO={}'.format(config.repository), '--build-arg', 'GIT_BRANCH={}'.format(config.branch), '--build-arg', 'VERBOSE_OUTPUT={}'.format('1' if config.verbose == True else '0') ] - builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + endpoint.args()) + builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + credentialArgs, secrets) # Build the UE4 Engine source build image, unless requested otherwise by the user ue4BuildArgs = prereqConsumerArgs + [ diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index c43a2449..78da1c0c 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -14,18 +14,34 @@ ARG GIT_BRANCH="" ARG HOST_ADDRESS_ARG="" ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} +{% if use_build_secrets %} + +# Install our git credential helper that retrieves credentials from build secrets +COPY --chown=ue4:ue4 git-credential-helper-secrets.sh /tmp/git-credential-helper-secrets.sh +ENV GIT_ASKPASS=/tmp/git-credential-helper-secrets.sh +RUN chmod +x /tmp/git-credential-helper-secrets.sh + +# Clone the UE4 git repository using the build secret credentials +RUN --mount=type=secret,id=username,uid=1000,required \ + --mount=type=secret,id=password,uid=1000,required \ + git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine + +{% else %} + # Retrieve the security token for communicating with the credential supplier ARG HOST_TOKEN_ARG="" ENV HOST_TOKEN=${HOST_TOKEN_ARG} -# Install our git credential helper that forwards requests to the host -COPY --chown=ue4:ue4 git-credential-helper.sh /tmp/git-credential-helper.sh -ENV GIT_ASKPASS=/tmp/git-credential-helper.sh -RUN chmod +x /tmp/git-credential-helper.sh +# Install our git credential helper that forwards requests to the credential HTTP endpoint on the host +COPY --chown=ue4:ue4 git-credential-helper-endpoint.sh /tmp/git-credential-helper-endpoint.sh +ENV GIT_ASKPASS=/tmp/git-credential-helper-endpoint.sh +RUN chmod +x /tmp/git-credential-helper-endpoint.sh -# Clone the UE4 git repository using the host-supplied credentials +# Clone the UE4 git repository using the endpoint-supplied credentials RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine +{% endif %} + # Ensure our package lists are up to date, since Setup.sh doesn't call `apt-get update` USER root RUN apt-get update diff --git a/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper.sh b/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/git-credential-helper.sh rename to ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh diff --git a/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh b/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh new file mode 100644 index 00000000..3063419f --- /dev/null +++ b/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +if [[ "$1" == "Password for"* ]]; then + cat /run/secrets/password +else + cat /run/secrets/username +fi diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 04f89f26..82815766 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -50,6 +50,14 @@ def build(tags, context, args): tagArgs = [['-t', tag] for tag in tags] return ['docker', 'build'] + list(itertools.chain.from_iterable(tagArgs)) + [context] + args + @staticmethod + def buildx(tags, context, args, secrets): + ''' + Returns the `docker buildx` command to build an image with the BuildKit backend + ''' + tagArgs = [['-t', tag] for tag in tags] + return ['docker', 'buildx', 'build'] + list(itertools.chain.from_iterable(tagArgs)) + [context] + ['--progress=plain'] + args + list(itertools.chain.from_iterable([['--secret', s] for s in secrets])) + @staticmethod def pull(image): ''' diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 5b7301f9..425c9b1c 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,7 +1,7 @@ from .DockerUtils import DockerUtils from .FilesystemUtils import FilesystemUtils from .GlobalConfiguration import GlobalConfiguration -import humanfriendly, os, shutil, subprocess, time +import humanfriendly, os, shutil, subprocess, tempfile, time from jinja2 import Environment, Template class ImageBuilder(object): @@ -18,7 +18,7 @@ def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDi self.layoutDir = layoutDir self.templateContext = templateContext if templateContext is not None else {} - def build(self, name, tags, args): + def build(self, name, tags, args, secrets=None): ''' Builds the specified image if it doesn't exist or if we're forcing a rebuild ''' @@ -39,15 +39,32 @@ def build(self, name, tags, args): '' ]) - # Build the image if it doesn't already exist - imageTags = self._formatTags(name, tags) - self._processImage( - imageTags[0], - name, - DockerUtils.build(imageTags, self.context(name), args), - 'build', - 'built' - ) + # Create a temporary directory to hold any files needed for the build + with tempfile.TemporaryDirectory() as tempDir: + + # Determine whether we are building using `docker buildx` with build secrets + imageTags = self._formatTags(name, tags) + command = DockerUtils.build(imageTags, self.context(name), args) + if self.platform == 'linux' and secrets is not None and len(secrets) > 0: + + # Create temporary files to store the contents of each of our secrets + secretFlags = [] + for secret, contents in secrets.items(): + secretFile = os.path.join(tempDir, secret) + FilesystemUtils.writeFile(secretFile, contents) + secretFlags.append('id={},src={}'.format(secret, secretFile)) + + # Generate the `docker buildx` command to use our build secrets + command = DockerUtils.buildx(imageTags, self.context(name), args, secretFlags) + + # Build the image if it doesn't already exist + self._processImage( + imageTags[0], + name, + command, + 'build', + 'built' + ) def context(self, name): ''' From 74efda6c03f505d29b5953c449574d24afe7e349 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 15:57:30 +1000 Subject: [PATCH 115/430] Move credential endpoint address arg into conditional block with endpoint logic --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 78da1c0c..8de82a33 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -10,10 +10,6 @@ ARG GIT_REPO="" # The git branch/tag that we will checkout ARG GIT_BRANCH="" -# Retrieve the address for the host that will supply git credentials -ARG HOST_ADDRESS_ARG="" -ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} - {% if use_build_secrets %} # Install our git credential helper that retrieves credentials from build secrets @@ -28,6 +24,10 @@ RUN --mount=type=secret,id=username,uid=1000,required \ {% else %} +# Retrieve the address for the host that will supply git credentials +ARG HOST_ADDRESS_ARG="" +ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} + # Retrieve the security token for communicating with the credential supplier ARG HOST_TOKEN_ARG="" ENV HOST_TOKEN=${HOST_TOKEN_ARG} From 57a600aa7b4899aa806696bbfe22f89947317915 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 16:53:14 +1000 Subject: [PATCH 116/430] Drop support for UE4.19, which allows us to use sudo under Linux --- .../ue4-build-prerequisites/linux/Dockerfile | 12 ++++-- .../dockerfiles/ue4-minimal/linux/Dockerfile | 6 --- .../ue4-minimal/windows/Dockerfile | 4 -- .../dockerfiles/ue4-source/linux/Dockerfile | 42 +++---------------- .../ue4-source/linux/patch-setup-linux.py | 23 ---------- .../ue4-source/linux/preinstall-mono.py | 38 ----------------- .../dockerfiles/ue4-source/windows/Dockerfile | 7 ---- .../windows/legacy-toolchain-fix.py | 29 ------------- .../infrastructure/BuildConfiguration.py | 4 +- 9 files changed, 15 insertions(+), 150 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py delete mode 100644 ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py delete mode 100644 ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 75e1366b..49d70cdc 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -18,14 +18,18 @@ RUN \ python3-dev \ python3-pip \ shared-mime-info \ + sudo \ tzdata \ unzip \ xdg-user-dirs \ zip && \ rm -rf /var/lib/apt/lists/* -# Since UE4 refuses to build or run as the root user under Linux, create a non-root user -RUN \ - useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ - usermod -a -G audio,video ue4 +# Disable the default "lecture" message the first time a user runs a command using sudo +RUN echo 'Defaults lecture="never"' >> /etc/sudoers + +# Unreal refuses to run as the root user, so create a non-root user with no password and allow them to run commands using sudo +RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ + passwd -d ue4 && \ + usermod -a -G audio,video,sudo ue4 USER ue4 diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index d00a45f4..b2a95008 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -50,14 +50,8 @@ FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} # Copy the Installed Build files from the builder image COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine -COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/root_commands.sh /tmp/root_commands.sh WORKDIR /home/ue4/UnrealEngine -# Run the post-setup commands that were previously extracted from `Setup.sh` -USER root -RUN /tmp/root_commands.sh -USER ue4 - # Add labels to the built image to identify which components (if any) were excluded from the build that it contains # (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) ARG EXCLUDE_DEBUG diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 0903a723..3534a32b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -43,10 +43,6 @@ FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine WORKDIR C:\UnrealEngine -# Install legacy toolchain components if we're building UE4.19 -# (This call is a no-op under newer Engine versions) -RUN python C:\UnrealEngine\legacy-toolchain-fix.py - # Add labels to the built image to identify which components (if any) were excluded from the build that it contains # (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) ARG EXCLUDE_DEBUG diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 8de82a33..fa9e4aaf 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -42,48 +42,16 @@ RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngi {% endif %} -# Ensure our package lists are up to date, since Setup.sh doesn't call `apt-get update` -USER root -RUN apt-get update -USER ue4 - -# Patch out all instances of `sudo` in Setup.sh, plus any commands that refuse to run as root -COPY --chown=ue4:ue4 patch-setup-linux.py /tmp/patch-setup-linux.py -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Setup.sh $VERBOSE_OUTPUT -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Engine/Build/BatchFiles/Linux/Setup.sh $VERBOSE_OUTPUT - -# Create a script to hold the list of post-clone setup commands that require root -WORKDIR /home/ue4/UnrealEngine -RUN echo "#!/usr/bin/env bash" >> ./root_commands.sh -RUN echo "set -x" >> ./root_commands.sh -RUN echo "apt-get update" >> ./root_commands.sh -RUN chmod a+x ./root_commands.sh - -{% if not disable_legacy_patches %} -# Preinstall mono for 4.19 so the GitDependencies.exe call triggered by Setup.sh works correctly -# (this is a no-op for newer Engine versions) -USER root -COPY --chown=ue4:ue4 preinstall-mono.py /tmp/preinstall-mono.py -RUN python3 /tmp/preinstall-mono.py -USER ue4 -{% endif %} - # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT -# Extract the list of post-clone setup commands that require root and add them to the script, -# running everything else as the non-root user to avoid creating files owned by root -RUN ./Setup.sh - -# Run the extracted root commands we gathered in our script -USER root -RUN ./root_commands.sh -USER ue4 - -# Make sure the root commands script cleans up the package lists when it is run in the ue4-minimal image -RUN echo "rm -rf /var/lib/apt/lists/*" >> ./root_commands.sh +# Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` +WORKDIR /home/ue4/UnrealEngine +RUN sudo apt-get update && \ + ./Setup.sh && \ + sudo rm -rf /var/lib/apt/lists/* # The linker bundled with UE4.20.0 onwards chokes on system libraries built with newer compilers, # so redirect the bundled clang to use the system linker instead diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py b/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py deleted file mode 100644 index ab26ce66..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Extract all commands requiring `sudo` in Setup.sh and place them in root_commands.sh -setupScript = sys.argv[1] -verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' -code = readFile(setupScript) -code = re.sub('(\\s)sudo ([^\\n]+)\\n', '\\1echo \\2 >> /home/ue4/UnrealEngine/root_commands.sh\\n', code) -writeFile(setupScript, code) - -# Print the patched code to stderr for debug purposes -if verboseOutput == True: - print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) -else: - print('PATCHED {}'.format(setupScript), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py b/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py deleted file mode 100644 index b98b8c3b..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -import json, os, subprocess, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Determine if we are building UE 4.19 -versionDetails = json.loads(readFile('/home/ue4/UnrealEngine/Engine/Build/Build.version')) -if versionDetails['MinorVersion'] == 19: - - # Our required packages, extracted from Setup.sh - packages = [ - 'mono-xbuild', - 'mono-dmcs', - 'libmono-microsoft-build-tasks-v4.0-4.0-cil', - 'libmono-system-data-datasetextensions4.0-cil', - 'libmono-system-web-extensions4.0-cil', - 'libmono-system-management4.0-cil', - 'libmono-system-xml-linq4.0-cil', - 'libmono-corlib4.5-cil', - 'libmono-windowsbase4.0-cil', - 'libmono-system-io-compression4.0-cil', - 'libmono-system-io-compression-filesystem4.0-cil', - 'libmono-system-runtime4.0-cil', - 'mono-devel' - ] - - # Preinstall the packages - result = subprocess.call(['apt-get', 'install', '-y'] + packages) - if result != 0: - sys.exit(result) - - # Add the package installation commands to root_commands.sh - # (This ensures the packages will subsequently be installed by the ue4-minimal image) - with open('/home/ue4/UnrealEngine/root_commands.sh', 'a') as script: - for package in packages: - script.write('apt-get install -y {}\n'.format(package)) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 84e56d54..bf4b3d94 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -27,13 +27,6 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat WORKDIR C:\ RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine -{% if not disable_legacy_patches %} -# Install legacy toolchain components if we're building UE4.19 -# (This call is a no-op under newer Engine versions) -COPY legacy-toolchain-fix.py C:\legacy-toolchain-fix.py -RUN python C:\legacy-toolchain-fix.py -{% endif %} - # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat COPY patch-setup-win.py C:\patch-setup-win.py diff --git a/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py b/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py deleted file mode 100644 index 4edfbd21..00000000 --- a/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import json, os, subprocess, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -version = json.loads(readFile('C:\\UnrealEngine\\Engine\\Build\\Build.version')) - -# UE4.19 has problems detecting the VS2017 Build Tools and doesn't use the Windows 10 SDK -# by default, so we need to install the VS2015 Build Tools and the Windows 8.1 SDK -if version['MinorVersion'] < 20: - print('Installing VS2015 Build Tools and Windows 8.1 SDK for UE4.19 compatibility...') - sys.stdout.flush() - run = lambda cmd: subprocess.run(cmd, check=True) - installerFile = '{}\\vs_buildtools.exe'.format(os.environ['TEMP']) - run(['curl', '--progress', '-L', 'https://aka.ms/vs/16/release/vs_buildtools.exe', '--output', installerFile]) - run([ - installerFile, - '--quiet', '--wait', '--norestart', '--nocache', - '--installPath', 'C:\BuildTools', - '--channelUri', 'https://aka.ms/vs/15/release/channel', - '--installChannelUri', 'https://aka.ms/vs/15/release/channel', - '--channelId', 'VisualStudio.15.Release', - '--productId', 'Microsoft.VisualStudio.Product.BuildTools', - '--add', 'Microsoft.VisualStudio.Component.VC.140', - '--add', 'Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81' - ]) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 11b538e3..f32c1ee4 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -64,7 +64,7 @@ def addArguments(parser): ''' Registers our supported command-line arguments with the supplied argument parser ''' - parser.add_argument('release', help='UE4 release to build, in semver format (e.g. 4.19.0) or "custom" for a custom repo and branch') + parser.add_argument('release', help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch') parser.add_argument('--linux', action='store_true', help='Build Linux container images under Windows') parser.add_argument('--rebuild', action='store_true', help='Rebuild images even if they already exist') parser.add_argument('--dry-run', action='store_true', help='Print `docker build` commands instead of running them') @@ -129,7 +129,7 @@ def __init__(self, parser, argv): raise Exception() self.release = semver.format_version(ue4Version['major'], ue4Version['minor'], ue4Version['patch']) except: - raise RuntimeError('invalid UE4 release number "{}", full semver format required (e.g. "4.19.0")'.format(self.args.release)) + raise RuntimeError('invalid UE4 release number "{}", full semver format required (e.g. "4.20.0")'.format(self.args.release)) # Use the default repository and the release tag for the specified version self.repository = DEFAULT_GIT_REPO From 038e3f63338c5ef438b526e62b73fd631ecfae95 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 16:54:29 +1000 Subject: [PATCH 117/430] Fix from @mentalnote: prepend iptables rules in setup instead of appending them --- ue4docker/setup_cmd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/setup_cmd.py b/ue4docker/setup_cmd.py index 95958f8f..b8f7dac9 100644 --- a/ue4docker/setup_cmd.py +++ b/ue4docker/setup_cmd.py @@ -34,7 +34,7 @@ def _setupLinux(): print('Creating firewall rule for credential endpoint...') # Create the firewall rule - subprocess.run(['iptables', '-A', 'INPUT', '-p', 'tcp', '--dport', '9876', '-j', 'ACCEPT'], check=True) + subprocess.run(['iptables', '-I', 'INPUT', '-p', 'tcp', '--dport', '9876', '-j', 'ACCEPT'], check=True) # Ensure the firewall rule persists after reboot # (Requires the `iptables-persistent` service to be installed and running) From 891bd47588b647c90231ddeedfed802eac90c6f6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 16:57:38 +1000 Subject: [PATCH 118/430] Clean up remnants of 4.19 support that were previously missed --- README.md | 2 +- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 3 --- .../ue4-minimal/windows/exclude-components.py | 10 ---------- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/README.md b/README.md index b863b4ef..5d77c32e 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The ue4-docker Python package contains a set of Dockerfiles and accompanying bui Key features include: -- Unreal Engine 4.19.0 and newer is supported. +- Unreal Engine 4.20.0 and newer is supported. - Both Windows containers and Linux containers are supported. - Building and packaging UE4 projects is supported. - Running automation tests is supported. diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index b1da322c..613d3b11 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -46,9 +46,6 @@ RUN ue4 conan generate --profile-only # Build the Conan packages for the UE4Capture dependencies RUN ue4 conan build MediaIPC-ue4 -# Patch the problematic UE4 header files under 4.19.x (this call is a no-op under newer Engine versions) -RUN python3 /home/ue4/UE4Capture/scripts/patch-headers.py - # Enable PulseAudio support USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py index a40c49d9..d4213e9b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py @@ -30,16 +30,6 @@ def readFile(filename): os.truncate(pdbFile, 0) except: log(' Warning: failed to truncate PDB file {}.'.format(pdbFile)) - - # Under UE4.19, we need to delete the PDB files for AutomationTool entirely, since truncated files cause issues - if version['MinorVersion'] < 20: - pdbFiles = glob.glob(join(rootDir, 'Engine', 'Source', 'Programs', 'AutomationTool', '**', '*.pdb'), recursive=True) - for pdbFile in pdbFiles: - log('Removing PDB file {}...'.format(pdbFile)) - try: - os.unlink(pdbFile) - except: - log(' Warning: failed to remove PDB file {}.'.format(pdbFile)) # Determine if we are excluding the Engine's template projects and samples excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' From a0828d59fbbee8ec0b0ad6870a5fb82c5bb54e8e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 17 Feb 2021 16:59:29 +1000 Subject: [PATCH 119/430] Update license year --- LICENSE | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index e3d824d0..40b1f548 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 - 2019 Adam Rehn +Copyright (c) 2018 - 2021 Adam Rehn Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 5d77c32e..c1496743 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/maste ## Legal -Copyright © 2018 - 2019, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. +Copyright © 2018 - 2021, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. From 733bcf5ae84c269b004835a306ba2dbccef33438 Mon Sep 17 00:00:00 2001 From: Florian Karydes Date: Wed, 17 Feb 2021 16:28:16 +0100 Subject: [PATCH 120/430] Upgrade pip in linux full image --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 613d3b11..c4ec39fa 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -9,7 +9,7 @@ ARG CONAN_UE4CLI_VERSION # Install ue4cli and conan-ue4cli USER root -RUN pip3 install setuptools wheel +RUN pip3 install --upgrade pip setuptools wheel RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" USER ue4 @@ -28,7 +28,7 @@ RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture # Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake -RUN pip3 install setuptools wheel +RUN pip3 install --upgrade pip setuptools wheel RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers USER ue4 From 706b373d1302fefa59b61854eaadf6afe34eb342 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 19 Feb 2021 11:40:15 +1000 Subject: [PATCH 121/430] Add a bunch of options to control patching behaviour in Dockerfiles --- .../ue4-build-prerequisites/linux/Dockerfile | 5 +++-- .../ue4-build-prerequisites/windows/Dockerfile | 4 ++++ ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 12 ++++++++++++ ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 10 ++++++---- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 10 ++++++++++ ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 10 ++++++++++ ue4docker/infrastructure/ImageBuilder.py | 6 ++++++ 7 files changed, 51 insertions(+), 6 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 49d70cdc..9da77611 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -1,15 +1,16 @@ ARG BASEIMAGE FROM ${BASEIMAGE} +{% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} # Disable interactive prompts during package installation ENV DEBIAN_FRONTEND=noninteractive # Install our build prerequisites -RUN \ - apt-get update && apt-get install -y --no-install-recommends \ +RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ ca-certificates \ curl \ diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 5e105dfd..33c85290 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -3,8 +3,10 @@ ARG BASEIMAGE FROM ${BASEIMAGE} AS dlls SHELL ["cmd", "/S", "/C"] +{% if not disable_labels %} # Include our sentinel so `ue4-docker clean` can find this intermediate image LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} # Create a directory in which to gather the DLL files we need RUN mkdir C:\GatheredDlls @@ -46,8 +48,10 @@ FROM ${BASEIMAGE} SHELL ["cmd", "/S", "/C"] COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ +{% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} # Enable long path support RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index b2a95008..b71e7952 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -10,13 +10,17 @@ RUN rm -rf /home/ue4/UnrealEngine/.git COPY set-changelist.py /tmp/set-changelist.py RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version +{% if (not disable_all_patches) and (not disable_opengl_patch) %} # Enable the OpenGL RHI for Engine versions where it is present but deprecated COPY enable-opengl.py /tmp/enable-opengl.py RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngine.ini +{% endif %} +{% if (not disable_all_patches) and (not disable_buildgraph_patches) %} # Patch the default settings in InstalledEngineBuild.xml and increase the output verbosity of the DDC generation step COPY patch-build-graph.py /tmp/patch-build-graph.py RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version +{% endif %} # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, @@ -34,16 +38,22 @@ RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC && \ python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES +{% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini COPY fix-targets.py /tmp/fix-targets.py RUN python3 /tmp/fix-targets.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Config/BaseEngine.ini +{% endif %} +{% if (not disable_all_patches) and (not disable_unrealpak_copy) %} # Some versions of the Engine fail to include UnrealPak in the Installed Build, so copy it manually RUN cp ./Engine/Binaries/Linux/UnrealPak ./LocalBuilds/Engine/Linux/Engine/Binaries/Linux/UnrealPak +{% endif %} +{% if (not disable_all_patches) and (not disable_toolchain_copy) %} # Ensure the bundled toolchain included in 4.20.0 and newer is copied to the Installed Build COPY --chown=ue4:ue4 copy-toolchain.py /tmp/copy-toolchain.py RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine +{% endif %} # Copy the Installed Build into a clean image, discarding the source build FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} @@ -52,9 +62,11 @@ FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine WORKDIR /home/ue4/UnrealEngine +{% if not disable_labels %} # Add labels to the built image to identify which components (if any) were excluded from the build that it contains # (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} +{% endif %} diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 3534a32b..b25633b3 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -11,6 +11,7 @@ RUN del /s /q C:\UnrealEngine\.git COPY set-changelist.py C:\set-changelist.py RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version +{% if (not disable_all_patches) and (not disable_buildgraph_patches) %} # Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 COPY patch-filters-xml.py C:\patch-filters-xml.py RUN python C:\patch-filters-xml.py C:\UnrealEngine\Engine\Build\InstalledEngineFilters.xml @@ -18,6 +19,7 @@ RUN python C:\patch-filters-xml.py C:\UnrealEngine\Engine\Build\InstalledEngineF # Patch out problematic entries in InstalledEngineBuild.xml introduced in UE4.23.0 COPY patch-build-graph.py C:\patch-build-graph.py RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineBuild.xml +{% endif %} # Create an Installed Build of the Engine # (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) @@ -30,22 +32,22 @@ RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Buil del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL && ` python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% +{% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini COPY fix-targets.py C:\fix-targets.py RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\Config\BaseEngine.ini - -# Copy our legacy toolchain installation script into the Installed Build -# (This ensures we only need one COPY directive below) -RUN C:\copy.py C:\legacy-toolchain-fix.py C:\UnrealEngine\LocalBuilds\Engine\Windows\ +{% endif %} # Copy the Installed Build into a clean image, discarding the source tree FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine WORKDIR C:\UnrealEngine +{% if not disable_labels %} # Add labels to the built image to identify which components (if any) were excluded from the build that it contains # (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} +{% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index fa9e4aaf..5685c38d 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -1,8 +1,10 @@ ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% if not disable_all_patches %} # Enable verbose output for steps that patch files? ARG VERBOSE_OUTPUT=0 +{% endif %} # The git repository that we will clone ARG GIT_REPO="" @@ -42,10 +44,12 @@ RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngi {% endif %} +{% if (not disable_all_patches) and (not disable_release_patches) %} # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT +{% endif %} # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` WORKDIR /home/ue4/UnrealEngine @@ -53,15 +57,21 @@ RUN sudo apt-get update && \ ./Setup.sh && \ sudo rm -rf /var/lib/apt/lists/* +{% if (not disable_all_patches) and (not disable_linker_fixup) %} # The linker bundled with UE4.20.0 onwards chokes on system libraries built with newer compilers, # so redirect the bundled clang to use the system linker instead COPY --chown=ue4:ue4 linker-fixup.py /tmp/linker-fixup.py RUN python3 /tmp/linker-fixup.py /home/ue4/UnrealEngine/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64 `which ld` +{% endif %} +{% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX +{% endif %} +{% if (not disable_all_patches) and (not disable_ubt_patches) %} # Apply our bugfix patches to UnrealBuildTool (UBT) COPY --chown=ue4:ue4 patch-ubt.py /tmp/patch-ubt.py RUN python3 /tmp/patch-ubt.py /home/ue4/UnrealEngine/Engine/Source/Programs/UnrealBuildTool +{% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index bf4b3d94..e1a5baae 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -2,8 +2,10 @@ ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% if not disable_all_patches %} # Enable verbose output for steps that patch files? ARG VERBOSE_OUTPUT=0 +{% endif %} # The git repository that we will clone ARG GIT_REPO="" @@ -27,24 +29,32 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat WORKDIR C:\ RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine +{% if (not disable_all_patches) and (not disable_prerequisites_patch) %} # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat COPY patch-setup-win.py C:\patch-setup-win.py RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat %VERBOSE_OUTPUT% +{% endif %} +{% if (not disable_all_patches) and (not disable_release_patches) %} # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) COPY patch-broken-releases.py C:\patch-broken-releases.py RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% +{% endif %} # Run post-clone setup steps WORKDIR C:\UnrealEngine RUN Setup.bat +{% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 +{% endif %} +{% if (not disable_all_patches) and (not disable_ubt_patches) %} # Apply our bugfix patches to UnrealBuildTool (UBT) COPY patch-ubt.py C:\patch-ubt.py RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildTool +{% endif %} diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 425c9b1c..e5e1ac89 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -28,6 +28,12 @@ def build(self, name, tags, args, secrets=None): dockerfile = os.path.join(self.context(name), 'Dockerfile') templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) rendered = templateInstance.render(self.templateContext) + + # Compress excess whitespace introduced during Jinja rendering and save the contents back to disk + # (Ensure that we still have a single trailing newline at the end of the Dockerfile) + while '\n\n\n' in rendered: + rendered = rendered.replace('\n\n\n', '\n\n') + rendered = rendered.rstrip('\n') + '\n' FilesystemUtils.writeFile(dockerfile, rendered) # Inject our filesystem layer commit message after each RUN directive in the Dockerfile From 521567cb759f367176f387b941b0550a53ca6b1a Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Sat, 20 Feb 2021 23:41:24 +1000 Subject: [PATCH 122/430] Allow option values to be JSON objects or arrays --- ue4docker/build.py | 4 ++-- ue4docker/infrastructure/BuildConfiguration.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index ea42a6cf..20c8f53f 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -1,4 +1,4 @@ -import argparse, getpass, humanfriendly, os, shutil, sys, tempfile, time +import argparse, getpass, humanfriendly, json, os, shutil, sys, tempfile, time from .infrastructure import * from os.path import join @@ -102,7 +102,7 @@ def build(): if len(config.opts) > 0: logger.info('ADVANCED CONFIGURATION OPTIONS:', False) for key, value in config.opts.items(): - logger.info('{}: {}'.format(key, value), False) + logger.info('{}: {}'.format(key, json.dumps(value)), False) print('', file=sys.stderr, flush=True) # Determine if we are building Windows or Linux containers diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index f32c1ee4..3ceaad62 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -1,7 +1,7 @@ from .DockerUtils import DockerUtils from .PackageUtils import PackageUtils from .WindowsUtils import WindowsUtils -import humanfriendly, os, platform, random +import humanfriendly, json, os, platform, random from pkg_resources import parse_version # Import the `semver` package even when the conflicting `node-semver` package is present @@ -298,4 +298,12 @@ def _processTemplateValue(self, value): elif value.lower() in ['false', '0']: return False + # If the value is a JSON object or array then attempt to parse it + if (value.startswith('{') and value.endswith('}')) or (value.startswith('[') and value.endswith(']')): + try: + return json.loads(value) + except: + print('Warning: could not parse option value "{}" as JSON, treating value as a string'.format(value)) + + # Treat all other values as strings return value From 495ef709ceb469e43cd233d19838bb13dd2eaf1d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 22 Feb 2021 10:48:30 +1000 Subject: [PATCH 123/430] Use string for credential mode instead of bool and add source mode option --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 12 +++++++++++- .../dockerfiles/ue4-source/windows/Dockerfile | 10 ++++++++++ ue4docker/infrastructure/BuildConfiguration.py | 14 ++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 5685c38d..596af05f 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -6,13 +6,21 @@ FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} ARG VERBOSE_OUTPUT=0 {% endif %} +{% if source_mode == "copy" %} + +# Copy the Unreal Engine source code from the host system +ARG SOURCE_LOCATION +COPY ${SOURCE_LOCATION} /home/ue4/UnrealEngine + +{% else %} + # The git repository that we will clone ARG GIT_REPO="" # The git branch/tag that we will checkout ARG GIT_BRANCH="" -{% if use_build_secrets %} +{% if credential_mode == "secrets" %} # Install our git credential helper that retrieves credentials from build secrets COPY --chown=ue4:ue4 git-credential-helper-secrets.sh /tmp/git-credential-helper-secrets.sh @@ -44,6 +52,8 @@ RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngi {% endif %} +{% endif %} + {% if (not disable_all_patches) and (not disable_release_patches) %} # Apply our bugfix patches to broken Engine releases such as 4.25.4 # (Make sure we do this before the post-clone setup steps are run) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index e1a5baae..741e87de 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -7,6 +7,14 @@ FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} ARG VERBOSE_OUTPUT=0 {% endif %} +{% if source_mode == "copy" %} + +# Copy the Unreal Engine source code from the host system +ARG SOURCE_LOCATION +COPY ${SOURCE_LOCATION} C:\UnrealEngine + +{% else %} + # The git repository that we will clone ARG GIT_REPO="" @@ -29,6 +37,8 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat WORKDIR C:\ RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine +{% endif %} + {% if (not disable_all_patches) and (not disable_prerequisites_patch) %} # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 3ceaad62..dfd3dcec 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -166,6 +166,20 @@ def __init__(self, parser, argv): else: self.opts[o.replace('-', '_')] = True + # If the user requested an option that is only compatible with generated Dockerfiles then ensure `-layout` was specified + if self.layoutDir is None and self.opts.get('source_mode', 'git') != 'git': + raise RuntimeError('the `-layout` flag must be used when specifying a non-default value for the `source_mode` option') + + # Verify that the value for `source_mode` is valid if specified + validSourceModes = ['git', 'copy'] + if self.opts.get('source_mode', 'git') not in validSourceModes: + raise RuntimeError('invalid value specified for the `source_mode` option, valid values are {}'.format(validSourceModes)) + + # Verify that the value for `credential_mode` is valid if specified + validCredentialModes = ['endpoint', 'secrets'] if self.containerPlatform == 'linux' else ['endpoint'] + if self.opts.get('credential_mode', 'endpoint') not in validCredentialModes: + raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title())) + # Generate our flags for keeping or excluding components self.exclusionFlags = [ From bcf3cf03c1b7a740ec82b1fe3940113a64982d36 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 22 Feb 2021 12:21:55 +1000 Subject: [PATCH 124/430] Bump version to 0.0.78 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 953095ee..ca0a6c78 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.77' +__version__ = '0.0.78' From 82c636aa9ba8e47120493032d78105a0eff27c64 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 22 Feb 2021 13:36:21 +1000 Subject: [PATCH 125/430] Better name for the flag to disable Windows Setup.bat patches --- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 741e87de..e3505782 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -39,7 +39,7 @@ RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine {% endif %} -{% if (not disable_all_patches) and (not disable_prerequisites_patch) %} +{% if (not disable_all_patches) and (not disable_windows_setup_patch) %} # Since the UE4 prerequisites installer appears to break when newer versions # of the VC++ runtime are present, patch out the prereqs call in Setup.bat COPY patch-setup-win.py C:\patch-setup-win.py From ab80abec3454d2e270521afdc2a366f2a8a85758 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 22 Feb 2021 13:43:51 +1000 Subject: [PATCH 126/430] Bump version to 0.0.79 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index ca0a6c78..8396f55d 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.78' +__version__ = '0.0.79' From a577e5e08ce063b173cf4320b49c035296d45b27 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Mar 2021 12:07:18 +1000 Subject: [PATCH 127/430] Add option to combine generated Dockerfiles --- ue4docker/build.py | 2 +- .../ue4-build-prerequisites/linux/Dockerfile | 2 +- .../windows/Dockerfile | 2 +- .../dockerfiles/ue4-engine/linux/Dockerfile | 4 ++ .../dockerfiles/ue4-engine/windows/Dockerfile | 4 ++ .../dockerfiles/ue4-full/linux/Dockerfile | 18 +++++-- .../dockerfiles/ue4-full/windows/Dockerfile | 18 +++++-- .../dockerfiles/ue4-minimal/linux/Dockerfile | 8 +++ .../ue4-minimal/windows/Dockerfile | 8 +++ .../dockerfiles/ue4-source/linux/Dockerfile | 4 ++ .../dockerfiles/ue4-source/windows/Dockerfile | 4 ++ .../infrastructure/BuildConfiguration.py | 12 +++++ ue4docker/infrastructure/ImageBuilder.py | 53 +++++++++++++++---- 13 files changed, 116 insertions(+), 23 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 20c8f53f..1738d04c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -72,7 +72,7 @@ def build(): shutil.copytree(contextOrig, contextRoot) # Create the builder instance to build the Docker images - builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir, config.opts) + builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir, config.opts, config.combine) # Resolve our main set of tags for the generated images mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 9da77611..068803bd 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -1,5 +1,5 @@ ARG BASEIMAGE -FROM ${BASEIMAGE} +FROM ${BASEIMAGE} as prerequisites {% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 33c85290..33440b45 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -44,7 +44,7 @@ RUN choco install -y windows-sdk-10-version-1809-windbg RUN C:\copy.py "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\pdbcopy.exe" C:\GatheredDlls\ # Copy our gathered DLLs (and pdbcopy.exe) into a clean image to reduce image size -FROM ${BASEIMAGE} +FROM ${BASEIMAGE} as prerequisites SHELL ["cmd", "/S", "/C"] COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ diff --git a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile b/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile index fe1f894b..3d4caa2d 100644 --- a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile @@ -1,7 +1,11 @@ +{% if combine %} +FROM source as engine +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} +{% endif %} # Build UBT and build the Engine RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -WaitMutex diff --git a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile b/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile index 0c1830c3..8d2bc70d 100644 --- a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile @@ -1,8 +1,12 @@ # escape=` +{% if combine %} +FROM source as engine +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} +{% endif %} # Build UBT and build the Engine RUN GenerateProjectFiles.bat diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index c4ec39fa..7cfd4588 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -1,9 +1,13 @@ +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +{% if combine %} +FROM source as conan +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG -ARG UE4CLI_VERSION="ue4cli>=0.0.45" -ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan +{% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION @@ -18,7 +22,11 @@ RUN ue4 setroot /home/ue4/UnrealEngine RUN ue4 conan generate # Copy the generated Conan packages into a new image with our Installed Build +{% if combine %} +FROM minimal as full +{% else %} FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +{% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION @@ -36,8 +44,8 @@ USER ue4 # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) ENV UE4CLI_CONFIG_DIR /home/ue4/.config/ue4cli -# Copy the Conan configuration settings and package cache from the builder image -COPY --from=builder --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan +# Copy the Conan configuration settings and package cache from the previous build stage +COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan # Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) RUN ue4 setroot /home/ue4/UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index 798d4995..ceab3634 100644 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -1,10 +1,14 @@ # escape=` +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +{% if combine %} +FROM source as conan +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG -ARG UE4CLI_VERSION="ue4cli>=0.0.45" -ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan +{% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION @@ -19,7 +23,11 @@ RUN ue4 setroot C:\UnrealEngine RUN ue4 conan generate && ue4 conan generate --remove-only # Copy the generated Conan packages into a new image with our Installed Build +{% if combine %} +FROM minimal as full +{% else %} FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +{% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION @@ -31,8 +39,8 @@ RUN pip install "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue4-ci-helpers --no- # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) ENV UE4CLI_CONFIG_DIR C:\Users\ContainerAdministrator\AppData\Roaming\ue4cli -# Copy the Conan configuration settings and package cache from the builder image -COPY --from=builder C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan +# Copy the Conan configuration settings and package cache from the previous build stage +COPY --from=conan C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan # Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) RUN ue4 setroot C:\UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index b71e7952..907a2cff 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -1,7 +1,11 @@ +{% if combine %} +FROM source as builder +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +{% endif %} # Remove the .git directory to disable UBT `git status` calls and speed up the build process RUN rm -rf /home/ue4/UnrealEngine/.git @@ -56,7 +60,11 @@ RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine {% endif %} # Copy the Installed Build into a clean image, discarding the source build +{% if combine %} +FROM prerequisites as minimal +{% else %} FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} # Copy the Installed Build files from the builder image COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index b25633b3..94e7242b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -1,8 +1,12 @@ # escape=` +{% if combine %} +FROM source as builder +{% else %} ARG NAMESPACE ARG TAG ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +{% endif %} # Remove the .git directory to disable UBT `git status` calls and speed up the build process RUN del /s /q C:\UnrealEngine\.git @@ -39,7 +43,11 @@ RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\C {% endif %} # Copy the Installed Build into a clean image, discarding the source tree +{% if combine %} +FROM prerequisites as minimal +{% else %} FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine WORKDIR C:\UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 596af05f..7d4972e8 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -1,5 +1,9 @@ +{% if combine %} +FROM prerequisites as source +{% else %} ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} {% if not disable_all_patches %} # Enable verbose output for steps that patch files? diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index e3505782..e1c251d8 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -1,6 +1,10 @@ # escape=` +{% if combine %} +FROM prerequisites as source +{% else %} ARG PREREQS_TAG FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} {% if not disable_all_patches %} # Enable verbose output for steps that patch files? diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index dfd3dcec..597dcdb7 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -89,6 +89,7 @@ def addArguments(parser): parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image') parser.add_argument('-conan-ue4cli', default=None, help='Override the default version of conan-ue4cli installed in the ue4-full image') parser.add_argument('-layout', default=None, help='Copy generated Dockerfiles to the specified directory and don\'t build the images') + parser.add_argument('--combine', action='store_true', help='Combine generated Dockerfiles into a single multi-stage build Dockerfile') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') @@ -152,6 +153,7 @@ def __init__(self, parser, argv): self.ignoreBlacklist = self.args.ignore_blacklist self.verbose = self.args.verbose self.layoutDir = self.args.layout + self.combine = self.args.combine # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) @@ -166,9 +168,19 @@ def __init__(self, parser, argv): else: self.opts[o.replace('-', '_')] = True + # If we are generating Dockerfiles then generate them for all images that have not been explicitly excluded + if self.layoutDir is not None: + self.rebuild = True + + # If we are generating Dockerfiles and combining them then set the corresponding Jinja context value + if self.layoutDir is not None and self.combine == True: + self.opts['combine'] = True + # If the user requested an option that is only compatible with generated Dockerfiles then ensure `-layout` was specified if self.layoutDir is None and self.opts.get('source_mode', 'git') != 'git': raise RuntimeError('the `-layout` flag must be used when specifying a non-default value for the `source_mode` option') + if self.layoutDir is None and self.combine == True: + raise RuntimeError('the `-layout` flag must be used when specifying the `--combine` flag') # Verify that the value for `source_mode` is valid if specified validSourceModes = ['git', 'copy'] diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index e5e1ac89..bb36764e 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,12 +1,13 @@ from .DockerUtils import DockerUtils from .FilesystemUtils import FilesystemUtils from .GlobalConfiguration import GlobalConfiguration -import humanfriendly, os, shutil, subprocess, tempfile, time +import glob, humanfriendly, os, shutil, subprocess, tempfile, time +from os.path import basename, exists, join from jinja2 import Environment, Template class ImageBuilder(object): - def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None, templateContext=None): + def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None, templateContext=None, combine=False): ''' Creates an ImageBuilder for the specified build parameters ''' @@ -17,6 +18,7 @@ def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDi self.dryRun = dryRun self.layoutDir = layoutDir self.templateContext = templateContext if templateContext is not None else {} + self.combine = combine def build(self, name, tags, args, secrets=None): ''' @@ -25,7 +27,7 @@ def build(self, name, tags, args, secrets=None): # Create a Jinja template environment and render the Dockerfile template environment = Environment(autoescape=False, trim_blocks=True, lstrip_blocks=True) - dockerfile = os.path.join(self.context(name), 'Dockerfile') + dockerfile = join(self.context(name), 'Dockerfile') templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) rendered = templateInstance.render(self.templateContext) @@ -33,7 +35,7 @@ def build(self, name, tags, args, secrets=None): # (Ensure that we still have a single trailing newline at the end of the Dockerfile) while '\n\n\n' in rendered: rendered = rendered.replace('\n\n\n', '\n\n') - rendered = rendered.rstrip('\n') + '\n' + rendered = rendered.strip('\n') + '\n' FilesystemUtils.writeFile(dockerfile, rendered) # Inject our filesystem layer commit message after each RUN directive in the Dockerfile @@ -56,7 +58,7 @@ def build(self, name, tags, args, secrets=None): # Create temporary files to store the contents of each of our secrets secretFlags = [] for secret, contents in secrets.items(): - secretFile = os.path.join(tempDir, secret) + secretFile = join(tempDir, secret) FilesystemUtils.writeFile(secretFile, contents) secretFlags.append('id={},src={}'.format(secret, secretFile)) @@ -76,7 +78,7 @@ def context(self, name): ''' Resolve the full path to the build context for the specified image ''' - return os.path.join(self.root, os.path.basename(name), self.platform) + return join(self.root, basename(name), self.platform) def pull(self, image): ''' @@ -128,11 +130,42 @@ def _processImage(self, image, name, command, actionPresentTense, actionPastTens # Determine if we're just copying the Dockerfile to an output directory if self.layoutDir is not None: + + # Determine whether we're performing a simple copy or combining generated Dockerfiles source = self.context(name) - dest = os.path.join(self.layoutDir, os.path.basename(name)) - self.logger.action('Copying "{}" to "{}"...'.format(source, dest), newline=False) - shutil.copytree(source, dest) - self.logger.action('Copied Dockerfile for image "{}".'.format(image), newline=False) + if self.combine == True: + + # Ensure the destination directory exists + dest = join(self.layoutDir, 'combined') + self.logger.action('Merging "{}" into "{}"...'.format(source, dest), newline=False) + os.makedirs(dest, exist_ok=True) + + # Merge the source Dockerfile with any existing Dockerfile contents in the destination directory + # (Insert a single newline between merged file contents and ensure we have a single trailing newline) + sourceDockerfile = join(source, 'Dockerfile') + destDockerfile = join(dest, 'Dockerfile') + dockerfileContents = FilesystemUtils.readFile(destDockerfile) if exists(destDockerfile) else '' + dockerfileContents = dockerfileContents + '\n' + FilesystemUtils.readFile(sourceDockerfile) + dockerfileContents = dockerfileContents.strip('\n') + '\n' + FilesystemUtils.writeFile(destDockerfile, dockerfileContents) + + # Copy any supplemental files from the source directory to the destination directory + # (Exclude any extraneous files which are not referenced in the Dockerfile contents) + for file in glob.glob(join(source, '*.*')): + if basename(file) in dockerfileContents: + shutil.copy(file, join(dest, basename(file))) + + # Report our success + self.logger.action('Merged Dockerfile for image "{}".'.format(image), newline=False) + + else: + + # Copy the source directory to the destination + dest = join(self.layoutDir, basename(name)) + self.logger.action('Copying "{}" to "{}"...'.format(source, dest), newline=False) + shutil.copytree(source, dest) + self.logger.action('Copied Dockerfile for image "{}".'.format(image), newline=False) + return # Attempt to process the image using the supplied command From 24f8ba2598eaf2b3edf59b72f50389e61cc9c8f2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Mar 2021 14:03:37 +1000 Subject: [PATCH 128/430] Add option to place component exclusion in a separate step to improve caching at the expense of disk space usage --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 6 ++++-- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 907a2cff..22580797 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -39,8 +39,10 @@ ARG EXCLUDE_TEMPLATES WORKDIR /home/ue4/UnrealEngine COPY exclude-components.py /tmp/exclude-components.py RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC && \ - rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC && \ - python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES + rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC{%+ if not separate_exclude_step %} && \ + {%+ else %} + +RUN {%+ endif %}python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 94e7242b..f06686a8 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -33,8 +33,10 @@ ARG EXCLUDE_TEMPLATES WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% && ` - del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL && ` - python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% + del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL{%+ if not separate_exclude_step %} && ` + {%+ else %} + +RUN {%+ endif %}python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini From a17dbfb3c9cdf95c11884d5d2d85c539db5c08fd Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Mar 2021 14:50:04 +1000 Subject: [PATCH 129/430] Add template option to specify additional BuildGraph args --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 2 +- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 22580797..576341f5 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -38,7 +38,7 @@ ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES WORKDIR /home/ue4/UnrealEngine COPY exclude-components.py /tmp/exclude-components.py -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC && \ +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC{%+ if not separate_exclude_step %} && \ {%+ else %} diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index f06686a8..87236743 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -32,7 +32,7 @@ ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL{%+ if not separate_exclude_step %} && ` {%+ else %} From 3c3019970b5df6251dd06e7e7f833ceb7b9bde8c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Mar 2021 16:28:49 +1000 Subject: [PATCH 130/430] Always split component removal --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 11 +++++------ ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 576341f5..027db196 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -32,17 +32,16 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -Clean # Create an Installed Build of the Engine -# (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) ARG BUILD_DDC -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES WORKDIR /home/ue4/UnrealEngine COPY exclude-components.py /tmp/exclude-components.py RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC {{ buildgraph_args }} && \ - rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC{%+ if not separate_exclude_step %} && \ - {%+ else %} + rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC -RUN {%+ endif %}python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES +# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size +ARG EXCLUDE_DEBUG +ARG EXCLUDE_TEMPLATES +RUN python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 87236743..9f8ef8f4 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -26,17 +26,16 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB {% endif %} # Create an Installed Build of the Engine -# (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) ARG BUILD_DDC -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` - del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL{%+ if not separate_exclude_step %} && ` - {%+ else %} + del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL -RUN {%+ endif %}python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% +# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size +ARG EXCLUDE_DEBUG +ARG EXCLUDE_TEMPLATES +RUN python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini From 416e63590dea4084d1c971f8056421e7d9f321bf Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Mar 2021 16:52:54 +1000 Subject: [PATCH 131/430] Report generation options for generated Dockerfiles --- ue4docker/build.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 1738d04c..978267fe 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -1,5 +1,6 @@ import argparse, getpass, humanfriendly, json, os, shutil, sys, tempfile, time from .infrastructure import * +from .version import __version__ from os.path import join def _getCredential(args, name, envVar, promptFunc): @@ -101,7 +102,7 @@ def build(): # Report any advanced configuration options that were specified if len(config.opts) > 0: logger.info('ADVANCED CONFIGURATION OPTIONS:', False) - for key, value in config.opts.items(): + for key, value in sorted(config.opts.items()): logger.info('{}: {}'.format(key, json.dumps(value)), False) print('', file=sys.stderr, flush=True) @@ -294,6 +295,35 @@ def build(): else: logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') + # If we are generating Dockerfiles then include information about the options used to generate them + if config.layoutDir is not None: + + # Determine whether we generated a single combined Dockerfile or a set of Dockerfiles + if config.combine == True: + + # Generate a comment to place at the top of the single combined Dockerfile + lines = ['This file was generated by ue4-docker version {} with the following options:'.format(__version__), ''] + lines.extend(['- {}: {}'.format(key, json.dumps(value)) for key, value in sorted(config.opts.items())]) + comment = '\n'.join(['# {}'.format(line) for line in lines]) + + # Inject the comment at the top of the Dockerfile, being sure to place it after any `escape` parser directive + dockerfile = join(config.layoutDir, 'combined', 'Dockerfile') + dockerfileContents = FilesystemUtils.readFile(dockerfile) + if dockerfileContents.startswith('# escape'): + newline = dockerfileContents.index('\n') + dockerfileContents = dockerfileContents[0:newline+1] + '\n' + comment + '\n\n' + dockerfileContents[newline+1:] + else: + dockerfileContents = comment + '\n\n' + dockerfileContents + FilesystemUtils.writeFile(dockerfile, dockerfileContents) + + else: + + # Create a JSON file to accompany the set of generated Dockerfiles + FilesystemUtils.writeFile(join(config.layoutDir, 'generated.json'), json.dumps({ + 'version': __version__, + 'opts': config.opts + }, indent=4, sort_keys=True)) + # Report the total execution time endTime = time.time() logger.action('Total execution time: {}'.format(humanfriendly.format_timespan(endTime - startTime))) From 12ca335c6d40c13446878025b972c18d1808c44d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 2 Mar 2021 12:06:16 +0300 Subject: [PATCH 132/430] Disable GitDependencies.exe caching to .git/ue4-gitdeps This commit reduces amount of data written to disk during ue4-source image build by ~9GB. --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 3 ++- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 7d4972e8..cfd6ffc2 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -67,8 +67,9 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` WORKDIR /home/ue4/UnrealEngine +# -no-cache disables GitDependencies.exe saving a copy of all dependencies in .git/ue4-gitdeps (around 9GB) RUN sudo apt-get update && \ - ./Setup.sh && \ + ./Setup.sh -no-cache && \ sudo rm -rf /var/lib/apt/lists/* {% if (not disable_all_patches) and (not disable_linker_fixup) %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index e1c251d8..8fe94aa1 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -59,7 +59,8 @@ RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% # Run post-clone setup steps WORKDIR C:\UnrealEngine -RUN Setup.bat +# -no-cache disables GitDependencies.exe saving a copy of all dependencies in .git/ue4-gitdeps (around 9GB) +RUN Setup.bat -no-cache {% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards From d6724075732e12b5b08f4b4c649c84ec1a57b2da Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 4 Mar 2021 15:03:03 +1000 Subject: [PATCH 133/430] Bump version to 0.0.80 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8396f55d..a7f66607 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.79' +__version__ = '0.0.80' From b42ae1d8206badb2d6115a8b256f60a3678c4f77 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 4 Mar 2021 12:44:10 +0300 Subject: [PATCH 134/430] Require semver 2.x.x If there's no semver python module installed and user does `python setup.py develop --user` in ue4-docker repo in order to work on ue4-docker, then semver 3.x.x is pulled in. 1. Semver [website](https://github.com/python-semver/python-semver) clearly says that semver 3.x.x is not for production use until 3.0.0 is released 2. ue4-docker fails to import semver.py because there is no such file in semver 3.x.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e7dad001..b36d5f26 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ 'packaging>=19.1', 'psutil', 'requests', - 'semver>=2.7.9', + 'semver>=2.7.9,<3.0.0', 'setuptools>=38.6.0', 'termcolor', 'twine>=1.11.0', From 2387b8fdddf01cdce1fe31248e5718f3e5d70bc9 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 8 Mar 2021 14:03:51 +1000 Subject: [PATCH 135/430] Include image list in generated Dockerfile output --- ue4docker/build.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 978267fe..fa6d4b14 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -232,17 +232,21 @@ def build(): shutil.rmtree(config.layoutDir) os.makedirs(config.layoutDir) + # Keep track of the images we've built + builtImages = [] + # Compute the build options for the UE4 build prerequisites image # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] if config.containerPlatform == 'windows': prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_VERSION=' + WindowsUtils.getWindowsBuild()] - # Build or pull the UE4 build prerequisites image - if config.pullPrerequisites == True: + # Build or pull the UE4 build prerequisites image (don't pull it if we're copying Dockerfiles to an output directory) + if config.layoutDir is None and config.pullPrerequisites == True: builder.pull('adamrehn/ue4-build-prerequisites:{}'.format(config.prereqsTag)) else: builder.build('adamrehn/ue4-build-prerequisites', [config.prereqsTag], config.platformArgs + prereqsArgs) + builtImages.append('ue4-build-prerequisites') # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets secrets = {} @@ -261,6 +265,7 @@ def build(): '--build-arg', 'VERBOSE_OUTPUT={}'.format('1' if config.verbose == True else '0') ] builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + credentialArgs, secrets) + builtImages.append('ue4-source') # Build the UE4 Engine source build image, unless requested otherwise by the user ue4BuildArgs = prereqConsumerArgs + [ @@ -269,6 +274,7 @@ def build(): ] if config.noEngine == False: builder.build('ue4-engine', mainTags, config.platformArgs + ue4BuildArgs) + builtImages.append('ue4-engine') else: logger.info('User specified `--no-engine`, skipping ue4-engine image build.') @@ -276,6 +282,7 @@ def build(): buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs) + builtImages.append('ue4-minimal') else: logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') @@ -292,6 +299,7 @@ def build(): # Build the image builder.build('ue4-full', mainTags, config.platformArgs + ue4BuildArgs + infrastructureFlags) + builtImages.append('ue4-full') else: logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') @@ -304,6 +312,8 @@ def build(): # Generate a comment to place at the top of the single combined Dockerfile lines = ['This file was generated by ue4-docker version {} with the following options:'.format(__version__), ''] lines.extend(['- {}: {}'.format(key, json.dumps(value)) for key, value in sorted(config.opts.items())]) + lines.extend(['', 'This Dockerfile combines the steps for the following images:', '']) + lines.extend(['- {}'.format(image) for image in builtImages]) comment = '\n'.join(['# {}'.format(line) for line in lines]) # Inject the comment at the top of the Dockerfile, being sure to place it after any `escape` parser directive @@ -321,6 +331,7 @@ def build(): # Create a JSON file to accompany the set of generated Dockerfiles FilesystemUtils.writeFile(join(config.layoutDir, 'generated.json'), json.dumps({ 'version': __version__, + 'images': builtImages, 'opts': config.opts }, indent=4, sort_keys=True)) From 8aef350005200a25292c34a8f93ecbc96bb25edc Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 8 Mar 2021 14:07:04 +1000 Subject: [PATCH 136/430] Enable Git LFS under Linux to match Windows --- .../dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 068803bd..a0f894ad 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ git \ + git-lfs \ python3 \ python3-dev \ python3-pip \ @@ -34,3 +35,6 @@ RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ passwd -d ue4 && \ usermod -a -G audio,video,sudo ue4 USER ue4 + +# Enable Git Large File Storage (LFS) support +RUN git lfs install From 865b2aafd233a3afbbda47075107f5dc6550a18c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 9 Mar 2021 16:54:24 +1000 Subject: [PATCH 137/430] Tidy up comments describing the -no-cache Setup flag --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 2 +- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index cfd6ffc2..41c531dd 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -66,8 +66,8 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT {% endif %} # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` +# (Note that the `-no-cache` flag disables caching of dependency data in `.git/ue4-gitdeps`, saving disk space) WORKDIR /home/ue4/UnrealEngine -# -no-cache disables GitDependencies.exe saving a copy of all dependencies in .git/ue4-gitdeps (around 9GB) RUN sudo apt-get update && \ ./Setup.sh -no-cache && \ sudo rm -rf /var/lib/apt/lists/* diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 8fe94aa1..3264f43f 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -58,8 +58,8 @@ RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% {% endif %} # Run post-clone setup steps +# (Note that the `-no-cache` flag disables caching of dependency data in `.git/ue4-gitdeps`, saving disk space) WORKDIR C:\UnrealEngine -# -no-cache disables GitDependencies.exe saving a copy of all dependencies in .git/ue4-gitdeps (around 9GB) RUN Setup.bat -no-cache {% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} From 9cf1375c33098a2787b5f514fd0ff36167a12b96 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 9 Mar 2021 13:34:51 +0300 Subject: [PATCH 138/430] Fix dockerfiles to actually remove unneeded directories on Windows It turns out that del /s is actually not deleting directory recursively --- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 9f8ef8f4..2dfc9053 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -9,7 +9,7 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder {% endif %} # Remove the .git directory to disable UBT `git status` calls and speed up the build process -RUN del /s /q C:\UnrealEngine\.git +RUN rmdir /s /q C:\UnrealEngine\.git # Set the changelist number in Build.version to ensure our Build ID is generated correctly COPY set-changelist.py C:\set-changelist.py @@ -30,7 +30,7 @@ ARG BUILD_DDC WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` - del /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL + rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size ARG EXCLUDE_DEBUG diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 3264f43f..27661ad9 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -65,7 +65,7 @@ RUN Setup.bat -no-cache {% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards # (For details of what this is, see: ) -RUN del /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 +RUN rmdir /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 {% endif %} {% if (not disable_all_patches) and (not disable_ubt_patches) %} From b144e2816ebdc9c9352f59ba8669ee33603016e0 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 10 Mar 2021 17:02:14 +1000 Subject: [PATCH 139/430] Fix #132 --- ue4docker/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index fa6d4b14..97060c36 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -162,7 +162,7 @@ def build(): logger.error('SETUP REQUIRED:') logger.error('The max image size for Windows containers must be set to at least {}GB.'.format(requiredLimit)) logger.error('See the Microsoft documentation for configuration instructions:') - logger.error('https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size') + logger.error('https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits') logger.error('Under Windows Server, the command `{} setup` can be used to automatically configure the system.'.format(sys.argv[0])) sys.exit(1) From 76fefc7b741827a51f65873ad52e81ea031e1db5 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 15 Mar 2021 14:06:55 +1000 Subject: [PATCH 140/430] Bump version to 0.0.81 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index a7f66607..10783b58 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.80' +__version__ = '0.0.81' From 718d4081469967fb8e96fcb81a3a0ee1f986e684 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 17 Mar 2021 15:30:42 +0300 Subject: [PATCH 141/430] Fix engine build with --exclude ddc failing to remove nonexistent directory It turns out that rmdir returns exitcode=2 if directory to delete doesn't exist. So, only delete DDC directory if it actually exists. Broken by 9cf1375c33098a2787b5f514fd0ff36167a12b96. --- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 2dfc9053..514aab52 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -30,7 +30,7 @@ ARG BUILD_DDC WORKDIR C:\UnrealEngine COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` - rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC 2>NUL + (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size ARG EXCLUDE_DEBUG From 3bbf652ecdb63c0c954c9b5205e0449cbc704377 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 25 Mar 2021 13:34:04 +1000 Subject: [PATCH 142/430] Fix #141 --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 027db196..6a2a0d39 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -79,3 +79,10 @@ ARG EXCLUDE_TEMPLATES LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} {% endif %} + +# Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` +# (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) +# See the comments on this issue for details, including the need to ensure $HOME is set correctly: +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -NoUBTMakefiles -SkipBuild && \ + mkdir -p ./Engine/Programs/AutomationTool/Saved && \ + chmod a+rw ./Engine/Programs/AutomationTool/Saved From fd4f4b34b17a316c4292db1840bd94a5f9d6c534 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 31 Mar 2021 12:58:34 +0300 Subject: [PATCH 143/430] Add diagnostic for 20GiB COPY Docker bug on Windows --- ue4docker/diagnostics/__init__.py | 1 + ue4docker/diagnostics/diagnostic_20gig.py | 71 +++++++++++++++++++ ue4docker/diagnostics/diagnostic_all.py | 3 +- ue4docker/diagnostics_cmd.py | 1 + .../diagnostics/20gig/windows/Dockerfile | 22 ++++++ 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 ue4docker/diagnostics/diagnostic_20gig.py create mode 100644 ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile diff --git a/ue4docker/diagnostics/__init__.py b/ue4docker/diagnostics/__init__.py index 8c86f7a4..7b91b60c 100644 --- a/ue4docker/diagnostics/__init__.py +++ b/ue4docker/diagnostics/__init__.py @@ -1,4 +1,5 @@ from .diagnostic_all import allDiagnostics from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_20gig import diagnostic20Gig from .diagnostic_maxsize import diagnosticMaxSize from .diagnostic_network import diagnosticNetwork diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/ue4docker/diagnostics/diagnostic_20gig.py new file mode 100644 index 00000000..7f6864c3 --- /dev/null +++ b/ue4docker/diagnostics/diagnostic_20gig.py @@ -0,0 +1,71 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase + +import argparse, os, platform +from os.path import abspath, dirname, join + +class diagnostic20Gig(DiagnosticBase): + + # The tag we use for built images + IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:20gig' + + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 20gig') + self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + + def getName(self): + ''' + Returns the human-readable name of the diagnostic + ''' + return 'Check for Docker 20GiB COPY bug' + + def getDescription(self): + ''' + Returns a description of what the diagnostic does + ''' + return '\n'.join([ + 'This diagnostic determines if the Docker daemon suffers from 20GiB COPY bug', + 'reported at https://github.com/moby/moby/issues/37352 (affects Windows containers only)', + '', + '#37352 was fixed in https://github.com/moby/moby/pull/41636 but that fix was not released yet', + '', + self._parser.format_help() + ]) + + def getPrefix(self): + ''' + Returns the short name of the diagnostic for use in log output + ''' + return '20gig' + + def run(self, logger, args=[]): + ''' + Runs the diagnostic + ''' + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which platform we are running on + containerPlatform = platform.system().lower() + + if containerPlatform != 'windows': + logger.action('[20gig] Diagnostic skipped. Current platform is not affected by the bug this diagnostic checks\n') + return True + + buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + + # Attempt to build the Dockerfile + logger.action('[20gig] Attempting to COPY more than 20GiB between layers...', False) + built = self._buildDockerfile(logger, containerPlatform, diagnostic20Gig.IMAGE_TAG, buildArgs) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action('[20gig] Diagnostic succeeded! The Docker daemon can COPY more than 20GiB between layers.\n') + else: + logger.error('[20gig] Diagnostic failed! The Docker daemon cannot COPY more than 20GiB between layers.\n', True) + + return built diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index 09996ee3..2101aa93 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -1,5 +1,6 @@ from .base import DiagnosticBase from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_20gig import diagnostic20Gig from .diagnostic_maxsize import diagnosticMaxSize from .diagnostic_network import diagnosticNetwork @@ -24,7 +25,7 @@ def run(self, logger, args=[]): # Run all available diagnostics in turn, storing the results results = [] - diagnostics = [diagnostic8Gig(), diagnosticMaxSize(), diagnosticNetwork()] + diagnostics = [diagnostic8Gig(), diagnostic20Gig(), diagnosticMaxSize(), diagnosticNetwork()] for index, diagnostic in enumerate(diagnostics): # Run the diagnostic and report its result diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index 4749b716..ba3e3ebb 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -8,6 +8,7 @@ def diagnostics(): DIAGNOSTICS = { 'all': allDiagnostics(), '8gig': diagnostic8Gig(), + '20gig': diagnostic20Gig(), 'maxsize': diagnosticMaxSize(), 'network': diagnosticNetwork() } diff --git a/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile b/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile new file mode 100644 index 00000000..b12c58cb --- /dev/null +++ b/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile @@ -0,0 +1,22 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} as intermediate + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +WORKDIR C:\stuff + +# Create three 7GB files +# We do not use a single 21GiB file in order to distinguish this diagnostic from 8GiB diagnostic +RUN fsutil file createnew C:\stuff\bigfile_1.txt 7516192768 +RUN fsutil file createnew C:\stuff\bigfile_2.txt 7516192768 +RUN fsutil file createnew C:\stuff\bigfile_3.txt 7516192768 + +FROM mcr.microsoft.com/windows/servercore:${BASETAG} + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Copy more than 20GiB between docker layers +COPY --from=intermediate C:\stuff C:\stuff From 9c95c5a14d760be4060bffb0e3328484771805c3 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 6 Apr 2021 20:15:07 +1000 Subject: [PATCH 144/430] Bump version to 0.0.82 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 10783b58..56472280 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.81' +__version__ = '0.0.82' From c7c648eae9171de8e3224f32c09d7727ac9568ba Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 7 Apr 2021 11:26:27 -0700 Subject: [PATCH 145/430] Enable TSL1.2 when installing Chocolatey This commit fixes "Could not create SSL/TLS secure channel" error on Windows Server LTSC 2016. In Windows Server LTSC 2016, TLS 1.2 is disabled by default and chocolatey.org doesn't accept older TLS versions since Feb 2020. See https://github.com/adamrehn/ue4-docker/issues/151#issuecomment-815072013 and https://blog.chocolatey.org/2020/01/remove-support-for-old-tls-versions/ --- ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile | 2 +- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile b/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile index 9cda65ad..adff349b 100644 --- a/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile +++ b/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile @@ -7,4 +7,4 @@ SHELL ["cmd", "/S", "/C"] LABEL com.adamrehn.ue4-docker.sentinel="1" # Attempt to download the Chocolatey installation script -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "(New-Object System.Net.WebClient).DownloadFile('https://chocolatey.org/install.ps1', 'install.ps1')" +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; (New-Object System.Net.WebClient).DownloadFile('https://chocolatey.org/install.ps1', 'install.ps1')" diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 33440b45..18677497 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -13,7 +13,7 @@ RUN mkdir C:\GatheredDlls # Install 7-Zip, curl, and Python using Chocolatey # (Note that these need to be separate RUN directives for `choco` to work) -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" RUN choco install -y 7zip curl && choco install -y python --version=3.7.5 # Copy the required DirectSound/DirectDraw and OpenGL DLL files from the host system (since these ship with Windows and don't have installers) @@ -57,7 +57,7 @@ LABEL com.adamrehn.ue4-docker.sentinel="1" RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f # Install Chocolatey -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Install the rest of our build prerequisites and clean up afterwards to minimise image size COPY buildtools-exitcode.py copy.py copy-pdbcopy.py install-prerequisites.bat C:\ From 3907f45f87f3e3458e16770f9bc45f76fc5490d4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 7 Apr 2021 21:54:08 +0300 Subject: [PATCH 146/430] Fix curl --progress flag This commit fixes "curl: option --progress: is ambiguous" error curl never had --progress flag. Instead, it has --progress-bar. --progress worked before due to heuristic logic, but not it became ambiguous. See https://github.com/moby/moby/pull/40914 --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 4 ++-- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 33440b45..6a614afc 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -27,7 +27,7 @@ RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ RUN python C:\verify-host-dlls.py %HOST_VERSION% C:\GatheredDlls # Gather the required DirectX runtime files, since Windows Server Core does not include them -RUN curl --progress -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe +RUN curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\GatheredDlls\ && ` expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\GatheredDlls\ && ` @@ -36,7 +36,7 @@ RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\GatheredDlls\ # Gather the Vulkan runtime library -RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip +RUN curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" # Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 12a89f9a..09f69348 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -12,7 +12,7 @@ git config --system credential.helper "" || goto :error @rem Install the Visual Studio 2017 Build Tools workloads and components we need, excluding components with known issues in containers @rem (Note that we use the Visual Studio 2019 installer here because the old installer now breaks, but explicitly install the VS2017 Build Tools) -curl --progress -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error +curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ --channelUri "https://aka.ms/vs/15/release/channel" ^ From 7094952619e488b2ec23cea3515b7a6056d7ed4d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 18 May 2021 14:17:30 +1000 Subject: [PATCH 147/430] Always print error when connecting to the Docker daemon fails --- ue4docker/infrastructure/DockerUtils.py | 4 ++-- ue4docker/main.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 82815766..b4f798a6 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -9,10 +9,10 @@ def installed(): Determines if Docker is installed ''' try: - return DockerUtils.version() is not None + return (DockerUtils.version() is not None), None except Exception as e: logging.debug(str(e)) - return False + return False, e @staticmethod def version(): diff --git a/ue4docker/main.py b/ue4docker/main.py index 7b42e5de..75f47b2c 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -27,8 +27,9 @@ def main(): sys.argv = list([arg for arg in sys.argv if arg not in ['-v', '--verbose']]) # Verify that Docker is installed - if DockerUtils.installed() == False: - _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.') + installed, error = DockerUtils.installed() + if installed == False: + _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}'.format(error)) # Under Windows, verify that the host is a supported version if platform.system() == 'Windows' and WindowsUtils.isSupportedWindowsVersion() == False: From a0cc68084f208cda6ab15c2d7d17e77bda54ac58 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 18 May 2021 14:17:49 +1000 Subject: [PATCH 148/430] Bump version to 0.0.83 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 56472280..8ced4167 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.82' +__version__ = '0.0.83' From b8bcf72549dd79080592f0d26825d0fe6a9001ce Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 18 May 2021 15:16:39 +1000 Subject: [PATCH 149/430] Bump version to 0.0.84 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8ced4167..51a7f17a 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.83' +__version__ = '0.0.84' From 2393cf8bb38b8154109e1f88807def884732a9fe Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 18 May 2021 12:44:20 +0300 Subject: [PATCH 150/430] Reduce ue4-build-prerequisites Windows image (#144) This commit: 1. Replaces base image from mcr.microsoft.com/dotnet/framework/sdk to mcr.microsoft.com/windows/servercore The former is 9GB while the latter is just 5GB. dotnet/framework/sdk contains lots of things that are not actually needed to build ue4, the hugest is VS Test Agent + VS Build Tools 2019 that take together 2.2GB 2. Stops installing unneeded VS Build Tools workloads/components and only installs required minimum: * MSBuild (needed to build MSBuild-based UE4 tools, like UBT) * NuGet (needed to run MSBuild) * C# compiler (needed to build C# UE4 tools) * .NET SDK 4.5 + 4.6.2 (same as above) * Windows 10 SDK (needed to build C++ code) * VCTools: C++ compiler, VC redist (same as above) Cumulatively, this commit reduces ue4-build-prerequisites image from 24.5GB down to 10.7GB. The most important consequence of this change is that it reduces ue4-minimal/ue4-full image sizes by the same amount --- .../windows/install-prerequisites.bat | 14 ++++++++------ ue4docker/infrastructure/BuildConfiguration.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 09f69348..dd773774 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -10,7 +10,7 @@ call refreshenv @rem Forcibly disable the git credential manager git config --system credential.helper "" || goto :error -@rem Install the Visual Studio 2017 Build Tools workloads and components we need, excluding components with known issues in containers +@rem Install the Visual Studio 2017 Build Tools workloads and components we need @rem (Note that we use the Visual Studio 2019 installer here because the old installer now breaks, but explicitly install the VS2017 Build Tools) curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ @@ -19,12 +19,14 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output --installChannelUri "https://aka.ms/vs/15/release/channel" ^ --channelId VisualStudio.15.Release ^ --productId Microsoft.VisualStudio.Product.BuildTools ^ - --add Microsoft.VisualStudio.Workload.VCTools;includeRecommended ^ - --add Microsoft.VisualStudio.Workload.ManagedDesktopBuildTools;includeRecommended ^ - --add Microsoft.VisualStudio.Workload.UniversalBuildTools ^ - --add Microsoft.VisualStudio.Workload.NetCoreBuildTools ^ + --add Microsoft.VisualStudio.Workload.VCTools ^ --add Microsoft.VisualStudio.Workload.MSBuildTools ^ - --add Microsoft.VisualStudio.Component.NuGet + --add Microsoft.VisualStudio.Component.NuGet ^ + --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ + --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ^ + --add Microsoft.Net.Component.4.5.TargetingPack ^ + --add Microsoft.Net.Component.4.6.2.SDK ^ + --add Microsoft.Net.Component.4.6.2.TargetingPack python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error @rem Copy pdbcopy.exe to the expected location(s) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 597dcdb7..29284e8a 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -231,7 +231,7 @@ def _generateWindowsConfig(self): # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag - self.baseImage = 'mcr.microsoft.com/dotnet/framework/sdk:4.8-windowsservercore-' + self.basetag + self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag self.prereqsTag = self.basetag # Verify that any user-specified base tag is valid From 5c23b050120b31193b624a77819461697fd36a7d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 18 May 2021 12:47:11 +0300 Subject: [PATCH 151/430] Only copy exclude-components.py to ue4-minimal when it is actually needed (#148) This change allows to avoid full engine rebuild when making changes to exclude-components.py --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 2 +- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 6a2a0d39..0015395b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -34,11 +34,11 @@ RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -Clean # Create an Installed Build of the Engine ARG BUILD_DDC WORKDIR /home/ue4/UnrealEngine -COPY exclude-components.py /tmp/exclude-components.py RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size +COPY exclude-components.py /tmp/exclude-components.py ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES RUN python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 514aab52..14bba681 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -28,11 +28,11 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB # Create an Installed Build of the Engine ARG BUILD_DDC WORKDIR C:\UnrealEngine -COPY exclude-components.py C:\exclude-components.py RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size +COPY exclude-components.py C:\exclude-components.py ARG EXCLUDE_DEBUG ARG EXCLUDE_TEMPLATES RUN python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% From d832c11d8017fcca2f2894b44a1421152aab51db Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 18 May 2021 13:03:07 +0300 Subject: [PATCH 152/430] Force English locale when installing VS Build Tools (#156) Before this commit, VS Build Tools installer would pick up system locale from host machine, leading to inconsistent results depending on machine that is used to build ue4-build-prerequisites image --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index dd773774..384e0d1e 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -19,6 +19,7 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output --installChannelUri "https://aka.ms/vs/15/release/channel" ^ --channelId VisualStudio.15.Release ^ --productId Microsoft.VisualStudio.Product.BuildTools ^ + --locale en-US ^ --add Microsoft.VisualStudio.Workload.VCTools ^ --add Microsoft.VisualStudio.Workload.MSBuildTools ^ --add Microsoft.VisualStudio.Component.NuGet ^ From 83d0de21ed9ac3feb93dbef9045024894efe2519 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 19 May 2021 12:58:59 +1000 Subject: [PATCH 153/430] Enable Vulkan support in ue4-minimal as well as ue4-full --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 13 ------------- .../dockerfiles/ue4-minimal/linux/Dockerfile | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 7cfd4588..35451c8c 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -59,19 +59,6 @@ USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf -# Enable Vulkan support -RUN apt-get install -y --no-install-recommends libvulkan1 && \ - VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \ - mkdir -p /etc/vulkan/icd.d/ && \ - echo \ - "{\ - \"file_format_version\" : \"1.0.0\",\ - \"ICD\": {\ - \"library_path\": \"libGLX_nvidia.so.0\",\ - \"api_version\" : \"${VULKAN_API_VERSION}\"\ - }\ - }" > /etc/vulkan/icd.d/nvidia_icd.json - # Enable X11 support (including the libraries required by CEF) and xvfb so we can create a dummy display if needed RUN apt-get install -y --no-install-recommends \ libasound2 \ diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 0015395b..1689ecf4 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -86,3 +86,19 @@ LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -NoUBTMakefiles -SkipBuild && \ mkdir -p ./Engine/Programs/AutomationTool/Saved && \ chmod a+rw ./Engine/Programs/AutomationTool/Saved + +# Enable Vulkan support for NVIDIA GPUs +USER root +RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ + rm -rf /var/lib/apt/lists/* && \ + VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \ + mkdir -p /etc/vulkan/icd.d/ && \ + echo \ + "{\ + \"file_format_version\" : \"1.0.0\",\ + \"ICD\": {\ + \"library_path\": \"libGLX_nvidia.so.0\",\ + \"api_version\" : \"${VULKAN_API_VERSION}\"\ + }\ + }" > /etc/vulkan/icd.d/nvidia_icd.json +USER ue4 From fa4f21dbbb7cd874cfde931a61a237b1e6062a77 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 19 May 2021 12:59:33 +1000 Subject: [PATCH 154/430] Bump version to 0.0.85 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 51a7f17a..8536fab3 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.84' +__version__ = '0.0.85' From 4d183cdf3e63ba9d2d57beeda6e9d3ccbea7c814 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 19 May 2021 11:11:46 +0300 Subject: [PATCH 155/430] Add PyCharm files to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 36b9f1fa..cb94132f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +.idea/ .vscode/ __pycache__/ build dist +venv/ *.egg-info *.pyc From 6d2a8ee3c44f11126a0815f175b4a0b4d9a40619 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 20 May 2021 00:44:42 +0300 Subject: [PATCH 156/430] resolves #146 remove PDB files instead of truncating them (#149) This commit fixes 'error CS0045: Unexpected error creating debug information file' that occurs when UE4 builds Engine/Intermediate/Build/BuildRules/UE4Rules.dll on Windows --- .../ue4-minimal/windows/exclude-components.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py index d4213e9b..c92c43d0 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py @@ -17,19 +17,19 @@ def readFile(filename): version = json.loads(readFile(join(rootDir, 'Engine', 'Build', 'Build.version'))) # Determine if we are excluding debug symbols -truncateDebug = len(sys.argv) > 2 and sys.argv[2] == '1' -if truncateDebug == True: +excludeDebug = len(sys.argv) > 2 and sys.argv[2] == '1' +if excludeDebug == True: - # Truncate all PDB files to save space whilst avoiding the issues that would be caused by the files being missing - log('User opted to exclude debug symbols, truncating all PDB files.') + # Remove all PDB files to save space + log('User opted to exclude debug symbols, removing all PDB files.') log('Scanning for PDB files in directory {}...'.format(rootDir)) pdbFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True) for pdbFile in pdbFiles: - log('Truncating PDB file {}...'.format(pdbFile)) + log('Removing PDB file {}...'.format(pdbFile)) try: - os.truncate(pdbFile, 0) + os.remove(pdbFile) except: - log(' Warning: failed to truncate PDB file {}.'.format(pdbFile)) + log(' Warning: failed to remove PDB file {}.'.format(pdbFile)) # Determine if we are excluding the Engine's template projects and samples excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' From 8156c78c03a1b4495be792c1d9528dd0ee0b9a66 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 21 May 2021 01:16:36 +0300 Subject: [PATCH 157/430] Mark as EOL Windows releases no longer supported by Microsoft (#145) According to [Base image servicing lifecycles](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/base-image-lifecycle), these Windows Server Core releases are already EOL as of today --- ue4docker/infrastructure/WindowsUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 720eec50..aef8175d 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -22,7 +22,7 @@ class WindowsUtils(object): _blacklistedReleases = ['1903', '1909'] # The list of Windows Server Core container image releases that are unsupported due to having reached EOL - _eolReleases = ['1709'] + _eolReleases = ['1709', '1803', '1903'] @staticmethod def _getVersionRegKey(subkey): From 1c366feb65c3f2c0694f30dfe1846bee2d9e05b1 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 21 May 2021 01:43:39 +0300 Subject: [PATCH 158/430] Add Dependabot config --- .github/dependabot.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..e46a75d1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,8 @@ +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" From e1a7ce6d4f30b2582ec90797387d812258027ce4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 25 May 2021 11:04:45 +0300 Subject: [PATCH 159/430] Use Windows build number instead of ReleaseId wherever possible (#170) Motivation: ReleaseId is confusing. In Windows 20H2, ReleaseId is 2009 (so it doesn't match human-visible version). Even worse, ReleaseId is *also* 2009 in Windows 21H2 and we actually need to use DisplayName to distinguish between the two. Depending on how we proceed with fixing #138 and #166, we might completely drop ReleaseId and only operate on build numbers. This commit is not expected to change ue4-docker behavior in any way (possibly except for 21H1 that is not supported yet anyway). --- ue4docker/build.py | 15 +- ue4docker/diagnostics/diagnostic_maxsize.py | 2 +- .../windows/Dockerfile | 4 +- .../windows/verify-host-dlls.py | 15 +- ue4docker/info.py | 2 +- .../infrastructure/BuildConfiguration.py | 8 +- ue4docker/infrastructure/WindowsUtils.py | 129 ++++++++---------- ue4docker/main.py | 7 +- 8 files changed, 79 insertions(+), 103 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 97060c36..c8870df7 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -112,11 +112,12 @@ def build(): # Provide the user with feedback so they are aware of the Windows-specific values being used logger.info('WINDOWS CONTAINER SETTINGS', False) logger.info('Isolation mode: {}'.format(config.isolation), False) - logger.info('Base OS image tag: {} (host OS is {})'.format(config.basetag, WindowsUtils.systemStringShort()), False) + logger.info('Base OS image tag: {}'.format(config.basetag), False) + logger.info('Host OS: {}'.format(WindowsUtils.systemString()), False) logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) - + # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) if WindowsUtils.isEndOfLifeWindowsVersion(config.basetag) == True: logger.error('Error: detected EOL base OS image tag: {}'.format(config.basetag), False) @@ -124,10 +125,10 @@ def build(): logger.error('Microsoft no longer supports or maintains container base images for it.', False) logger.error('You will need to use a base image tag for a supported version of Windows.', False) sys.exit(1) - + # Verify that the host OS is not a release that is blacklisted due to critical bugs if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: - logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemStringShort()), False) + logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemString()), False) logger.error('', False) logger.error('This version of Windows contains one or more critical bugs that', False) logger.error('render it incapable of successfully building UE4 container images.', False) @@ -136,12 +137,12 @@ def build(): logger.error('For more information, see:', False) logger.error('https://unrealcontainers.com/docs/concepts/windows-containers', False) sys.exit(1) - + # Verify that the user is not attempting to build images with a newer kernel version than the host OS if WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag): logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') sys.exit(1) - + # Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32 differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag if config.pullPrerequisites == False and differentKernels == True and config.dlldir == config.defaultDllDir: @@ -239,7 +240,7 @@ def build(): # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] if config.containerPlatform == 'windows': - prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_VERSION=' + WindowsUtils.getWindowsBuild()] + prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild())] # Build or pull the UE4 build prerequisites image (don't pull it if we're copying Dockerfiles to an output directory) if config.layoutDir is None and config.pullPrerequisites == True: diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py index c7e30fe5..0b831eb7 100644 --- a/ue4docker/diagnostics/diagnostic_maxsize.py +++ b/ue4docker/diagnostics/diagnostic_maxsize.py @@ -47,7 +47,7 @@ def run(self, logger, args=[]): return False # Verify that we are running Windows Server or Windows 10 version 1903 or newer - if WindowsUtils.getWindowsVersion()['patch'] < 18362: + if WindowsUtils.getWindowsBuild() < 18362: logger.info('[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.', False) return True diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index f8fa73c4..e3e51967 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -20,11 +20,11 @@ RUN choco install -y 7zip curl && choco install -y python --version=3.7.5 COPY *.dll C:\GatheredDlls\ # Verify that the DLL files copied from the host can be loaded by the container OS -ARG HOST_VERSION +ARG HOST_BUILD RUN pip install pywin32 COPY copy.py verify-host-dlls.py C:\ RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ -RUN python C:\verify-host-dlls.py %HOST_VERSION% C:\GatheredDlls +RUN python C:\verify-host-dlls.py %HOST_BUILD% C:\GatheredDlls # Gather the required DirectX runtime files, since Windows Server Core does not include them RUN curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py index f58c84a2..757a8442 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py @@ -1,4 +1,5 @@ -import glob, os, platform, sys, win32api, winreg +import glob, os, sys, win32api + # Adapted from the code in this SO answer: def getDllVersion(dllPath): @@ -10,20 +11,10 @@ def getDllVersion(dllPath): info['FileVersionLS'] % 65536 ) -def getVersionRegKey(subkey): - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') - value = winreg.QueryValueEx(key, subkey) - winreg.CloseKey(key) - return value[0] - -def getOsVersion(): - version = platform.win32_ver()[1] - build = getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) # Print the host and container OS build numbers print('Host OS build number: {}'.format(sys.argv[1])) -print('Container OS build number: {}'.format(getOsVersion())) +print('Container OS build number: {}'.format(sys.getwindowsversion().build)) sys.stdout.flush() # Verify each DLL file in the directory specified by our command-line argument diff --git a/ue4docker/info.py b/ue4docker/info.py index 2db9935f..03f95df3 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -4,7 +4,7 @@ def _osName(dockerInfo): if platform.system() == 'Windows': - return WindowsUtils.systemStringLong() + return WindowsUtils.systemString() elif platform.system() == 'Darwin': return DarwinUtils.systemString() else: diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 29284e8a..ff9d61b1 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -233,15 +233,15 @@ def _generateWindowsConfig(self): self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag self.prereqsTag = self.basetag - + # Verify that any user-specified base tag is valid if WindowsUtils.isValidBaseTag(self.basetag) == False: raise RuntimeError('unrecognised Windows Server Core base image tag "{}", supported tags are {}'.format(self.basetag, WindowsUtils.getValidBaseTags())) - + # Verify that any user-specified tag suffix does not collide with our base tags if WindowsUtils.isValidBaseTag(self.suffix) == True: raise RuntimeError('tag suffix cannot be any of the Windows Server Core base image tags: {}'.format(WindowsUtils.getValidBaseTags())) - + # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect if self.args.isolation is not None: self.isolation = self.args.isolation @@ -249,7 +249,7 @@ def _generateWindowsConfig(self): # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag - hostSupportsProcess = WindowsUtils.isWindowsServer() or int(self.hostRelease) >= 1809 + hostSupportsProcess = WindowsUtils.supportsProcessIsolation() dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') if not differentKernels and hostSupportsProcess and dockerSupportsProcess: self.isolation = 'process' diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index aef8175d..33d73006 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -1,111 +1,90 @@ from .DockerUtils import DockerUtils -from .PackageUtils import PackageUtils from pkg_resources import parse_version -import os, platform +import platform, sys if platform.system() == 'Windows': import winreg -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - class WindowsUtils(object): - + # The latest Windows build version we recognise as a non-Insider build _latestReleaseBuild = 19042 - + # The list of Windows Server Core base image tags that we recognise, in ascending version number order _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004', '20H2'] - + # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) _blacklistedReleases = ['1903', '1909'] - + # The list of Windows Server Core container image releases that are unsupported due to having reached EOL _eolReleases = ['1709', '1803', '1903'] - + @staticmethod - def _getVersionRegKey(subkey): + def _getVersionRegKey(subkey : str) -> str: ''' Retrieves the specified Windows version key from the registry + + @raises FileNotFoundError if registry key doesn't exist ''' key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') value = winreg.QueryValueEx(key, subkey) winreg.CloseKey(key) return value[0] - + @staticmethod - def requiredHostDlls(basetag): + def requiredHostDlls(basetag: str) -> [str]: ''' Returns the list of required host DLL files for the specified container image base tag ''' - + # `ddraw.dll` is only required under Windows Server 2016 version 1607 common = ['dsound.dll', 'opengl32.dll', 'glu32.dll'] return ['ddraw.dll'] + common if basetag == 'ltsc2016' else common - + @staticmethod - def requiredSizeLimit(): + def requiredSizeLimit() -> float: ''' Returns the minimum required image size limit (in GB) for Windows containers ''' return 400.0 - + @staticmethod - def minimumRequiredVersion(): + def minimumRequiredBuild() -> int: ''' Returns the minimum required version of Windows 10 / Windows Server, which is release 1607 - + (1607 is the first build to support Windows containers, as per: ) ''' - return '10.0.14393' - - @staticmethod - def systemStringShort(): - ''' - Generates a concise human-readable version string for the Windows host system - ''' - return 'Windows {} version {}'.format( - 'Server' if WindowsUtils.isWindowsServer() else '10', - WindowsUtils.getWindowsRelease() - ) - + return 14393 + @staticmethod - def systemStringLong(): + def systemString() -> str: ''' Generates a verbose human-readable version string for the Windows host system ''' - return '{} Version {} (OS Build {}.{})'.format( + return '{} Version {} (Build {}.{})'.format( WindowsUtils._getVersionRegKey('ProductName'), WindowsUtils.getWindowsRelease(), - WindowsUtils.getWindowsVersion()['patch'], + WindowsUtils.getWindowsBuild(), WindowsUtils._getVersionRegKey('UBR') ) - - @staticmethod - def getWindowsVersion(): - ''' - Returns the version information for the Windows host system as a semver instance - ''' - return semver.parse(platform.win32_ver()[1]) - + @staticmethod - def getWindowsRelease(): + def getWindowsRelease() -> str: ''' Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system ''' return WindowsUtils._getVersionRegKey('ReleaseId') - + @staticmethod - def getWindowsBuild(): + def getWindowsBuild() -> int: ''' - Returns the full Windows version number as a string, including the build number + Returns build number for the Windows host system ''' - version = platform.win32_ver()[1] - build = WindowsUtils._getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) - + return sys.getwindowsversion().build + @staticmethod def isBlacklistedWindowsVersion(release=None): ''' @@ -115,7 +94,7 @@ def isBlacklistedWindowsVersion(release=None): dockerVersion = parse_version(DockerUtils.version()['Version']) release = WindowsUtils.getWindowsRelease() if release is None else release return release in WindowsUtils._blacklistedReleases and dockerVersion < parse_version('19.03.6') - + @staticmethod def isEndOfLifeWindowsVersion(release=None): ''' @@ -124,39 +103,32 @@ def isEndOfLifeWindowsVersion(release=None): ''' release = WindowsUtils.getWindowsRelease() if release is None else release return release in WindowsUtils._eolReleases - - @staticmethod - def isSupportedWindowsVersion(): - ''' - Verifies that the Windows host system meets our minimum Windows version requirements - ''' - return semver.compare(platform.win32_ver()[1], WindowsUtils.minimumRequiredVersion()) >= 0 - + @staticmethod - def isWindowsServer(): + def isWindowsServer() -> bool: ''' Determines if the Windows host system is Windows Server ''' + # TODO: Replace this with something more reliable return 'Windows Server' in WindowsUtils._getVersionRegKey('ProductName') - + @staticmethod - def isInsiderPreview(): + def isInsiderPreview() -> bool: ''' Determines if the Windows host system is a Windows Insider preview build ''' - version = WindowsUtils.getWindowsVersion() - return version['patch'] > WindowsUtils._latestReleaseBuild - + return WindowsUtils.getWindowsBuild() > WindowsUtils._latestReleaseBuild + @staticmethod - def getReleaseBaseTag(release): + def getReleaseBaseTag(release: str) -> str: ''' Retrieves the tag for the Windows Server Core base image matching the specified Windows 10 / Windows Server release ''' - + # For Windows Insider preview builds, build the latest release tag if WindowsUtils.isInsiderPreview(): return WindowsUtils._validTags[-1] - + # This lookup table is based on the list of valid tags from return { '1709': '1709', @@ -168,24 +140,33 @@ def getReleaseBaseTag(release): '2009': '20H2', '20H2': '20H2' }.get(release, 'ltsc2016') - + @staticmethod - def getValidBaseTags(): + def getValidBaseTags() -> [str]: ''' Returns the list of valid tags for the Windows Server Core base image, in ascending chronological release order ''' return WindowsUtils._validTags - + @staticmethod - def isValidBaseTag(tag): + def isValidBaseTag(tag: str) -> bool: ''' Determines if the specified tag is a valid Windows Server Core base image tag ''' return tag in WindowsUtils._validTags - + @staticmethod - def isNewerBaseTag(older, newer): + def isNewerBaseTag(older: str, newer: str) -> bool: ''' Determines if the base tag `newer` is chronologically newer than the base tag `older` ''' return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older) + + @staticmethod + def supportsProcessIsolation() -> bool: + ''' + Determines whether the Windows host system supports process isolation for containers + + @see https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container + ''' + return WindowsUtils.isWindowsServer() or WindowsUtils.getWindowsBuild() >= 17763 diff --git a/ue4docker/main.py b/ue4docker/main.py index 75f47b2c..4a4bc3d7 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -32,8 +32,11 @@ def main(): _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}'.format(error)) # Under Windows, verify that the host is a supported version - if platform.system() == 'Windows' and WindowsUtils.isSupportedWindowsVersion() == False: - _exitWithError('Error: the detected version of Windows ({}) is not supported. Windows 10 / Windows Server version 1607 or newer is required.'.format(platform.win32_ver()[1])) + if platform.system() == 'Windows': + host_build = WindowsUtils.getWindowsBuild() + min_build = WindowsUtils.minimumRequiredBuild() + if host_build < min_build: + _exitWithError('Error: the detected build of Windows ({}) is not supported. {} or newer is required.'.format(host_build, min_build)) # Under macOS, verify that the host is a supported version if platform.system() == 'Darwin' and DarwinUtils.isSupportedMacOsVersion() == False: From 057f0183b2765061dcb77221673fedacab00d51b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 25 May 2021 11:14:51 +0300 Subject: [PATCH 160/430] Add --ignore-eol option that allows to run ue4-docker build even on EOL Windows versions --- ue4docker/build.py | 2 +- ue4docker/infrastructure/BuildConfiguration.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index c8870df7..0bfb527d 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -119,7 +119,7 @@ def build(): logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) - if WindowsUtils.isEndOfLifeWindowsVersion(config.basetag) == True: + if not config.ignoreEOL and WindowsUtils.isEndOfLifeWindowsVersion(config.basetag): logger.error('Error: detected EOL base OS image tag: {}'.format(config.basetag), False) logger.error('This version of Windows has reached End Of Life (EOL), which means', False) logger.error('Microsoft no longer supports or maintains container base images for it.', False) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index ff9d61b1..247f82a7 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -92,6 +92,7 @@ def addArguments(parser): parser.add_argument('--combine', action='store_true', help='Combine generated Dockerfiles into a single multi-stage build Dockerfile') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') + parser.add_argument('--ignore-eol', action='store_true', help='Run builds even on EOL versions of Windows (advanced use only)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output during builds (useful for debugging)') @@ -150,6 +151,7 @@ def __init__(self, parser, argv): self.excludedComponents = set(self.args.exclude) self.baseImage = None self.prereqsTag = None + self.ignoreEOL = self.args.ignore_eol self.ignoreBlacklist = self.args.ignore_blacklist self.verbose = self.args.verbose self.layoutDir = self.args.layout From 8de07d3548d16c66252ba31b8a834bd425f870e9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 27 May 2021 10:48:53 +0300 Subject: [PATCH 161/430] resolves #163 Reduce number of installed vcredists Tested against 4.20.3 and 4.26.2 on Windows LTSC 2019 --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 384e0d1e..b5c4fba2 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,6 +1,6 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error -choco install -y curl vcredist-all || goto :error +choco install -y curl vcredist2010 vcredist140 || goto :error choco install -y python --version=3.7.5 || goto :error @rem Reload our environment variables from the registry so the `git` command works From 615cc30d0c885a7646c78c4d576c7d84e8055a44 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 27 May 2021 16:32:12 +0300 Subject: [PATCH 162/430] Mark Windows 1703 and 1909 as EOL Related: https://github.com/microsoft/dotnet-framework-docker/pull/765 --- ue4docker/infrastructure/WindowsUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 33d73006..8212382f 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -18,7 +18,7 @@ class WindowsUtils(object): _blacklistedReleases = ['1903', '1909'] # The list of Windows Server Core container image releases that are unsupported due to having reached EOL - _eolReleases = ['1709', '1803', '1903'] + _eolReleases = ['1703', '1709', '1803', '1903', '1909'] @staticmethod def _getVersionRegKey(subkey : str) -> str: From 88d686fba1bc128103cbd0f3fdb2f94e73f0e7d6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 1 Jun 2021 10:09:41 +0300 Subject: [PATCH 163/430] Add support for Visual Studio 2019 (#173) resolves #172 --- ue4docker/build.py | 8 +++-- .../windows/Dockerfile | 3 +- .../windows/install-prerequisites.bat | 18 ++++++---- .../ue4-minimal/windows/Dockerfile | 3 +- .../infrastructure/BuildConfiguration.py | 36 +++++++++++++++++-- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 0bfb527d..4941d659 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -116,6 +116,7 @@ def build(): logger.info('Host OS: {}'.format(WindowsUtils.systemString()), False) logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) + logger.info('Visual Studio: {}'.format(config.visualStudio), False) logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) @@ -240,7 +241,10 @@ def build(): # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] if config.containerPlatform == 'windows': - prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild())] + prereqsArgs = prereqsArgs + [ + '--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild()), + '--build-arg', 'VISUAL_STUDIO_BUILD_NUMBER=' + config.visualStudioBuildNumber, + ] # Build or pull the UE4 build prerequisites image (don't pull it if we're copying Dockerfiles to an output directory) if config.layoutDir is None and config.pullPrerequisites == True: @@ -282,7 +286,7 @@ def build(): # Build the minimal UE4 CI image, unless requested otherwise by the user buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: - builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs) + builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs + config.uatBuildFlags) builtImages.append('ue4-minimal') else: logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index e3e51967..4519a362 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -61,4 +61,5 @@ RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy # Install the rest of our build prerequisites and clean up afterwards to minimise image size COPY buildtools-exitcode.py copy.py copy-pdbcopy.py install-prerequisites.bat C:\ -RUN C:\install-prerequisites.bat +ARG VISUAL_STUDIO_BUILD_NUMBER +RUN C:\install-prerequisites.bat %VISUAL_STUDIO_BUILD_NUMBER% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index b5c4fba2..c7750a2d 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -10,24 +10,28 @@ call refreshenv @rem Forcibly disable the git credential manager git config --system credential.helper "" || goto :error -@rem Install the Visual Studio 2017 Build Tools workloads and components we need -@rem (Note that we use the Visual Studio 2019 installer here because the old installer now breaks, but explicitly install the VS2017 Build Tools) +set VISUAL_STUDIO_BUILD_NUMBER=%~1 + +@rem Install the Visual Studio Build Tools workloads and components we need +@rem NOTE: We use the Visual Studio 2019 installer even for Visual Studio 2017 here because the old installer now breaks +@rem NOTE: VS2019 Build Tools doesn't have 4.6.2 .NET SDK and what actually gets installed is 4.8 curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ - --channelUri "https://aka.ms/vs/15/release/channel" ^ - --installChannelUri "https://aka.ms/vs/15/release/channel" ^ - --channelId VisualStudio.15.Release ^ + --channelUri "https://aka.ms/vs/%VISUAL_STUDIO_BUILD_NUMBER%/release/channel" ^ + --installChannelUri "https://aka.ms/vs/%VISUAL_STUDIO_BUILD_NUMBER%/release/channel" ^ + --channelId VisualStudio.%VISUAL_STUDIO_BUILD_NUMBER%.Release ^ --productId Microsoft.VisualStudio.Product.BuildTools ^ --locale en-US ^ --add Microsoft.VisualStudio.Workload.VCTools ^ --add Microsoft.VisualStudio.Workload.MSBuildTools ^ --add Microsoft.VisualStudio.Component.NuGet ^ --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ^ --add Microsoft.Net.Component.4.5.TargetingPack ^ - --add Microsoft.Net.Component.4.6.2.SDK ^ - --add Microsoft.Net.Component.4.6.2.TargetingPack + --add Microsoft.Net.ComponentGroup.4.6.2.DeveloperTools + python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error @rem Copy pdbcopy.exe to the expected location(s) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 14bba681..c1b6f5a9 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -27,8 +27,9 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB # Create an Installed Build of the Engine ARG BUILD_DDC +ARG USE_VS2019 WORKDIR C:\UnrealEngine -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% {{ buildgraph_args }} && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% -set:VS2019=%USE_VS2019% {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 247f82a7..3c450d0c 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -27,6 +27,23 @@ # The default memory limit (in GB) under Windows DEFAULT_MEMORY_LIMIT = 10.0 +class VisualStudio(object): + VS2017 = '2017' + VS2019 = '2019' + + BuildNumbers = { + VS2017 : '15', + VS2019 : '16', + } + + MinSupportedUnreal = { + # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 + # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa + # + # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ + # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 + VS2019 : semver.VersionInfo(4, 25) + } class ExcludedComponent(object): ''' @@ -77,6 +94,7 @@ def addArguments(parser): parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.DDC, ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') parser.add_argument('--opt', action='append', default=[], help='Set an advanced configuration option (can be specified multiple times to specify multiple options)') parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') + parser.add_argument('--visual-studio', default=VisualStudio.VS2017, choices=VisualStudio.BuildNumbers.keys(), help='Specify Visual Studio Build Tools version to use for Windows containers') parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') parser.add_argument('-password', default=None, help='Specify the password or access token to use when cloning the git repository') parser.add_argument('-repo', default=None, help='Set the custom git repository to clone when "custom" is specified as the release value') @@ -201,6 +219,8 @@ def __init__(self, parser, argv): '--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0), '--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0) ] + + self.uatBuildFlags = [] # If we're building Windows containers, generate our Windows-specific configuration settings if self.containerPlatform == 'windows': @@ -226,7 +246,19 @@ def _generateWindowsConfig(self): hostSystem32 = os.path.join(os.environ['SystemRoot'], 'System32') self.defaultDllDir = hostSysnative if os.path.exists(hostSysnative) else hostSystem32 self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir - + + self.visualStudio = self.args.visual_studio + + if not self.custom: + # Check whether specified Unreal Engine release is compatible with specified Visual Studio + vsMinSupportedUnreal = VisualStudio.MinSupportedUnreal.get(self.visualStudio, None) + if vsMinSupportedUnreal is not None and semver.VersionInfo.parse(self.release) < vsMinSupportedUnreal: + raise RuntimeError('specified version of Unreal Engine cannot be built with Visual Studio {}, oldest supported is {}'.format(self.visualStudio, vsMinSupportedUnreal)) + + self.visualStudioBuildNumber = VisualStudio.BuildNumbers[self.visualStudio] + # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac + self.uatBuildFlags += ['--build-arg', 'USE_VS2019={}'.format('true' if self.visualStudio == VisualStudio.VS2019 else 'false')] + # Determine base tag for the Windows release of the host system self.hostRelease = WindowsUtils.getWindowsRelease() self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease) @@ -234,7 +266,7 @@ def _generateWindowsConfig(self): # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag - self.prereqsTag = self.basetag + self.prereqsTag = self.basetag + '-vs' + self.visualStudio # Verify that any user-specified base tag is valid if WindowsUtils.isValidBaseTag(self.basetag) == False: From 835678ae373b0ff9516c7e3d69e6da2558421686 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 2 Jun 2021 18:13:57 +0300 Subject: [PATCH 164/430] Remove bogus NuGet config file See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852377721 --- .../windows/install-prerequisites.bat | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index c7750a2d..6a7e8817 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -41,6 +41,11 @@ python C:\copy-pdbcopy.py || goto :error rmdir /S /Q \\?\%TEMP% mkdir %TEMP% +@rem Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file +@rem Just remove it, so a proper one will be generated on next NuGet run +@rem See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034 +if exist %APPDATA%\NuGet rmdir /s /q %APPDATA%\NuGet + @rem Display a human-readable completion message @echo off @echo Finished installing build prerequisites and cleaning up temporary files. From d2aafaa57df38b623db3248ca236c7a290954974 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 3 Jun 2021 17:37:20 +0300 Subject: [PATCH 165/430] Adjust patching of build graph so it doesn't accidentally make changes to UE-5.24 (in case such thing will even exist) --- ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index c6478b04..66f0864e 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -31,7 +31,7 @@ def writeFile(filename, data): # Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail # (See for details of the bug and its fix) -if versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3: +if versionData['MajorVersion'] == 4 and (versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3): code = code.replace( 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' From 89db7d20d8e41420063874b4f5b9889f722e2998 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 3 Jun 2021 17:39:45 +0300 Subject: [PATCH 166/430] Use ShaderCompileWorker instead of UE4Editor as a dummy target used to compile UBT This way our Dockerfile is compatible with both UE4 and UE5 Note that that since UE-4.22 [1] "Build.sh explicitly sets the target .NET framework version" is no longer true, so we can safely just drop this step when we end support for UE <= 4.22 [1]: https://github.com/EpicGames/UnrealEngine/commit/55cc21d08fe138fd9403d9bcca6c1092203010bf#diff-c23f687440063869ca47ea82dad1d5e3e71acac69b53de2a3b540c862fff3612L13 --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 1689ecf4..7fd147a8 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -29,7 +29,7 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -Clean +RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -Clean # Create an Installed Build of the Engine ARG BUILD_DDC From 63b49eaf0cbfa2b7b54be0c1def30806870f2c88 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 3 Jun 2021 17:45:50 +0300 Subject: [PATCH 167/430] Fix logic to avoid patching 4.24.0-4.24.2 This is a follow-up to commit d2aafaa57df38b623db3248ca236c7a290954974 --- ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index 66f0864e..b47bface 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -31,7 +31,7 @@ def writeFile(filename, data): # Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail # (See for details of the bug and its fix) -if versionData['MajorVersion'] == 4 and (versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3): +if versionData['MajorVersion'] != 4 or versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3: code = code.replace( 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' From e162eda1b7c709a1b4db2dc9f472cc75f7f7b97a Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 8 Jun 2021 15:45:58 +1000 Subject: [PATCH 168/430] Add `display` to NVIDIA_DRIVER_CAPABILITIES for ue4-full under Linux --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 35451c8c..aa4da63c 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -60,6 +60,7 @@ RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf # Enable X11 support (including the libraries required by CEF) and xvfb so we can create a dummy display if needed +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display RUN apt-get install -y --no-install-recommends \ libasound2 \ libatk1.0-0 \ From 7f64d630c552c9f28745dc00697877baa260ea77 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 9 Jun 2021 22:09:59 +1000 Subject: [PATCH 169/430] Fix #162 --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index c1496743..2b26e498 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Resources: - **Package on PyPI:** - **Related articles:** - **Unreal Containers community hub:** +- **Development channel on the Unreal Containers Community Discord server**: ## Contributing From 825f3f380c1167d68a6768e17e169d65f84a2ea5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 9 Jun 2021 23:29:36 +0300 Subject: [PATCH 170/430] Install .NET Core SDK for UE5 Note that 'Microsoft.NetCore.Component.SDK' component doesn't exist in VS Build Tools 2019, so it is a noop --- .../windows/install-prerequisites.bat | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 6a7e8817..51669fbe 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -15,6 +15,7 @@ set VISUAL_STUDIO_BUILD_NUMBER=%~1 @rem Install the Visual Studio Build Tools workloads and components we need @rem NOTE: We use the Visual Studio 2019 installer even for Visual Studio 2017 here because the old installer now breaks @rem NOTE: VS2019 Build Tools doesn't have 4.6.2 .NET SDK and what actually gets installed is 4.8 +@rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019. And it is actually *needed* only for UE5 curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ @@ -26,11 +27,11 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output --add Microsoft.VisualStudio.Workload.VCTools ^ --add Microsoft.VisualStudio.Workload.MSBuildTools ^ --add Microsoft.VisualStudio.Component.NuGet ^ - --add Microsoft.VisualStudio.Component.Roslyn.Compiler ^ --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ^ --add Microsoft.Net.Component.4.5.TargetingPack ^ - --add Microsoft.Net.ComponentGroup.4.6.2.DeveloperTools + --add Microsoft.Net.ComponentGroup.4.6.2.DeveloperTools ^ + --add Microsoft.NetCore.Component.SDK python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error From 55ee1534ba3bd86915554f8bf559be222edc54de Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 10 Jun 2021 17:43:06 +0300 Subject: [PATCH 171/430] Patch InstalledEngineFilters.xml in UE5 for Linux to add missing SetupDotnet.sh See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 3 +++ .../ue4-minimal/linux/patch-filters-xml.py | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 7fd147a8..462d3086 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -21,6 +21,9 @@ RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngin {% endif %} {% if (not disable_all_patches) and (not disable_buildgraph_patches) %} +COPY patch-filters-xml.py /tmp/patch-filters-xml.py +RUN python3 /tmp/patch-filters-xml.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineFilters.xml + # Patch the default settings in InstalledEngineBuild.xml and increase the output verbosity of the DDC generation step COPY patch-build-graph.py /tmp/patch-build-graph.py RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py new file mode 100644 index 00000000..c6afa462 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +import os, sys + +def readFile(filename): + with open(filename, 'rb') as f: + return f.read().decode('utf-8') + +def writeFile(filename, data): + with open(filename, 'wb') as f: + f.write(data.encode('utf-8')) + +filtersXml = sys.argv[1] +code = readFile(filtersXml) + +# Add missing SetupDotnet.sh in Unreal Engine 5.0.0-early-access-1 +# See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 +# and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 +dotnet = 'Engine/Binaries/ThirdParty/DotNet/Linux/...' +setup_dotnet = 'Engine/Build/BatchFiles/Linux/SetupDotnet.sh' +if dotnet in code and setup_dotnet not in code: + code = code.replace(dotnet, f'{dotnet}\n{setup_dotnet}') + +writeFile(filtersXml, code) From 8194357f4f792ec7867ccf9e6e05e5e0d418e6e0 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 10 Jun 2021 20:15:39 +0300 Subject: [PATCH 172/430] Use UnrealHeaderTool instead of UE4Editor to warmup UBT/Mono In Unreal Engine 5, there is no 'UE4Editor' target. Instead, there is 'UnrealEditor' target. In order to avoid conditional logic, we use just something that both UE4 and UE5 have in common. --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 462d3086..5fd9f5c2 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -32,7 +32,7 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -Clean +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild # Create an Installed Build of the Engine ARG BUILD_DDC @@ -86,7 +86,7 @@ LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} # Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` # (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) # See the comments on this issue for details, including the need to ensure $HOME is set correctly: -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -NoUBTMakefiles -SkipBuild && \ +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild && \ mkdir -p ./Engine/Programs/AutomationTool/Saved && \ chmod a+rw ./Engine/Programs/AutomationTool/Saved From b76b80bca4c49619827ae712628899766b9684a0 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 13 Jun 2021 13:23:46 +0300 Subject: [PATCH 173/430] Bump version to 0.0.86 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8536fab3..5fd62702 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.85' +__version__ = '0.0.86' From 40b764ce6a14f87d1110d6e9a06f0be6a97e9a6a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 18 Jun 2021 14:04:31 +0300 Subject: [PATCH 174/430] Do not pass VS2019 build graph arg to pre-VS2019 to avoid 'Unknown argument: VS2019' error Broken by commit 88d686fba1bc128103cbd0f3fdb2f94e73f0e7d6 --- ue4docker/build.py | 3 ++- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 5 ++--- ue4docker/infrastructure/BuildConfiguration.py | 12 +++++++----- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 4941d659..443116b0 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -286,7 +286,8 @@ def build(): # Build the minimal UE4 CI image, unless requested otherwise by the user buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: - builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs + config.uatBuildFlags) + buildGraphArg = ['--build-arg', 'BUILDGRAPH_ARGS=' + ' '.join(config.buildGraphArgs)] + builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg) builtImages.append('ue4-minimal') else: logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 5fd9f5c2..47bd6df0 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -35,9 +35,9 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild # Create an Installed Build of the Engine -ARG BUILD_DDC +ARG BUILDGRAPH_ARGS WORKDIR /home/ue4/UnrealEngine -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=$BUILD_DDC {{ buildgraph_args }} && \ +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true ${BUILDGRAPH_ARGS} {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index c1b6f5a9..fb644854 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -26,10 +26,9 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB {% endif %} # Create an Installed Build of the Engine -ARG BUILD_DDC -ARG USE_VS2019 +ARG BUILDGRAPH_ARGS WORKDIR C:\UnrealEngine -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC=%BUILD_DDC% -set:VS2019=%USE_VS2019% {{ buildgraph_args }} && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true %BUILDGRAPH_ARGS% {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) # Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 3c450d0c..db3f86b9 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -213,15 +213,14 @@ def __init__(self, parser, argv): raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title())) # Generate our flags for keeping or excluding components + self.buildGraphArgs = [ + '-set:WithDDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true') + ] self.exclusionFlags = [ - - '--build-arg', 'BUILD_DDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true'), '--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0), '--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0) ] - self.uatBuildFlags = [] - # If we're building Windows containers, generate our Windows-specific configuration settings if self.containerPlatform == 'windows': self._generateWindowsConfig() @@ -257,7 +256,10 @@ def _generateWindowsConfig(self): self.visualStudioBuildNumber = VisualStudio.BuildNumbers[self.visualStudio] # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac - self.uatBuildFlags += ['--build-arg', 'USE_VS2019={}'.format('true' if self.visualStudio == VisualStudio.VS2019 else 'false')] + # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. + # Otherwise, UAT errors out with "Unknown argument: VS2019". + if self.visualStudio != VisualStudio.VS2017: + self.buildGraphArgs += [f'-set:VS{self.visualStudio}=true'] # Determine base tag for the Windows release of the host system self.hostRelease = WindowsUtils.getWindowsRelease() From dbe54398500510cda0cc269199cc7624c5afdd8e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 17 Jun 2021 22:55:29 +0300 Subject: [PATCH 175/430] Simplify pdbcopy.exe workflow by directly installing Windows SDK Debugging Tools into ue4-build-prerequisites Tested by building 4.20.3 on Windows 2019 LTSC --- .../ue4-build-prerequisites/windows/Dockerfile | 8 ++------ .../windows/copy-pdbcopy.py | 16 ---------------- .../windows/install-prerequisites.bat | 6 ++---- 3 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 4519a362..38fa382c 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -39,11 +39,7 @@ RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` RUN curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" -# Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) -RUN choco install -y windows-sdk-10-version-1809-windbg -RUN C:\copy.py "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\pdbcopy.exe" C:\GatheredDlls\ - -# Copy our gathered DLLs (and pdbcopy.exe) into a clean image to reduce image size +# Copy our gathered DLLs into a clean image to reduce image size FROM ${BASEIMAGE} as prerequisites SHELL ["cmd", "/S", "/C"] COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ @@ -60,6 +56,6 @@ RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v Lo RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Install the rest of our build prerequisites and clean up afterwards to minimise image size -COPY buildtools-exitcode.py copy.py copy-pdbcopy.py install-prerequisites.bat C:\ +COPY buildtools-exitcode.py copy.py install-prerequisites.bat C:\ ARG VISUAL_STUDIO_BUILD_NUMBER RUN C:\install-prerequisites.bat %VISUAL_STUDIO_BUILD_NUMBER% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py deleted file mode 100644 index f1c2d812..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -import os, shutil - -# Copy pdbcopy.exe to the expected location for both newer Engine versions, -# as well as UE4 versions prior to the UE-51362 fix (https://issues.unrealengine.com/issue/UE-51362) -pdbcopyExe = 'C:\\Windows\\System32\\pdbcopy.exe' -destDirTemplate = 'C:\\Program Files (x86)\\MSBuild\\Microsoft\\VisualStudio\\v{}\\AppxPackage' -destDirs = [ - 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64', - destDirTemplate.format('12.0'), - destDirTemplate.format('14.0') -] -for destDir in destDirs: - destFile = os.path.join(destDir, 'pdbcopy.exe') - os.makedirs(destDir, exist_ok=True) - shutil.copy2(pdbcopyExe, destFile) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 51669fbe..f85a5b6d 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,6 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error -choco install -y curl vcredist2010 vcredist140 || goto :error +@rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine +choco install -y curl vcredist2010 vcredist140 windows-sdk-10-version-1809-windbg || goto :error choco install -y python --version=3.7.5 || goto :error @rem Reload our environment variables from the registry so the `git` command works @@ -35,9 +36,6 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error -@rem Copy pdbcopy.exe to the expected location(s) -python C:\copy-pdbcopy.py || goto :error - @rem Clean up any temp files generated during prerequisite installation rmdir /S /Q \\?\%TEMP% mkdir %TEMP% From 71679a43477ea6977d8289004d9394a12e4365b6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 21 Jun 2021 00:02:58 +0300 Subject: [PATCH 176/430] Revert "Workaround for xcopy issues under Windows Server 2016" This reverts commit 591c1265 It is believed that xcopy issue was caused by host/guest kernel mismatch. See https://github.com/moby/moby/issues/38425#issuecomment-850322791 Tested against 4.25.1 on Windows Server 2016 LTSC --- .../windows/Dockerfile | 6 +++--- .../ue4-build-prerequisites/windows/copy.py | 19 ------------------- ue4docker/exports/export_packages.py | 2 +- 3 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 38fa382c..94b54b1b 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -22,8 +22,8 @@ COPY *.dll C:\GatheredDlls\ # Verify that the DLL files copied from the host can be loaded by the container OS ARG HOST_BUILD RUN pip install pywin32 -COPY copy.py verify-host-dlls.py C:\ -RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ +COPY verify-host-dlls.py C:\ +RUN xcopy /y "C:\GatheredDlls\*.dll" C:\Windows\System32\ RUN python C:\verify-host-dlls.py %HOST_BUILD% C:\GatheredDlls # Gather the required DirectX runtime files, since Windows Server Core does not include them @@ -56,6 +56,6 @@ RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v Lo RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Install the rest of our build prerequisites and clean up afterwards to minimise image size -COPY buildtools-exitcode.py copy.py install-prerequisites.bat C:\ +COPY buildtools-exitcode.py install-prerequisites.bat C:\ ARG VISUAL_STUDIO_BUILD_NUMBER RUN C:\install-prerequisites.bat %VISUAL_STUDIO_BUILD_NUMBER% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py deleted file mode 100644 index 0f61a829..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -# This script is used to avoid issues with `xcopy.exe` under Windows Server 2016 (https://github.com/moby/moby/issues/38425) -import glob, os, shutil, sys - -# If the destination is an existing directory then expand wildcards in the source -destination = sys.argv[2] -if os.path.isdir(destination) == True: - sources = glob.glob(sys.argv[1]) -else: - sources = [sys.argv[1]] - -# Copy each of our source files/directories -for source in sources: - if os.path.isdir(source): - dest = os.path.join(destination, os.path.basename(source)) - shutil.copytree(source, dest) - else: - shutil.copy2(source, destination) - print('Copied {} to {}.'.format(source, destination), file=sys.stderr) diff --git a/ue4docker/exports/export_packages.py b/ue4docker/exports/export_packages.py index b9b890cc..5328ef01 100644 --- a/ue4docker/exports/export_packages.py +++ b/ue4docker/exports/export_packages.py @@ -65,7 +65,7 @@ def exportPackages(image, destination, extraArgs): 'windows': { 'rootCommand': ['timeout', '/t', '99999', '/nobreak'], 'mkdirCommand': ['cmd', '/S', '/C', 'mkdir'], - 'copyCommand': ['python', 'C:\\copy.py'], + 'copyCommand': ['xcopy', '/y'], 'dataDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\data', 'configDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\', From bfc535a48773288815ef34818f4b1cc9a92503f5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 23 Jun 2021 00:34:01 +0300 Subject: [PATCH 177/430] Revert "resolves #163 Reduce number of installed vcredists" This reverts commit 8de07d3548d16c66252ba31b8a834bd425f870e9. It turns out, older UE4 versions (at least, 4.20) need 2013 redists. Also, see #43 --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index f85a5b6d..9f3083d0 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,7 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error @rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y curl vcredist2010 vcredist140 windows-sdk-10-version-1809-windbg || goto :error +choco install -y curl vcredist-all windows-sdk-10-version-1809-windbg || goto :error choco install -y python --version=3.7.5 || goto :error @rem Reload our environment variables from the registry so the `git` command works From d4a2b4bee8221057afd077f1edd7703d2b8219cc Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 25 Jun 2021 16:40:14 +0300 Subject: [PATCH 178/430] Remove --pull-prerequisites flag Motivation: 1. We're not allowed to publish ue4-build-prerequisites for Windows because of VS Build Tools licensing. See https://github.com/adamrehn/ue4-docker/issues/167#issuecomment-863546435 2. ue4-build-prerequisites for Linux tends to become outdated, and is trivial to be built by user. See https://github.com/adamrehn/ue4-docker/issues/179#issuecomment-867272011 Additionally, fix ue4-build-prerequisites to respect UE4DOCKER_TAG_NAMESPACE env variable --- ue4docker/build.py | 31 +++++++++---------- .../dockerfiles/ue4-minimal/linux/Dockerfile | 3 +- .../ue4-minimal/windows/Dockerfile | 3 +- .../dockerfiles/ue4-source/linux/Dockerfile | 3 +- .../dockerfiles/ue4-source/windows/Dockerfile | 3 +- .../infrastructure/BuildConfiguration.py | 2 -- 6 files changed, 23 insertions(+), 22 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 443116b0..148087a0 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -146,17 +146,16 @@ def build(): # Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32 differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag - if config.pullPrerequisites == False and differentKernels == True and config.dlldir == config.defaultDllDir: + if differentKernels == True and config.dlldir == config.defaultDllDir: logger.error('Error: building images with a different kernel version than the host,', False) logger.error('but a custom DLL directory has not specified via the `-dlldir=DIR` arg.', False) logger.error('The DLL files will be the incorrect version and the container OS will', False) logger.error('refuse to load them, preventing the built Engine from running correctly.', False) sys.exit(1) - # Attempt to copy the required DLL files from the host system if we are building the prerequisites image - if config.pullPrerequisites == False: - for dll in WindowsUtils.requiredHostDlls(config.basetag): - shutil.copy2(join(config.dlldir, dll), join(builder.context('ue4-build-prerequisites'), dll)) + # Attempt to copy the required DLL files from the host system + for dll in WindowsUtils.requiredHostDlls(config.basetag): + shutil.copy2(join(config.dlldir, dll), join(builder.context('ue4-build-prerequisites'), dll)) # Ensure the Docker daemon is configured correctly requiredLimit = WindowsUtils.requiredSizeLimit() @@ -236,6 +235,10 @@ def build(): # Keep track of the images we've built builtImages = [] + + commonArgs = [ + '--build-arg', 'NAMESPACE={}'.format(GlobalConfiguration.getTagNamespace()), + ] # Compute the build options for the UE4 build prerequisites image # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) @@ -246,12 +249,9 @@ def build(): '--build-arg', 'VISUAL_STUDIO_BUILD_NUMBER=' + config.visualStudioBuildNumber, ] - # Build or pull the UE4 build prerequisites image (don't pull it if we're copying Dockerfiles to an output directory) - if config.layoutDir is None and config.pullPrerequisites == True: - builder.pull('adamrehn/ue4-build-prerequisites:{}'.format(config.prereqsTag)) - else: - builder.build('adamrehn/ue4-build-prerequisites', [config.prereqsTag], config.platformArgs + prereqsArgs) - builtImages.append('ue4-build-prerequisites') + # Build the UE4 build prerequisites image + builder.build('ue4-build-prerequisites', [config.prereqsTag], commonArgs + config.platformArgs + prereqsArgs) + builtImages.append('ue4-build-prerequisites') # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets secrets = {} @@ -269,16 +269,15 @@ def build(): '--build-arg', 'GIT_BRANCH={}'.format(config.branch), '--build-arg', 'VERBOSE_OUTPUT={}'.format('1' if config.verbose == True else '0') ] - builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + credentialArgs, secrets) + builder.build('ue4-source', mainTags, commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, secrets) builtImages.append('ue4-source') # Build the UE4 Engine source build image, unless requested otherwise by the user ue4BuildArgs = prereqConsumerArgs + [ '--build-arg', 'TAG={}'.format(mainTags[1]), - '--build-arg', 'NAMESPACE={}'.format(GlobalConfiguration.getTagNamespace()) ] if config.noEngine == False: - builder.build('ue4-engine', mainTags, config.platformArgs + ue4BuildArgs) + builder.build('ue4-engine', mainTags, commonArgs + config.platformArgs + ue4BuildArgs) builtImages.append('ue4-engine') else: logger.info('User specified `--no-engine`, skipping ue4-engine image build.') @@ -287,7 +286,7 @@ def build(): buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: buildGraphArg = ['--build-arg', 'BUILDGRAPH_ARGS=' + ' '.join(config.buildGraphArgs)] - builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg) + builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg) builtImages.append('ue4-minimal') else: logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') @@ -304,7 +303,7 @@ def build(): infrastructureFlags.extend(['--build-arg', 'CONAN_UE4CLI_VERSION={}'.format(config.conanUe4cliVersion)]) # Build the image - builder.build('ue4-full', mainTags, config.platformArgs + ue4BuildArgs + infrastructureFlags) + builder.build('ue4-full', mainTags, commonArgs + config.platformArgs + ue4BuildArgs + infrastructureFlags) builtImages.append('ue4-full') else: logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 47bd6df0..622c9957 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -67,7 +67,8 @@ RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine {% if combine %} FROM prerequisites as minimal {% else %} -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +ARG NAMESPACE +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} # Copy the Installed Build files from the builder image diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index fb644854..b334b8b8 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -47,7 +47,8 @@ RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\C {% if combine %} FROM prerequisites as minimal {% else %} -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +ARG NAMESPACE +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine WORKDIR C:\UnrealEngine diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 41c531dd..8402d83a 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -1,8 +1,9 @@ {% if combine %} FROM prerequisites as source {% else %} +ARG NAMESPACE ARG PREREQS_TAG -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} {% if not disable_all_patches %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 27661ad9..337117b2 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -2,8 +2,9 @@ {% if combine %} FROM prerequisites as source {% else %} +ARG NAMESPACE ARG PREREQS_TAG -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} {% if not disable_all_patches %} diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index db3f86b9..3d2e7d2f 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -85,7 +85,6 @@ def addArguments(parser): parser.add_argument('--linux', action='store_true', help='Build Linux container images under Windows') parser.add_argument('--rebuild', action='store_true', help='Rebuild images even if they already exist') parser.add_argument('--dry-run', action='store_true', help='Print `docker build` commands instead of running them') - parser.add_argument('--pull-prerequisites', action='store_true', help='Pull the ue4-build-prerequisites image from Docker Hub instead of building it') parser.add_argument('--no-engine', action='store_true', help='Don\'t build the ue4-engine image') parser.add_argument('--no-minimal', action='store_true', help='Don\'t build the ue4-minimal image') parser.add_argument('--no-full', action='store_true', help='Don\'t build the ue4-full image') @@ -160,7 +159,6 @@ def __init__(self, parser, argv): self.containerPlatform = 'windows' if platform.system() == 'Windows' and self.args.linux == False else 'linux' self.dryRun = self.args.dry_run self.rebuild = self.args.rebuild - self.pullPrerequisites = self.args.pull_prerequisites self.noEngine = self.args.no_engine self.noMinimal = self.args.no_minimal self.noFull = self.args.no_full From c559400f2aae71ef19ba44dab9dc6d105e52dacc Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 26 Jun 2021 22:14:35 +0300 Subject: [PATCH 179/430] Migrate CI to GitHub Actions resolves #178 --- .github/dependabot.yml | 4 ++++ .github/workflows/ci.yml | 22 ++++++++++++++++++++++ .travis.yml | 16 ---------------- 3 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e46a75d1..e6c28dc8 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,3 +6,7 @@ updates: directory: "/" schedule: interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..93287709 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: CI +on: [push, pull_request] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.7 + - name: Install dependencies + run: pip install setuptools wheel + - name: Build package + run: python setup.py sdist bdist_wheel + - name: Publish to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7990a448..00000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -if: tag IS present -dist: xenial -language: python -python: - - "3.7" -install: - - pip install wheel -script: echo "No build-time tests to run." -deploy: - provider: pypi - user: adamrehn - distributions: bdist_wheel - on: - tags: true - password: - secure: On10RovRYLb2S9TmoqzHCnHF1Bld5UJSbnoMCWytjwXKtp/+FphcT5htSNBKUpoBbjxdCT0we0Vy1GtGJi8auNZa+fhcegIikXVDg5iBn2UvgLVWrWZBQWDPCjkdL7fCij5MSAjJxBEuYRcfeGTM76wyYfYaO4omvNE96CVoWmkD3Ev0h1wrxeUoy7mAQ7Qy/46gaQ73LH0YnhaZ1s5frtESkoFsfqNwkrFY14Iu2gfnDZB6WDVqS00oOXhjiAjEyyGfCk6eW4GyUdVa2dWaapHDVeruvr6ICBj5DOXpf37dsIZvjqpWAqO76+4rbJ8tyPkjsr+DyQ2hq7a1Lq+TWlVLh78FkVoQATDRy/Y1Ai+X/nhnFkNrrIHHaS1B82Cnyq2XWLp/iUoxAZLTP7q11L4vrpOccSx8Ed5LaPoucTNvEVyM//eD8J8Z12OCue+ZAmWdq1E1slkfeBcp0bAkfTQ/9K0HJ7t0fCUj3+Y4DbSLB3H7zT0Ev6+roZWHzcwPf1Wct+Qwe2RdTZRdAt0C/tdSCTgB82/GrUWnaKLO4f2Flyr2BOUFMZDnOYR5MOs9826n8RnDz9mXDSqEZWOy01jjMS7H7jvKBQQwbVllpfxZqkd+JXumsf7k2l4nV3ETIQuU4NYExoTSe+Y32zP8Zgbe+Y9V0kte8a4m9FO7HvI= From a7111bdb1f4fb0c33d0eb47f916ef5556eae7fdc Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 21 Jul 2021 12:33:05 +0300 Subject: [PATCH 180/430] Bump version to 0.0.87 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 5fd62702..d2f4572b 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.86' +__version__ = '0.0.87' From 4e6e64bf69de18c9b7916351e4ec132551bd98db Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 11 Aug 2021 15:25:35 +1000 Subject: [PATCH 181/430] Split engine components into separate filesystem layers --- ue4docker/build.py | 3 +- .../dockerfiles/ue4-minimal/linux/Dockerfile | 28 +++++---- .../ue4-minimal/linux/exclude-components.py | 43 -------------- .../ue4-minimal/linux/split-components.py | 58 +++++++++++++++++++ .../ue4-minimal/windows/Dockerfile | 29 ++++++---- .../ue4-minimal/windows/exclude-components.py | 43 -------------- .../ue4-minimal/windows/split-components.py | 58 +++++++++++++++++++ .../infrastructure/BuildConfiguration.py | 20 +++---- 8 files changed, 160 insertions(+), 122 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py create mode 100644 ue4docker/dockerfiles/ue4-minimal/linux/split-components.py delete mode 100644 ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py create mode 100644 ue4docker/dockerfiles/ue4-minimal/windows/split-components.py diff --git a/ue4docker/build.py b/ue4docker/build.py index 148087a0..5166525f 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -285,8 +285,7 @@ def build(): # Build the minimal UE4 CI image, unless requested otherwise by the user buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: - buildGraphArg = ['--build-arg', 'BUILDGRAPH_ARGS=' + ' '.join(config.buildGraphArgs)] - builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + config.exclusionFlags + ue4BuildArgs + buildGraphArg) + builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + ue4BuildArgs) builtImages.append('ue4-minimal') else: logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 622c9957..3d7c85a6 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -35,16 +35,13 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild # Create an Installed Build of the Engine -ARG BUILDGRAPH_ARGS WORKDIR /home/ue4/UnrealEngine -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true ${BUILDGRAPH_ARGS} {{ buildgraph_args }} && \ +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC -# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size -COPY exclude-components.py /tmp/exclude-components.py -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -RUN python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES +# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers +COPY split-components.py /tmp/split-components.py +RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini @@ -73,15 +70,22 @@ FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} # Copy the Installed Build files from the builder image COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine +{% if excluded_components.ddc == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DDC /home/ue4/UnrealEngine +{% endif %} +{% if excluded_components.debug == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbols /home/ue4/UnrealEngine +{% endif %} +{% if excluded_components.templates == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine +{% endif %} WORKDIR /home/ue4/UnrealEngine {% if not disable_labels %} # Add labels to the built image to identify which components (if any) were excluded from the build that it contains -# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} -LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} +LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %} {% endif %} # Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py deleted file mode 100644 index 7420baae..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -import glob, json, os, shutil, sys -from os.path import join - -# Logs a message to stderr -def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() - -# Reads the contents of a file -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -rootDir = sys.argv[1] -version = json.loads(readFile(join(rootDir, 'Engine', 'Build', 'Build.version'))) - -# Determine if we are excluding debug symbols -truncateDebug = len(sys.argv) > 2 and sys.argv[2] == '1' -if truncateDebug == True: - - # Remove all *.debug and *.sym files - log('User opted to exclude debug symbols, removing all *.debug and *.sym files.') - log('Scanning for debug symbols in directory {}...'.format(rootDir)) - symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True) - for symbolFile in symbolFiles: - log('Removing debug symbol file {}...'.format(symbolFile)) - try: - os.unlink(symbolFile) - except: - log(' Warning: failed to remove debug symbol file {}.'.format(symbolFile)) - -# Determine if we are excluding the Engine's template projects and samples -excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' -if excludeTemplates == True: - log('User opted to exclude templates and samples.') - for subdir in ['FeaturePacks', 'Samples', 'Templates']: - log('Removing {} directory...'.format(subdir)) - try: - shutil.rmtree(join(rootDir, subdir)) - except: - log(' Warning: failed to remove {} directory...'.format(subdir)) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py new file mode 100644 index 00000000..4807e06b --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import glob, os, shutil, sys +from os.path import basename, dirname, exists, join + +# Logs a message to stderr +def log(message): + print(message, file=sys.stderr) + sys.stderr.flush() + +# Extracts the files and directories for the specified component and moves them to a separate output directory +def extractComponent(inputDir, outputDir, component, description, items): + + # Print progress output + log('\nExtracting {}...'.format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + + # Verify that the item exists + if not exists(item): + log('Skipping non-existent item: {}'.format(item)) + continue + + # Print progress output + log('Moving: {}'.format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move( + item, + join(parent, basename(item)) + ) + +# Retrieve the path to the root directory of the Installed Build +rootDir = sys.argv[1] + +# Retrieve the path to the root output directory for extracted components and ensure it exists +outputDir = sys.argv[2] +os.makedirs(outputDir, exist_ok=True) + +# Extract the DDC +ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')] +extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc) + +# Extract debug symbols +symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True) +extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles) + +# Extract template projects and samples +subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']] +extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index b334b8b8..2aeca816 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -28,14 +28,12 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB # Create an Installed Build of the Engine ARG BUILDGRAPH_ARGS WORKDIR C:\UnrealEngine -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true %BUILDGRAPH_ARGS% {{ buildgraph_args }} && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) -# Determine if we are removing debug symbols and/or template projects in order to reduce the final container image size -COPY exclude-components.py C:\exclude-components.py -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -RUN python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% +# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers +COPY split-components.py C:\split-components.py +RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini @@ -50,14 +48,23 @@ FROM prerequisites as minimal ARG NAMESPACE FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} + +# Copy the Installed Build files from the builder image COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine +{% if excluded_components.ddc == false %} +COPY --from=builder C:\UnrealEngine\Components\DDC C:\UnrealEngine +{% endif %} +{% if excluded_components.debug == false %} +COPY --from=builder C:\UnrealEngine\Components\DebugSymbols C:\UnrealEngine +{% endif %} +{% if excluded_components.templates == false %} +COPY --from=builder C:\UnrealEngine\Components\TemplatesAndSamples C:\UnrealEngine +{% endif %} WORKDIR C:\UnrealEngine {% if not disable_labels %} # Add labels to the built image to identify which components (if any) were excluded from the build that it contains -# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} -LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} +LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %} {% endif %} diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py deleted file mode 100644 index c92c43d0..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -import glob, json, os, shutil, sys -from os.path import join - -# Logs a message to stderr -def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() - -# Reads the contents of a file -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -rootDir = sys.argv[1] -version = json.loads(readFile(join(rootDir, 'Engine', 'Build', 'Build.version'))) - -# Determine if we are excluding debug symbols -excludeDebug = len(sys.argv) > 2 and sys.argv[2] == '1' -if excludeDebug == True: - - # Remove all PDB files to save space - log('User opted to exclude debug symbols, removing all PDB files.') - log('Scanning for PDB files in directory {}...'.format(rootDir)) - pdbFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True) - for pdbFile in pdbFiles: - log('Removing PDB file {}...'.format(pdbFile)) - try: - os.remove(pdbFile) - except: - log(' Warning: failed to remove PDB file {}.'.format(pdbFile)) - -# Determine if we are excluding the Engine's template projects and samples -excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' -if excludeTemplates == True: - log('User opted to exclude templates and samples.') - for subdir in ['FeaturePacks', 'Samples', 'Templates']: - log('Removing {} directory...'.format(subdir)) - try: - shutil.rmtree(join(rootDir, subdir)) - except: - log(' Warning: failed to remove {} directory...'.format(subdir)) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py new file mode 100644 index 00000000..116e7fc2 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +import glob, os, shutil, sys +from os.path import basename, dirname, exists, join + +# Logs a message to stderr +def log(message): + print(message, file=sys.stderr) + sys.stderr.flush() + +# Extracts the files and directories for the specified component and moves them to a separate output directory +def extractComponent(inputDir, outputDir, component, description, items): + + # Print progress output + log('\nExtracting {}...'.format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + + # Verify that the item exists + if not exists(item): + log('Skipping non-existent item: {}'.format(item)) + continue + + # Print progress output + log('Moving: {}'.format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move( + item, + join(parent, basename(item)) + ) + +# Retrieve the path to the root directory of the Installed Build +rootDir = sys.argv[1] + +# Retrieve the path to the root output directory for extracted components and ensure it exists +outputDir = sys.argv[2] +os.makedirs(outputDir, exist_ok=True) + +# Extract the DDC +ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')] +extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc) + +# Extract debug symbols +symbolFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True) +extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles) + +# Extract template projects and samples +subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']] +extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 3d2e7d2f..c575370c 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -210,15 +210,13 @@ def __init__(self, parser, argv): if self.opts.get('credential_mode', 'endpoint') not in validCredentialModes: raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title())) - # Generate our flags for keeping or excluding components - self.buildGraphArgs = [ - '-set:WithDDC={}'.format('false' if ExcludedComponent.DDC in self.excludedComponents else 'true') - ] - self.exclusionFlags = [ - '--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0), - '--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0) - ] - + # Generate Jinja context values for keeping or excluding components + self.opts['excluded_components'] = { + 'ddc': ExcludedComponent.DDC in self.excludedComponents, + 'debug': ExcludedComponent.Debug in self.excludedComponents, + 'templates': ExcludedComponent.Templates in self.excludedComponents + } + # If we're building Windows containers, generate our Windows-specific configuration settings if self.containerPlatform == 'windows': self._generateWindowsConfig() @@ -257,8 +255,8 @@ def _generateWindowsConfig(self): # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. # Otherwise, UAT errors out with "Unknown argument: VS2019". if self.visualStudio != VisualStudio.VS2017: - self.buildGraphArgs += [f'-set:VS{self.visualStudio}=true'] - + self.opts['buildgraph_args'] = self.opts.get('buildgraph_args', '') + f' -set:VS{self.visualStudio}=true' + # Determine base tag for the Windows release of the host system self.hostRelease = WindowsUtils.getWindowsRelease() self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease) From d25fd49dff2d46d6b8fe761fd9c3539c43968767 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 18 Aug 2021 22:31:45 +1000 Subject: [PATCH 182/430] Bump version to 0.0.88 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index d2f4572b..7464c991 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.87' +__version__ = '0.0.88' From 664359a6c5240ebc3b3d15f5c1ecc4d456fd6cd1 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 20 Aug 2021 00:06:27 +0300 Subject: [PATCH 183/430] Add release-drafter configuration (#189) Resolves #188 --- .github/release-drafter.yml | 8 ++++++++ .github/workflows/release-drafter.yml | 13 +++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 .github/release-drafter.yml create mode 100644 .github/workflows/release-drafter.yml diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..79c3ed97 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,8 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +version-resolver: + default: patch +template: | + ## Changes + + $CHANGES diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 00000000..86e477cd --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,13 @@ +name: Release Drafter + +on: + push: + branches: [ master ] + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From b52b9cbd4ffe2280f5461efa54dd9276591ce471 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 21 Aug 2021 14:25:39 +0300 Subject: [PATCH 184/430] Use mcr.microsoft.com/windows image as a source for DLLs instead of host system (#187) This commit: 1. Drops Windows 2016 LTSC support. Now the oldest supported Windows version is 2019 LTSC a.k.a. 1809 2. Stops copying DLLs from host system and instead uses mcr.microsoft.com/windows. This change allows building `ue4-build-prerequisites` even if host system version doesn't match. As a consequence of this change, `ue4-docker build` no longer has `-dlldir` command-line argument. 3. Extends list of copied DLLs to allow usage of Windows Advanced Rasterization Platform (WARP) in images derived from `ue4-build-prerequisites` image out of the box. Resolves #165 --- ue4docker/build.py | 19 +----- .../windows/Dockerfile | 60 +++++++----------- .../windows/install-prerequisites.bat | 20 ++++++ .../windows/verify-host-dlls.py | 49 --------------- .../infrastructure/BuildConfiguration.py | 12 +--- ue4docker/infrastructure/WindowsUtils.py | 61 ++++++++----------- ue4docker/setup_cmd.py | 43 ------------- 7 files changed, 72 insertions(+), 192 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py diff --git a/ue4docker/build.py b/ue4docker/build.py index 5166525f..359d3c90 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -112,12 +112,12 @@ def build(): # Provide the user with feedback so they are aware of the Windows-specific values being used logger.info('WINDOWS CONTAINER SETTINGS', False) logger.info('Isolation mode: {}'.format(config.isolation), False) - logger.info('Base OS image tag: {}'.format(config.basetag), False) + logger.info('Base OS image: {}'.format(config.baseImage), False) + logger.info('Dll source image: {}'.format(config.dllSrcImage), False) logger.info('Host OS: {}'.format(WindowsUtils.systemString()), False) logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Visual Studio: {}'.format(config.visualStudio), False) - logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) if not config.ignoreEOL and WindowsUtils.isEndOfLifeWindowsVersion(config.basetag): @@ -144,19 +144,6 @@ def build(): logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') sys.exit(1) - # Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32 - differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag - if differentKernels == True and config.dlldir == config.defaultDllDir: - logger.error('Error: building images with a different kernel version than the host,', False) - logger.error('but a custom DLL directory has not specified via the `-dlldir=DIR` arg.', False) - logger.error('The DLL files will be the incorrect version and the container OS will', False) - logger.error('refuse to load them, preventing the built Engine from running correctly.', False) - sys.exit(1) - - # Attempt to copy the required DLL files from the host system - for dll in WindowsUtils.requiredHostDlls(config.basetag): - shutil.copy2(join(config.dlldir, dll), join(builder.context('ue4-build-prerequisites'), dll)) - # Ensure the Docker daemon is configured correctly requiredLimit = WindowsUtils.requiredSizeLimit() if DockerUtils.maxsize() < requiredLimit: @@ -245,7 +232,7 @@ def build(): prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] if config.containerPlatform == 'windows': prereqsArgs = prereqsArgs + [ - '--build-arg', 'HOST_BUILD=' + str(WindowsUtils.getWindowsBuild()), + '--build-arg', 'DLLSRCIMAGE=' + config.dllSrcImage, '--build-arg', 'VISUAL_STUDIO_BUILD_NUMBER=' + config.visualStudioBuildNumber, ] diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 94b54b1b..5c1470bc 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -1,48 +1,30 @@ # escape=` ARG BASEIMAGE -FROM ${BASEIMAGE} AS dlls -SHELL ["cmd", "/S", "/C"] - -{% if not disable_labels %} -# Include our sentinel so `ue4-docker clean` can find this intermediate image -LABEL com.adamrehn.ue4-docker.sentinel="1" -{% endif %} - -# Create a directory in which to gather the DLL files we need -RUN mkdir C:\GatheredDlls +ARG DLLSRCIMAGE -# Install 7-Zip, curl, and Python using Chocolatey -# (Note that these need to be separate RUN directives for `choco` to work) -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" -RUN choco install -y 7zip curl && choco install -y python --version=3.7.5 - -# Copy the required DirectSound/DirectDraw and OpenGL DLL files from the host system (since these ship with Windows and don't have installers) -COPY *.dll C:\GatheredDlls\ - -# Verify that the DLL files copied from the host can be loaded by the container OS -ARG HOST_BUILD -RUN pip install pywin32 -COPY verify-host-dlls.py C:\ -RUN xcopy /y "C:\GatheredDlls\*.dll" C:\Windows\System32\ -RUN python C:\verify-host-dlls.py %HOST_BUILD% C:\GatheredDlls +FROM ${DLLSRCIMAGE} as dlls -# Gather the required DirectX runtime files, since Windows Server Core does not include them -RUN curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe -RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` - expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\GatheredDlls\ && ` - expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\GatheredDlls\ - -# Gather the Vulkan runtime library -RUN curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip -RUN 7z e %TEMP%\vulkan-runtime-components.zip -oC:\GatheredDlls -y "*\x64\vulkan-1.dll" - -# Copy our gathered DLLs into a clean image to reduce image size FROM ${BASEIMAGE} as prerequisites SHELL ["cmd", "/S", "/C"] -COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ + +# Gather the system DLLs that we need from the full Windows base image +COPY --from=dlls ` + C:\Windows\System32\avicap32.dll ` + C:\Windows\System32\avrt.dll ` + C:\Windows\System32\d3d10warp.dll ` + C:\Windows\System32\D3DSCache.dll ` + C:\Windows\System32\dsound.dll ` + C:\Windows\System32\dxva2.dll ` + C:\Windows\System32\glu32.dll ` + C:\Windows\System32\mf.dll ` + C:\Windows\System32\mfplat.dll ` + C:\Windows\System32\mfplay.dll ` + C:\Windows\System32\mfreadwrite.dll ` + C:\Windows\System32\msdmo.dll ` + C:\Windows\System32\msvfw32.dll ` + C:\Windows\System32\opengl32.dll ` + C:\Windows\System32\ResourcePolicyClient.dll ` + C:\Windows\System32\ {% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 9f3083d0..13277be3 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -11,6 +11,26 @@ call refreshenv @rem Forcibly disable the git credential manager git config --system credential.helper "" || goto :error +@rem Gather the required DirectX runtime files, since Windows Server Core does not include them +curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe && ^ +start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ^ +expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\Windows\System32\ && ^ +expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\Windows\System32\ && ^ +expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\Windows\System32\ && ^ +expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\Windows\System32\ && ^ +expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\Windows\System32\ || goto :error + +@rem Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) +curl --progress -L "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" --output %TEMP%\dxc.zip && ^ +powershell -Command "Expand-Archive -Path \"$env:TEMP\dxc.zip\" -DestinationPath $env:TEMP" && ^ +xcopy /y %TEMP%\bin\x64\dxcompiler.dll C:\Windows\System32\ && ^ +xcopy /y %TEMP%\bin\x64\dxil.dll C:\Windows\System32\ || goto :error + +@rem Gather the Vulkan runtime library +curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip && ^ +powershell -Command "Expand-Archive -Path \"$env:TEMP\vulkan-runtime-components.zip\" -DestinationPath $env:TEMP" && ^ +powershell -Command "Copy-Item -Path \"*\x64\vulkan-1.dll\" -Destination C:\Windows\System32" || goto :error + set VISUAL_STUDIO_BUILD_NUMBER=%~1 @rem Install the Visual Studio Build Tools workloads and components we need diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py deleted file mode 100644 index 757a8442..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ /dev/null @@ -1,49 +0,0 @@ -import glob, os, sys, win32api - - -# Adapted from the code in this SO answer: -def getDllVersion(dllPath): - info = win32api.GetFileVersionInfo(dllPath, '\\') - return '{:d}.{:d}.{:d}.{:d}'.format( - info['FileVersionMS'] // 65536, - info['FileVersionMS'] % 65536, - info['FileVersionLS'] // 65536, - info['FileVersionLS'] % 65536 - ) - - -# Print the host and container OS build numbers -print('Host OS build number: {}'.format(sys.argv[1])) -print('Container OS build number: {}'.format(sys.getwindowsversion().build)) -sys.stdout.flush() - -# Verify each DLL file in the directory specified by our command-line argument -dlls = glob.glob(os.path.join(sys.argv[2], '*.dll')) -for dll in dlls: - - # Attempt to retrieve the version number of the DLL - dllName = os.path.basename(dll) - try: - dllVersion = getDllVersion(dll) - except Exception as err: - print('\nError: could not read the version string from the DLL file "{}".'.format(dllName), file=sys.stderr) - print('Please ensure the DLLs copied from the host are valid DLL files.', file=sys.stderr) - print('\nError details:', file=sys.stderr) - print(err, file=sys.stderr) - sys.exit(1) - - # Print the DLL details - print('Found host DLL file "{}" with version string "{}".'.format(dllName, dllVersion)) - sys.stdout.flush() - - # Determine if the container OS will load the DLL - try: - handle = win32api.LoadLibrary(dll) - win32api.FreeLibrary(handle) - except Exception as err: - print('\nError: the container OS cannot load the DLL file "{}".'.format(dllName), file=sys.stderr) - print('This typically indicates that the DLL is from a newer version of Windows.', file=sys.stderr) - print('Please ensure the DLLs copied from the host match the container OS version.', file=sys.stderr) - print('\nError details:', file=sys.stderr) - print(err, file=sys.stderr) - sys.exit(1) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index c575370c..38db5fba 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -100,7 +100,6 @@ def addArguments(parser): parser.add_argument('-branch', default=None, help='Set the custom branch/tag to clone when "custom" is specified as the release value') parser.add_argument('-isolation', default=None, help='Set the isolation mode to use for Windows containers (process or hyperv)') parser.add_argument('-basetag', default=None, help='Windows Server Core base image tag to use for Windows containers (default is the host OS version)') - parser.add_argument('-dlldir', default=None, help='Set the directory to copy required Windows DLLs from (default is the host System32 directory)') parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images') parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image') @@ -235,13 +234,6 @@ def describeExcludedComponents(self): return sorted([ExcludedComponent.description(component) for component in self.excludedComponents]) def _generateWindowsConfig(self): - - # Store the path to the directory containing our required Windows DLL files - hostSysnative = os.path.join(os.environ['SystemRoot'], 'Sysnative') - hostSystem32 = os.path.join(os.environ['SystemRoot'], 'System32') - self.defaultDllDir = hostSysnative if os.path.exists(hostSysnative) else hostSystem32 - self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir - self.visualStudio = self.args.visual_studio if not self.custom: @@ -264,6 +256,7 @@ def _generateWindowsConfig(self): # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag + self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) self.prereqsTag = self.basetag + '-vs' + self.visualStudio # Verify that any user-specified base tag is valid @@ -281,9 +274,8 @@ def _generateWindowsConfig(self): # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag - hostSupportsProcess = WindowsUtils.supportsProcessIsolation() dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') - if not differentKernels and hostSupportsProcess and dockerSupportsProcess: + if not differentKernels and dockerSupportsProcess: self.isolation = 'process' else: self.isolation = DockerUtils.info()['Isolation'] diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 8212382f..c4bdbdf1 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -7,18 +7,21 @@ class WindowsUtils(object): + # The oldest Windows build we support + _minimumRequiredBuild = 17763 + # The latest Windows build version we recognise as a non-Insider build _latestReleaseBuild = 19042 # The list of Windows Server Core base image tags that we recognise, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019', '1903', '1909', '2004', '20H2'] + _validTags = ['ltsc2019', '1903', '1909', '2004', '20H2'] # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) _blacklistedReleases = ['1903', '1909'] # The list of Windows Server Core container image releases that are unsupported due to having reached EOL - _eolReleases = ['1703', '1709', '1803', '1903', '1909'] + _eolReleases = ['1903', '1909'] @staticmethod def _getVersionRegKey(subkey : str) -> str: @@ -32,16 +35,6 @@ def _getVersionRegKey(subkey : str) -> str: winreg.CloseKey(key) return value[0] - @staticmethod - def requiredHostDlls(basetag: str) -> [str]: - ''' - Returns the list of required host DLL files for the specified container image base tag - ''' - - # `ddraw.dll` is only required under Windows Server 2016 version 1607 - common = ['dsound.dll', 'opengl32.dll', 'glu32.dll'] - return ['ddraw.dll'] + common if basetag == 'ltsc2016' else common - @staticmethod def requiredSizeLimit() -> float: ''' @@ -52,12 +45,9 @@ def requiredSizeLimit() -> float: @staticmethod def minimumRequiredBuild() -> int: ''' - Returns the minimum required version of Windows 10 / Windows Server, which is release 1607 - - (1607 is the first build to support Windows containers, as per: - ) + Returns the minimum required version of Windows 10 / Windows Server ''' - return 14393 + return WindowsUtils._minimumRequiredBuild @staticmethod def systemString() -> str: @@ -76,7 +66,13 @@ def getWindowsRelease() -> str: ''' Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system ''' - return WindowsUtils._getVersionRegKey('ReleaseId') + try: + # Starting with Windows 20H2 (also known as 2009), Microsoft stopped updating ReleaseId + # and instead updates DisplayVersion + return WindowsUtils._getVersionRegKey('DisplayVersion') + except FileNotFoundError: + # Fallback to ReleaseId for pre-20H2 releases that didn't have DisplayVersion + return WindowsUtils._getVersionRegKey('ReleaseId') @staticmethod def getWindowsBuild() -> int: @@ -131,15 +127,19 @@ def getReleaseBaseTag(release: str) -> str: # This lookup table is based on the list of valid tags from return { - '1709': '1709', - '1803': '1803', '1809': 'ltsc2019', - '1903': '1903', - '1909': '1909', - '2004': '2004', - '2009': '20H2', - '20H2': '20H2' - }.get(release, 'ltsc2016') + }.get(release, release) + + @staticmethod + def getDllSrcImage(basetag: str) -> str: + ''' + Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image + ''' + tag = { + 'ltsc2019': '1809', + }.get(basetag, basetag) + + return f'mcr.microsoft.com/windows:{tag}' @staticmethod def getValidBaseTags() -> [str]: @@ -161,12 +161,3 @@ def isNewerBaseTag(older: str, newer: str) -> bool: Determines if the base tag `newer` is chronologically newer than the base tag `older` ''' return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older) - - @staticmethod - def supportsProcessIsolation() -> bool: - ''' - Determines whether the Windows host system supports process isolation for containers - - @see https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container - ''' - return WindowsUtils.isWindowsServer() or WindowsUtils.getWindowsBuild() >= 17763 diff --git a/ue4docker/setup_cmd.py b/ue4docker/setup_cmd.py index b8f7dac9..4a8e0c35 100644 --- a/ue4docker/setup_cmd.py +++ b/ue4docker/setup_cmd.py @@ -94,49 +94,6 @@ def _setupWindowsServer(): else: print('Firewall rule for credential endpoint is already configured.') - - # Determine if the host system is Windows Server Core and lacks the required DLL files for building our containers - hostRelease = WindowsUtils.getWindowsRelease() - requiredDLLs = WindowsUtils.requiredHostDlls(hostRelease) - dllDir = os.path.join(os.environ['SystemRoot'], 'System32') - existing = [dll for dll in requiredDLLs if os.path.exists(os.path.join(dllDir, dll))] - if len(existing) != len(requiredDLLs): - - # Determine if we can extract DLL files from the full Windows base image (version 1809 and newer only) - tags = requests.get('https://mcr.microsoft.com/v2/windows/tags/list').json()['tags'] - if hostRelease in tags: - - # Pull the full Windows base image with the appropriate tag if it does not already exist - image = 'mcr.microsoft.com/windows:{}'.format(hostRelease) - print('Pulling full Windows base image "{}"...'.format(image)) - subprocess.run(['docker', 'pull', image], check=True) - - # Start a container from which we will copy the DLL files, bind-mounting our DLL destination directory - print('Starting a container to copy DLL files from...') - mountPath = 'C:\\dlldir' - container = DockerUtils.start( - image, - ['timeout', '/t', '99999', '/nobreak'], - mounts = [docker.types.Mount(mountPath, dllDir, 'bind')], - stdin_open = True, - tty = True, - remove = True - ) - - # Copy the DLL files to the host - print('Copying DLL files to the host system...') - DockerUtils.execMultiple(container, [['xcopy', '/y', os.path.join(dllDir, dll), mountPath + '\\'] for dll in requiredDLLs]) - - # Stop the container - print('Stopping the container...') - container.stop() - - else: - print('The following DLL files will need to be manually copied into {}:'.format(dllDir)) - print('\n'.join(['- {}'.format(dll) for dll in requiredDLLs if dll not in existing])) - - else: - print('All required DLL files are already present on the host system.') def setup(): From bb2acecb2baa5a79f5edcb45ca588e9516791a8b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 12:20:49 +0300 Subject: [PATCH 185/430] Stop relying on ReleaseId/DisplayVersion registry keys (#191) Instead, we now use Windows build numbers to identify host system. This commit also allows to use any existing Windows Server Core base tag. This is a preparation step to add support for Windows Server 2022 LTSC, whose DisplayVersion (21H2) collides with Windows 10 21H2 and Windows 11 21H2. --- ue4docker/build.py | 15 +-- ue4docker/diagnostics/base.py | 10 +- ue4docker/diagnostics/diagnostic_20gig.py | 2 +- ue4docker/diagnostics/diagnostic_8gig.py | 2 +- ue4docker/diagnostics/diagnostic_network.py | 2 +- .../infrastructure/BuildConfiguration.py | 19 ++-- ue4docker/infrastructure/WindowsUtils.py | 98 ++++++------------- 7 files changed, 53 insertions(+), 95 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 359d3c90..c9733d7a 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -119,16 +119,8 @@ def build(): logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) logger.info('Visual Studio: {}'.format(config.visualStudio), False) - # Verify that the specified base image tag is not a release that has reached End Of Life (EOL) - if not config.ignoreEOL and WindowsUtils.isEndOfLifeWindowsVersion(config.basetag): - logger.error('Error: detected EOL base OS image tag: {}'.format(config.basetag), False) - logger.error('This version of Windows has reached End Of Life (EOL), which means', False) - logger.error('Microsoft no longer supports or maintains container base images for it.', False) - logger.error('You will need to use a base image tag for a supported version of Windows.', False) - sys.exit(1) - # Verify that the host OS is not a release that is blacklisted due to critical bugs - if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsVersion() == True: + if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsHost() == True: logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemString()), False) logger.error('', False) logger.error('This version of Windows contains one or more critical bugs that', False) @@ -140,9 +132,12 @@ def build(): sys.exit(1) # Verify that the user is not attempting to build images with a newer kernel version than the host OS - if WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag): + newer_check = WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag) + if newer_check: logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') sys.exit(1) + elif newer_check is None: + logger.info('Warning: unable to determine whether host system is new enough to use specified base tag') # Ensure the Docker daemon is configured correctly requiredLimit = WindowsUtils.requiredSizeLimit() diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py index d974118f..6575b0c3 100644 --- a/ue4docker/diagnostics/base.py +++ b/ue4docker/diagnostics/base.py @@ -61,8 +61,12 @@ def _generateWindowsBuildArgs(self, logger, basetagOverride=None, isolationOverr # Determine the appropriate container image base tag for the host system release unless the user specified a base tag buildArgs = [] - defaultBaseTag = WindowsUtils.getReleaseBaseTag(WindowsUtils.getWindowsRelease()) - baseTag = basetagOverride if basetagOverride is not None else defaultBaseTag + hostBaseTag = WindowsUtils.getHostBaseTag() + baseTag = basetagOverride if basetagOverride is not None else hostBaseTag + + if baseTag is None: + raise RuntimeError('unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag') + buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] # Use the default isolation mode unless requested otherwise @@ -72,7 +76,7 @@ def _generateWindowsBuildArgs(self, logger, basetagOverride=None, isolationOverr # If the user specified process isolation mode and a different base tag to the host system then warn them prefix = self.getPrefix() - if isolation == 'process' and baseTag != defaultBaseTag: + if isolation == 'process' and baseTag != hostBaseTag: logger.info('[{}] Warning: attempting to use different Windows container/host versions'.format(prefix), False) logger.info('[{}] when running in process isolation mode, this will usually break!'.format(prefix), False) diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/ue4docker/diagnostics/diagnostic_20gig.py index 7f6864c3..97572c22 100644 --- a/ue4docker/diagnostics/diagnostic_20gig.py +++ b/ue4docker/diagnostics/diagnostic_20gig.py @@ -14,7 +14,7 @@ def __init__(self): # Setup our argument parser so we can use its help message output in our description text self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 20gig') self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") def getName(self): ''' diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index a7c86cc7..2fb53322 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -16,7 +16,7 @@ def __init__(self): self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") self._parser.add_argument('--random', action='store_true', help="Create a file filled with random bytes instead of zeroes under Windows") self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") def getName(self): ''' diff --git a/ue4docker/diagnostics/diagnostic_network.py b/ue4docker/diagnostics/diagnostic_network.py index 3d65a075..35338f13 100644 --- a/ue4docker/diagnostics/diagnostic_network.py +++ b/ue4docker/diagnostics/diagnostic_network.py @@ -13,7 +13,7 @@ def __init__(self): self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics network') self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getValidBaseTags(), help="Override the default base image tag when testing Windows containers") + self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") def getName(self): ''' diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 38db5fba..da3eda1b 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -108,7 +108,6 @@ def addArguments(parser): parser.add_argument('--combine', action='store_true', help='Combine generated Dockerfiles into a single multi-stage build Dockerfile') parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') - parser.add_argument('--ignore-eol', action='store_true', help='Run builds even on EOL versions of Windows (advanced use only)') parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output during builds (useful for debugging)') @@ -166,7 +165,6 @@ def __init__(self, parser, argv): self.excludedComponents = set(self.args.exclude) self.baseImage = None self.prereqsTag = None - self.ignoreEOL = self.args.ignore_eol self.ignoreBlacklist = self.args.ignore_blacklist self.verbose = self.args.verbose self.layoutDir = self.args.layout @@ -250,30 +248,25 @@ def _generateWindowsConfig(self): self.opts['buildgraph_args'] = self.opts.get('buildgraph_args', '') + f' -set:VS{self.visualStudio}=true' # Determine base tag for the Windows release of the host system - self.hostRelease = WindowsUtils.getWindowsRelease() - self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease) + self.hostBasetag = WindowsUtils.getHostBaseTag() # Store the tag for the base Windows Server Core image self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag + + if self.basetag is None: + raise RuntimeError('unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag') + self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) self.prereqsTag = self.basetag + '-vs' + self.visualStudio - # Verify that any user-specified base tag is valid - if WindowsUtils.isValidBaseTag(self.basetag) == False: - raise RuntimeError('unrecognised Windows Server Core base image tag "{}", supported tags are {}'.format(self.basetag, WindowsUtils.getValidBaseTags())) - - # Verify that any user-specified tag suffix does not collide with our base tags - if WindowsUtils.isValidBaseTag(self.suffix) == True: - raise RuntimeError('tag suffix cannot be any of the Windows Server Core base image tags: {}'.format(WindowsUtils.getValidBaseTags())) - # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect if self.args.isolation is not None: self.isolation = self.args.isolation else: # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode - differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag + differentKernels = self.basetag != self.hostBasetag dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') if not differentKernels and dockerSupportsProcess: self.isolation = 'process' diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index c4bdbdf1..a68732ec 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -1,6 +1,7 @@ from .DockerUtils import DockerUtils from pkg_resources import parse_version import platform, sys +from typing import Optional if platform.system() == 'Windows': import winreg @@ -10,18 +11,22 @@ class WindowsUtils(object): # The oldest Windows build we support _minimumRequiredBuild = 17763 - # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 19042 + # This lookup table is based on the list of valid tags from + # and list of build-to-release mapping from https://docs.microsoft.com/en-us/windows/release-health/release-information + _knownTagsByBuildNumber = { + 17763: 'ltsc2019', + 18362: '1903', + 18363: '1909', + 19041: '2004', + 19042: '20H2', + 19043: '21H1', + } - # The list of Windows Server Core base image tags that we recognise, in ascending version number order - _validTags = ['ltsc2019', '1903', '1909', '2004', '20H2'] + _knownTags = list(_knownTagsByBuildNumber.values()) # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs # (See: ) - _blacklistedReleases = ['1903', '1909'] - - # The list of Windows Server Core container image releases that are unsupported due to having reached EOL - _eolReleases = ['1903', '1909'] + _blacklistedHosts = [18362, 18363] @staticmethod def _getVersionRegKey(subkey : str) -> str: @@ -54,25 +59,21 @@ def systemString() -> str: ''' Generates a verbose human-readable version string for the Windows host system ''' - return '{} Version {} (Build {}.{})'.format( + return '{} (Build {}.{})'.format( WindowsUtils._getVersionRegKey('ProductName'), - WindowsUtils.getWindowsRelease(), WindowsUtils.getWindowsBuild(), WindowsUtils._getVersionRegKey('UBR') ) @staticmethod - def getWindowsRelease() -> str: + def getHostBaseTag() -> Optional[str]: ''' - Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system + Retrieves the tag for the Windows Server Core base image matching the host Windows system ''' - try: - # Starting with Windows 20H2 (also known as 2009), Microsoft stopped updating ReleaseId - # and instead updates DisplayVersion - return WindowsUtils._getVersionRegKey('DisplayVersion') - except FileNotFoundError: - # Fallback to ReleaseId for pre-20H2 releases that didn't have DisplayVersion - return WindowsUtils._getVersionRegKey('ReleaseId') + + hostBuild = WindowsUtils.getWindowsBuild() + + return WindowsUtils._knownTagsByBuildNumber.get(hostBuild) @staticmethod def getWindowsBuild() -> int: @@ -82,23 +83,14 @@ def getWindowsBuild() -> int: return sys.getwindowsversion().build @staticmethod - def isBlacklistedWindowsVersion(release=None): + def isBlacklistedWindowsHost() -> bool: ''' - Determines if the specified Windows release is one with bugs that make it unsuitable for use + Determines if host Windows version is one with bugs that make it unsuitable for use (defaults to checking the host OS release if one is not specified) ''' dockerVersion = parse_version(DockerUtils.version()['Version']) - release = WindowsUtils.getWindowsRelease() if release is None else release - return release in WindowsUtils._blacklistedReleases and dockerVersion < parse_version('19.03.6') - - @staticmethod - def isEndOfLifeWindowsVersion(release=None): - ''' - Determines if the specified Windows release is one that has reached End Of Life (EOL) - (defaults to checking the host OS release if one is not specified) - ''' - release = WindowsUtils.getWindowsRelease() if release is None else release - return release in WindowsUtils._eolReleases + build = WindowsUtils.getWindowsBuild() + return build in WindowsUtils._blacklistedHosts and dockerVersion < parse_version('19.03.6') @staticmethod def isWindowsServer() -> bool: @@ -108,28 +100,6 @@ def isWindowsServer() -> bool: # TODO: Replace this with something more reliable return 'Windows Server' in WindowsUtils._getVersionRegKey('ProductName') - @staticmethod - def isInsiderPreview() -> bool: - ''' - Determines if the Windows host system is a Windows Insider preview build - ''' - return WindowsUtils.getWindowsBuild() > WindowsUtils._latestReleaseBuild - - @staticmethod - def getReleaseBaseTag(release: str) -> str: - ''' - Retrieves the tag for the Windows Server Core base image matching the specified Windows 10 / Windows Server release - ''' - - # For Windows Insider preview builds, build the latest release tag - if WindowsUtils.isInsiderPreview(): - return WindowsUtils._validTags[-1] - - # This lookup table is based on the list of valid tags from - return { - '1809': 'ltsc2019', - }.get(release, release) - @staticmethod def getDllSrcImage(basetag: str) -> str: ''' @@ -142,22 +112,18 @@ def getDllSrcImage(basetag: str) -> str: return f'mcr.microsoft.com/windows:{tag}' @staticmethod - def getValidBaseTags() -> [str]: - ''' - Returns the list of valid tags for the Windows Server Core base image, in ascending chronological release order - ''' - return WindowsUtils._validTags - - @staticmethod - def isValidBaseTag(tag: str) -> bool: + def getKnownBaseTags() -> [str]: ''' - Determines if the specified tag is a valid Windows Server Core base image tag + Returns the list of known tags for the Windows Server Core base image, in ascending chronological release order ''' - return tag in WindowsUtils._validTags + return WindowsUtils._knownTags @staticmethod - def isNewerBaseTag(older: str, newer: str) -> bool: + def isNewerBaseTag(older: str, newer: str) -> Optional[bool]: ''' Determines if the base tag `newer` is chronologically newer than the base tag `older` ''' - return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older) + try: + return WindowsUtils._knownTags.index(newer) > WindowsUtils._knownTags.index(older) + except ValueError: + return None From b82431c582e10231d4885224011a4f41af37698d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 15:06:40 +0300 Subject: [PATCH 186/430] Introduce codestyle via Black (#186) --- .github/CONTRIBUTING.md | 9 + .github/workflows/ci.yml | 7 +- setup.py | 112 +-- ue4docker/__main__.py | 12 +- ue4docker/build.py | 864 +++++++++++------- ue4docker/clean.py | 134 +-- ue4docker/diagnostics/base.py | 259 +++--- ue4docker/diagnostics/diagnostic_20gig.py | 152 +-- ue4docker/diagnostics/diagnostic_8gig.py | 179 ++-- ue4docker/diagnostics/diagnostic_all.py | 83 +- ue4docker/diagnostics/diagnostic_maxsize.py | 143 +-- ue4docker/diagnostics/diagnostic_network.py | 172 ++-- ue4docker/diagnostics_cmd.py | 127 +-- .../ue4-minimal/linux/copy-toolchain.py | 51 +- .../ue4-minimal/linux/enable-opengl.py | 16 +- .../ue4-minimal/linux/fix-targets.py | 17 +- .../ue4-minimal/linux/patch-build-graph.py | 41 +- .../ue4-minimal/linux/patch-filters-xml.py | 17 +- .../ue4-minimal/linux/set-changelist.py | 15 +- .../ue4-minimal/linux/split-components.py | 75 +- .../ue4-minimal/windows/fix-targets.py | 17 +- .../ue4-minimal/windows/patch-build-graph.py | 23 +- .../ue4-minimal/windows/patch-filters-xml.py | 15 +- .../ue4-minimal/windows/set-changelist.py | 15 +- .../ue4-minimal/windows/split-components.py | 73 +- .../ue4-source/linux/linker-fixup.py | 12 +- .../ue4-source/linux/patch-broken-releases.py | 81 +- .../dockerfiles/ue4-source/linux/patch-ubt.py | 24 +- .../windows/patch-broken-releases.py | 81 +- .../ue4-source/windows/patch-setup-win.py | 35 +- .../ue4-source/windows/patch-ubt.py | 24 +- ue4docker/export.py | 172 ++-- ue4docker/exports/export_installed.py | 115 ++- ue4docker/exports/export_packages.py | 290 +++--- ue4docker/info.py | 149 +-- .../infrastructure/BuildConfiguration.py | 844 ++++++++++------- .../infrastructure/CredentialEndpoint.py | 156 ++-- ue4docker/infrastructure/DarwinUtils.py | 69 +- ue4docker/infrastructure/DockerUtils.py | 402 ++++---- ue4docker/infrastructure/FilesystemUtils.py | 31 +- .../infrastructure/GlobalConfiguration.py | 65 +- ue4docker/infrastructure/ImageBuilder.py | 389 ++++---- ue4docker/infrastructure/ImageCleaner.py | 46 +- ue4docker/infrastructure/Logger.py | 58 +- ue4docker/infrastructure/NetworkUtils.py | 32 +- ue4docker/infrastructure/PackageUtils.py | 34 +- ue4docker/infrastructure/PrettyPrinting.py | 27 +- ue4docker/infrastructure/ResourceMonitor.py | 155 ++-- ue4docker/infrastructure/SubprocessUtils.py | 91 +- ue4docker/infrastructure/WindowsUtils.py | 250 ++--- ue4docker/main.py | 208 +++-- ue4docker/setup_cmd.py | 241 +++-- ue4docker/test.py | 139 +-- ue4docker/tests/build-and-package.py | 26 +- ue4docker/tests/consume-external-deps.py | 87 +- ue4docker/version.py | 2 +- ue4docker/version_cmd.py | 3 +- 57 files changed, 3991 insertions(+), 2975 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 7d9c7899..96ff826b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -18,3 +18,12 @@ If you are creating an issue for a feature request, you can disregard the defaul ## Creating pull requests Before creating a pull request, please ensure sure you have tested your changes on all platforms to which the change applies (Linux / Windows Server / Windows 10 / macOS). + +## Autoformatting your changes + +ue4-docker uses [Black](https://github.com/psf/black) for its code formatting. +If you are submitting changes, make sure to install and run Black: + +- `pip install --user black` +- `black .` (invoked in repository root) +- Now your code is properly formatted diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93287709..fb3bde99 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,13 @@ name: CI on: [push, pull_request] jobs: + lint: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: psf/black@stable build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/setup.py b/setup.py index b36d5f26..a081ab12 100644 --- a/setup.py +++ b/setup.py @@ -2,64 +2,64 @@ from setuptools import setup # Read the README markdown data from README.md -with open(abspath(join(dirname(__file__), 'README.md')), 'rb') as readmeFile: - __readme__ = readmeFile.read().decode('utf-8') +with open(abspath(join(dirname(__file__), "README.md")), "rb") as readmeFile: + __readme__ = readmeFile.read().decode("utf-8") # Read the version number from version.py -with open(abspath(join(dirname(__file__), 'ue4docker', 'version.py'))) as versionFile: - __version__ = versionFile.read().strip().replace('__version__ = ', '').replace("'", '') +with open(abspath(join(dirname(__file__), "ue4docker", "version.py"))) as versionFile: + __version__ = ( + versionFile.read().strip().replace("__version__ = ", "").replace("'", "") + ) setup( - name='ue4-docker', - version=__version__, - description='Windows and Linux containers for Unreal Engine 4', - long_description=__readme__, - long_description_content_type='text/markdown', - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Software Development :: Build Tools', - 'Environment :: Console' - ], - keywords='epic unreal engine docker', - url='http://github.com/adamrehn/ue4-docker', - author='Adam Rehn', - author_email='adam@adamrehn.com', - license='MIT', - packages=[ - 'ue4docker', - 'ue4docker.diagnostics', - 'ue4docker.exports', - 'ue4docker.infrastructure' - ], - zip_safe=False, - python_requires = '>=3.6', - install_requires = [ - 'colorama', - 'container-utils', - 'docker>=3.0.0', - 'flask', - 'humanfriendly', - 'Jinja2>=2.11.3', - 'packaging>=19.1', - 'psutil', - 'requests', - 'semver>=2.7.9,<3.0.0', - 'setuptools>=38.6.0', - 'termcolor', - 'twine>=1.11.0', - 'wheel>=0.31.0' - ], - package_data = { - 'ue4docker': [ - 'dockerfiles/*/*/.dockerignore', - 'dockerfiles/diagnostics/*/*/*', - 'dockerfiles/*/*/*', - 'tests/*.py' - ] - }, - entry_points = { - 'console_scripts': ['ue4-docker=ue4docker:main'] - } + name="ue4-docker", + version=__version__, + description="Windows and Linux containers for Unreal Engine 4", + long_description=__readme__, + long_description_content_type="text/markdown", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Topic :: Software Development :: Build Tools", + "Environment :: Console", + ], + keywords="epic unreal engine docker", + url="http://github.com/adamrehn/ue4-docker", + author="Adam Rehn", + author_email="adam@adamrehn.com", + license="MIT", + packages=[ + "ue4docker", + "ue4docker.diagnostics", + "ue4docker.exports", + "ue4docker.infrastructure", + ], + zip_safe=False, + python_requires=">=3.6", + install_requires=[ + "colorama", + "container-utils", + "docker>=3.0.0", + "flask", + "humanfriendly", + "Jinja2>=2.11.3", + "packaging>=19.1", + "psutil", + "requests", + "semver>=2.7.9,<3.0.0", + "setuptools>=38.6.0", + "termcolor", + "twine>=1.11.0", + "wheel>=0.31.0", + ], + package_data={ + "ue4docker": [ + "dockerfiles/*/*/.dockerignore", + "dockerfiles/diagnostics/*/*/*", + "dockerfiles/*/*/*", + "tests/*.py", + ] + }, + entry_points={"console_scripts": ["ue4-docker=ue4docker:main"]}, ) diff --git a/ue4docker/__main__.py b/ue4docker/__main__.py index a6f4407d..ebae417b 100644 --- a/ue4docker/__main__.py +++ b/ue4docker/__main__.py @@ -1,9 +1,9 @@ from .main import main import os, sys -if __name__ == '__main__': - - # Rewrite sys.argv[0] so our help prompts display the correct base command - interpreter = sys.executable if sys.executable not in [None, ''] else 'python3' - sys.argv[0] = '{} -m ue4docker'.format(os.path.basename(interpreter)) - main() +if __name__ == "__main__": + + # Rewrite sys.argv[0] so our help prompts display the correct base command + interpreter = sys.executable if sys.executable not in [None, ""] else "python3" + sys.argv[0] = "{} -m ue4docker".format(os.path.basename(interpreter)) + main() diff --git a/ue4docker/build.py b/ue4docker/build.py index c9733d7a..31faead5 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -3,338 +3,548 @@ from .version import __version__ from os.path import join + def _getCredential(args, name, envVar, promptFunc): - - # Check if the credential was specified via the command-line - if getattr(args, name, None) is not None: - print('Using {} specified via `-{}` command-line argument.'.format(name, name)) - return getattr(args, name) - - # Check if the credential was specified via an environment variable - if envVar in os.environ: - print('Using {} specified via {} environment variable.'.format(name, envVar)) - return os.environ[envVar] - - # Fall back to prompting the user for the value - return promptFunc() + + # Check if the credential was specified via the command-line + if getattr(args, name, None) is not None: + print("Using {} specified via `-{}` command-line argument.".format(name, name)) + return getattr(args, name) + + # Check if the credential was specified via an environment variable + if envVar in os.environ: + print("Using {} specified via {} environment variable.".format(name, envVar)) + return os.environ[envVar] + + # Fall back to prompting the user for the value + return promptFunc() + def _getUsername(args): - return _getCredential(args, 'username', 'UE4DOCKER_USERNAME', lambda: input("Username: ")) + return _getCredential( + args, "username", "UE4DOCKER_USERNAME", lambda: input("Username: ") + ) + def _getPassword(args): - return _getCredential(args, 'password', 'UE4DOCKER_PASSWORD', lambda: getpass.getpass("Password: ")) + return _getCredential( + args, "password", "UE4DOCKER_PASSWORD", lambda: getpass.getpass("Password: ") + ) def build(): - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} build] '.format(sys.argv[0])) - - # Register our supported command-line arguments - parser = argparse.ArgumentParser(prog='{} build'.format(sys.argv[0])) - BuildConfiguration.addArguments(parser) - - # If no command-line arguments were supplied, display the help message and exit - if len(sys.argv) < 2: - parser.print_help() - sys.exit(0) - - # Parse the supplied command-line arguments - try: - config = BuildConfiguration(parser, sys.argv[1:]) - except RuntimeError as e: - logger.error('Error: {}'.format(e)) - sys.exit(1) - - # Verify that Docker is installed - if DockerUtils.installed() == False: - logger.error('Error: could not detect Docker version. Please ensure Docker is installed.') - sys.exit(1) - - # Verify that we aren't trying to build Windows containers under Windows 10 when in Linux container mode (or vice versa) - # (Note that we don't bother performing this check when we're just copying Dockerfiles to an output directory) - if config.layoutDir is None: - dockerPlatform = DockerUtils.info()['OSType'].lower() - if config.containerPlatform == 'windows' and dockerPlatform == 'linux': - logger.error('Error: cannot build Windows containers when Docker Desktop is in Linux container', False) - logger.error('mode. Use the --linux flag if you want to build Linux containers instead.', False) - sys.exit(1) - elif config.containerPlatform == 'linux' and dockerPlatform == 'windows': - logger.error('Error: cannot build Linux containers when Docker Desktop is in Windows container', False) - logger.error('mode. Remove the --linux flag if you want to build Windows containers instead.', False) - sys.exit(1) - - # Create an auto-deleting temporary directory to hold our build context - with tempfile.TemporaryDirectory() as tempDir: - - # Copy our Dockerfiles to the temporary directory - contextOrig = join(os.path.dirname(os.path.abspath(__file__)), 'dockerfiles') - contextRoot = join(tempDir, 'dockerfiles') - shutil.copytree(contextOrig, contextRoot) - - # Create the builder instance to build the Docker images - builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun, config.layoutDir, config.opts, config.combine) - - # Resolve our main set of tags for the generated images - mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] - - # Print the command-line invocation that triggered this build, masking any supplied passwords - args = ['*******' if config.args.password is not None and arg == config.args.password else arg for arg in sys.argv] - logger.info('COMMAND-LINE INVOCATION:', False) - logger.info(str(args), False) - - # Print the details of the Unreal Engine version being built - logger.info('UNREAL ENGINE VERSION SETTINGS:') - logger.info('Custom build: {}'.format('Yes' if config.custom == True else 'No'), False) - if config.custom == True: - logger.info('Custom name: ' + config.release, False) - else: - logger.info('Release: ' + config.release, False) - logger.info('Repository: ' + config.repository, False) - logger.info('Branch/tag: ' + config.branch + '\n', False) - - # Determine if we are using a custom version for ue4cli or conan-ue4cli - if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: - logger.info('CUSTOM PACKAGE VERSIONS:', False) - logger.info('ue4cli: {}'.format(config.ue4cliVersion if config.ue4cliVersion is not None else 'default'), False) - logger.info('conan-ue4cli: {}\n'.format(config.conanUe4cliVersion if config.conanUe4cliVersion is not None else 'default'), False) - - # Report any advanced configuration options that were specified - if len(config.opts) > 0: - logger.info('ADVANCED CONFIGURATION OPTIONS:', False) - for key, value in sorted(config.opts.items()): - logger.info('{}: {}'.format(key, json.dumps(value)), False) - print('', file=sys.stderr, flush=True) - - # Determine if we are building Windows or Linux containers - if config.containerPlatform == 'windows': - - # Provide the user with feedback so they are aware of the Windows-specific values being used - logger.info('WINDOWS CONTAINER SETTINGS', False) - logger.info('Isolation mode: {}'.format(config.isolation), False) - logger.info('Base OS image: {}'.format(config.baseImage), False) - logger.info('Dll source image: {}'.format(config.dllSrcImage), False) - logger.info('Host OS: {}'.format(WindowsUtils.systemString()), False) - logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) - logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) - logger.info('Visual Studio: {}'.format(config.visualStudio), False) - - # Verify that the host OS is not a release that is blacklisted due to critical bugs - if config.ignoreBlacklist == False and WindowsUtils.isBlacklistedWindowsHost() == True: - logger.error('Error: detected blacklisted host OS version: {}'.format(WindowsUtils.systemString()), False) - logger.error('', False) - logger.error('This version of Windows contains one or more critical bugs that', False) - logger.error('render it incapable of successfully building UE4 container images.', False) - logger.error('You will need to use an older or newer version of Windows.', False) - logger.error('', False) - logger.error('For more information, see:', False) - logger.error('https://unrealcontainers.com/docs/concepts/windows-containers', False) - sys.exit(1) - - # Verify that the user is not attempting to build images with a newer kernel version than the host OS - newer_check = WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag) - if newer_check: - logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') - sys.exit(1) - elif newer_check is None: - logger.info('Warning: unable to determine whether host system is new enough to use specified base tag') - - # Ensure the Docker daemon is configured correctly - requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit: - logger.error('SETUP REQUIRED:') - logger.error('The max image size for Windows containers must be set to at least {}GB.'.format(requiredLimit)) - logger.error('See the Microsoft documentation for configuration instructions:') - logger.error('https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits') - logger.error('Under Windows Server, the command `{} setup` can be used to automatically configure the system.'.format(sys.argv[0])) - sys.exit(1) - - elif config.containerPlatform == 'linux': - - # Determine if we are building CUDA-enabled container images - capabilities = 'CUDA {} + OpenGL'.format(config.cuda) if config.cuda is not None else 'OpenGL' - logger.info('LINUX CONTAINER SETTINGS', False) - logger.info('Building GPU-enabled images compatible with NVIDIA Docker ({} support).\n'.format(capabilities), False) - - # Report which Engine components are being excluded (if any) - logger.info('GENERAL SETTINGS', False) - if len(config.excludedComponents) > 0: - logger.info('Excluding the following Engine components:', False) - for component in config.describeExcludedComponents(): - logger.info('- {}'.format(component), False) - else: - logger.info('Not excluding any Engine components.', False) - - # Determine if we need to prompt for credentials - if config.dryRun == True: - - # Don't bother prompting the user for any credentials during a dry run - logger.info('Performing a dry run, `docker build` commands will be printed and not executed.', False) - username = '' - password = '' - - elif config.layoutDir is not None: - - # Don't bother prompting the user for any credentials when we're just copying the Dockerfiles to a directory - logger.info('Copying generated Dockerfiles to: {}'.format(config.layoutDir), False) - username = '' - password = '' - - elif builder.willBuild('ue4-source', mainTags) == False: - - # Don't bother prompting the user for any credentials if we're not building the ue4-source image - logger.info('Not building the ue4-source image, no Git credentials required.', False) - username = '' - password = '' - - else: - - # Retrieve the Git username and password from the user when building the ue4-source image - print('\nRetrieving the Git credentials that will be used to clone the UE4 repo') - username = _getUsername(config.args) - password = _getPassword(config.args) - print() - - # If resource monitoring has been enabled, start the resource monitoring background thread - resourceMonitor = ResourceMonitor(logger, config.args.interval) - if config.args.monitor == True: - resourceMonitor.start() - - # Start the HTTP credential endpoint as a child process and wait for it to start - endpoint = CredentialEndpoint(username, password) - endpoint.start() - - try: - - # Keep track of our starting time - startTime = time.time() - - # If we're copying Dockerfiles to an output directory then make sure it exists and is empty - if config.layoutDir is not None: - if os.path.exists(config.layoutDir): - shutil.rmtree(config.layoutDir) - os.makedirs(config.layoutDir) - - # Keep track of the images we've built - builtImages = [] - - commonArgs = [ - '--build-arg', 'NAMESPACE={}'.format(GlobalConfiguration.getTagNamespace()), - ] - - # Compute the build options for the UE4 build prerequisites image - # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) - prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] - if config.containerPlatform == 'windows': - prereqsArgs = prereqsArgs + [ - '--build-arg', 'DLLSRCIMAGE=' + config.dllSrcImage, - '--build-arg', 'VISUAL_STUDIO_BUILD_NUMBER=' + config.visualStudioBuildNumber, - ] - - # Build the UE4 build prerequisites image - builder.build('ue4-build-prerequisites', [config.prereqsTag], commonArgs + config.platformArgs + prereqsArgs) - builtImages.append('ue4-build-prerequisites') - - # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets - secrets = {} - if config.opts.get('use_build_secrets', False) == True: - secrets = { - 'username': username, - 'password': password - } - - # Build the UE4 source image - prereqConsumerArgs = ['--build-arg', 'PREREQS_TAG={}'.format(config.prereqsTag)] - credentialArgs = [] if len(secrets) > 0 else endpoint.args() - ue4SourceArgs = prereqConsumerArgs + [ - '--build-arg', 'GIT_REPO={}'.format(config.repository), - '--build-arg', 'GIT_BRANCH={}'.format(config.branch), - '--build-arg', 'VERBOSE_OUTPUT={}'.format('1' if config.verbose == True else '0') - ] - builder.build('ue4-source', mainTags, commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, secrets) - builtImages.append('ue4-source') - - # Build the UE4 Engine source build image, unless requested otherwise by the user - ue4BuildArgs = prereqConsumerArgs + [ - '--build-arg', 'TAG={}'.format(mainTags[1]), - ] - if config.noEngine == False: - builder.build('ue4-engine', mainTags, commonArgs + config.platformArgs + ue4BuildArgs) - builtImages.append('ue4-engine') - else: - logger.info('User specified `--no-engine`, skipping ue4-engine image build.') - - # Build the minimal UE4 CI image, unless requested otherwise by the user - buildUe4Minimal = config.noMinimal == False - if buildUe4Minimal == True: - builder.build('ue4-minimal', mainTags, commonArgs + config.platformArgs + ue4BuildArgs) - builtImages.append('ue4-minimal') - else: - logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') - - # Build the full UE4 CI image, unless requested otherwise by the user - buildUe4Full = buildUe4Minimal == True and config.noFull == False - if buildUe4Full == True: - - # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them - infrastructureFlags = [] - if config.ue4cliVersion is not None: - infrastructureFlags.extend(['--build-arg', 'UE4CLI_VERSION={}'.format(config.ue4cliVersion)]) - if config.conanUe4cliVersion is not None: - infrastructureFlags.extend(['--build-arg', 'CONAN_UE4CLI_VERSION={}'.format(config.conanUe4cliVersion)]) - - # Build the image - builder.build('ue4-full', mainTags, commonArgs + config.platformArgs + ue4BuildArgs + infrastructureFlags) - builtImages.append('ue4-full') - else: - logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') - - # If we are generating Dockerfiles then include information about the options used to generate them - if config.layoutDir is not None: - - # Determine whether we generated a single combined Dockerfile or a set of Dockerfiles - if config.combine == True: - - # Generate a comment to place at the top of the single combined Dockerfile - lines = ['This file was generated by ue4-docker version {} with the following options:'.format(__version__), ''] - lines.extend(['- {}: {}'.format(key, json.dumps(value)) for key, value in sorted(config.opts.items())]) - lines.extend(['', 'This Dockerfile combines the steps for the following images:', '']) - lines.extend(['- {}'.format(image) for image in builtImages]) - comment = '\n'.join(['# {}'.format(line) for line in lines]) - - # Inject the comment at the top of the Dockerfile, being sure to place it after any `escape` parser directive - dockerfile = join(config.layoutDir, 'combined', 'Dockerfile') - dockerfileContents = FilesystemUtils.readFile(dockerfile) - if dockerfileContents.startswith('# escape'): - newline = dockerfileContents.index('\n') - dockerfileContents = dockerfileContents[0:newline+1] + '\n' + comment + '\n\n' + dockerfileContents[newline+1:] - else: - dockerfileContents = comment + '\n\n' + dockerfileContents - FilesystemUtils.writeFile(dockerfile, dockerfileContents) - - else: - - # Create a JSON file to accompany the set of generated Dockerfiles - FilesystemUtils.writeFile(join(config.layoutDir, 'generated.json'), json.dumps({ - 'version': __version__, - 'images': builtImages, - 'opts': config.opts - }, indent=4, sort_keys=True)) - - # Report the total execution time - endTime = time.time() - logger.action('Total execution time: {}'.format(humanfriendly.format_timespan(endTime - startTime))) - - # Stop the resource monitoring background thread if it is running - resourceMonitor.stop() - - # Stop the HTTP server - endpoint.stop() - - except (Exception, KeyboardInterrupt) as e: - - # One of the images failed to build - logger.error('Error: {}'.format(e)) - resourceMonitor.stop() - endpoint.stop() - sys.exit(1) + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} build] ".format(sys.argv[0])) + + # Register our supported command-line arguments + parser = argparse.ArgumentParser(prog="{} build".format(sys.argv[0])) + BuildConfiguration.addArguments(parser) + + # If no command-line arguments were supplied, display the help message and exit + if len(sys.argv) < 2: + parser.print_help() + sys.exit(0) + + # Parse the supplied command-line arguments + try: + config = BuildConfiguration(parser, sys.argv[1:]) + except RuntimeError as e: + logger.error("Error: {}".format(e)) + sys.exit(1) + + # Verify that Docker is installed + if DockerUtils.installed() == False: + logger.error( + "Error: could not detect Docker version. Please ensure Docker is installed." + ) + sys.exit(1) + + # Verify that we aren't trying to build Windows containers under Windows 10 when in Linux container mode (or vice versa) + # (Note that we don't bother performing this check when we're just copying Dockerfiles to an output directory) + if config.layoutDir is None: + dockerPlatform = DockerUtils.info()["OSType"].lower() + if config.containerPlatform == "windows" and dockerPlatform == "linux": + logger.error( + "Error: cannot build Windows containers when Docker Desktop is in Linux container", + False, + ) + logger.error( + "mode. Use the --linux flag if you want to build Linux containers instead.", + False, + ) + sys.exit(1) + elif config.containerPlatform == "linux" and dockerPlatform == "windows": + logger.error( + "Error: cannot build Linux containers when Docker Desktop is in Windows container", + False, + ) + logger.error( + "mode. Remove the --linux flag if you want to build Windows containers instead.", + False, + ) + sys.exit(1) + + # Create an auto-deleting temporary directory to hold our build context + with tempfile.TemporaryDirectory() as tempDir: + + # Copy our Dockerfiles to the temporary directory + contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles") + contextRoot = join(tempDir, "dockerfiles") + shutil.copytree(contextOrig, contextRoot) + + # Create the builder instance to build the Docker images + builder = ImageBuilder( + contextRoot, + config.containerPlatform, + logger, + config.rebuild, + config.dryRun, + config.layoutDir, + config.opts, + config.combine, + ) + + # Resolve our main set of tags for the generated images + mainTags = [ + "{}{}-{}".format(config.release, config.suffix, config.prereqsTag), + config.release + config.suffix, + ] + + # Print the command-line invocation that triggered this build, masking any supplied passwords + args = [ + "*******" + if config.args.password is not None and arg == config.args.password + else arg + for arg in sys.argv + ] + logger.info("COMMAND-LINE INVOCATION:", False) + logger.info(str(args), False) + + # Print the details of the Unreal Engine version being built + logger.info("UNREAL ENGINE VERSION SETTINGS:") + logger.info( + "Custom build: {}".format("Yes" if config.custom == True else "No"), False + ) + if config.custom == True: + logger.info("Custom name: " + config.release, False) + else: + logger.info("Release: " + config.release, False) + logger.info("Repository: " + config.repository, False) + logger.info("Branch/tag: " + config.branch + "\n", False) + + # Determine if we are using a custom version for ue4cli or conan-ue4cli + if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: + logger.info("CUSTOM PACKAGE VERSIONS:", False) + logger.info( + "ue4cli: {}".format( + config.ue4cliVersion + if config.ue4cliVersion is not None + else "default" + ), + False, + ) + logger.info( + "conan-ue4cli: {}\n".format( + config.conanUe4cliVersion + if config.conanUe4cliVersion is not None + else "default" + ), + False, + ) + + # Report any advanced configuration options that were specified + if len(config.opts) > 0: + logger.info("ADVANCED CONFIGURATION OPTIONS:", False) + for key, value in sorted(config.opts.items()): + logger.info("{}: {}".format(key, json.dumps(value)), False) + print("", file=sys.stderr, flush=True) + + # Determine if we are building Windows or Linux containers + if config.containerPlatform == "windows": + + # Provide the user with feedback so they are aware of the Windows-specific values being used + logger.info("WINDOWS CONTAINER SETTINGS", False) + logger.info( + "Isolation mode: {}".format(config.isolation), False + ) + logger.info( + "Base OS image: {}".format(config.baseImage), False + ) + logger.info( + "Dll source image: {}".format(config.dllSrcImage), False + ) + logger.info( + "Host OS: {}".format(WindowsUtils.systemString()), + False, + ) + logger.info( + "Memory limit: {}".format( + "No limit" + if config.memLimit is None + else "{:.2f}GB".format(config.memLimit) + ), + False, + ) + logger.info( + "Detected max image size: {:.0f}GB".format(DockerUtils.maxsize()), + False, + ) + logger.info( + "Visual Studio: {}".format(config.visualStudio), False + ) + + # Verify that the host OS is not a release that is blacklisted due to critical bugs + if ( + config.ignoreBlacklist == False + and WindowsUtils.isBlacklistedWindowsHost() == True + ): + logger.error( + "Error: detected blacklisted host OS version: {}".format( + WindowsUtils.systemString() + ), + False, + ) + logger.error("", False) + logger.error( + "This version of Windows contains one or more critical bugs that", + False, + ) + logger.error( + "render it incapable of successfully building UE4 container images.", + False, + ) + logger.error( + "You will need to use an older or newer version of Windows.", False + ) + logger.error("", False) + logger.error("For more information, see:", False) + logger.error( + "https://unrealcontainers.com/docs/concepts/windows-containers", + False, + ) + sys.exit(1) + + # Verify that the user is not attempting to build images with a newer kernel version than the host OS + newer_check = WindowsUtils.isNewerBaseTag( + config.hostBasetag, config.basetag + ) + if newer_check: + logger.error( + "Error: cannot build container images with a newer kernel version than that of the host OS!" + ) + sys.exit(1) + elif newer_check is None: + logger.info( + "Warning: unable to determine whether host system is new enough to use specified base tag" + ) + + # Ensure the Docker daemon is configured correctly + requiredLimit = WindowsUtils.requiredSizeLimit() + if DockerUtils.maxsize() < requiredLimit: + logger.error("SETUP REQUIRED:") + logger.error( + "The max image size for Windows containers must be set to at least {}GB.".format( + requiredLimit + ) + ) + logger.error( + "See the Microsoft documentation for configuration instructions:" + ) + logger.error( + "https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits" + ) + logger.error( + "Under Windows Server, the command `{} setup` can be used to automatically configure the system.".format( + sys.argv[0] + ) + ) + sys.exit(1) + + elif config.containerPlatform == "linux": + + # Determine if we are building CUDA-enabled container images + capabilities = ( + "CUDA {} + OpenGL".format(config.cuda) + if config.cuda is not None + else "OpenGL" + ) + logger.info("LINUX CONTAINER SETTINGS", False) + logger.info( + "Building GPU-enabled images compatible with NVIDIA Docker ({} support).\n".format( + capabilities + ), + False, + ) + + # Report which Engine components are being excluded (if any) + logger.info("GENERAL SETTINGS", False) + if len(config.excludedComponents) > 0: + logger.info("Excluding the following Engine components:", False) + for component in config.describeExcludedComponents(): + logger.info("- {}".format(component), False) + else: + logger.info("Not excluding any Engine components.", False) + + # Determine if we need to prompt for credentials + if config.dryRun == True: + + # Don't bother prompting the user for any credentials during a dry run + logger.info( + "Performing a dry run, `docker build` commands will be printed and not executed.", + False, + ) + username = "" + password = "" + + elif config.layoutDir is not None: + + # Don't bother prompting the user for any credentials when we're just copying the Dockerfiles to a directory + logger.info( + "Copying generated Dockerfiles to: {}".format(config.layoutDir), False + ) + username = "" + password = "" + + elif builder.willBuild("ue4-source", mainTags) == False: + + # Don't bother prompting the user for any credentials if we're not building the ue4-source image + logger.info( + "Not building the ue4-source image, no Git credentials required.", False + ) + username = "" + password = "" + + else: + + # Retrieve the Git username and password from the user when building the ue4-source image + print( + "\nRetrieving the Git credentials that will be used to clone the UE4 repo" + ) + username = _getUsername(config.args) + password = _getPassword(config.args) + print() + + # If resource monitoring has been enabled, start the resource monitoring background thread + resourceMonitor = ResourceMonitor(logger, config.args.interval) + if config.args.monitor == True: + resourceMonitor.start() + + # Start the HTTP credential endpoint as a child process and wait for it to start + endpoint = CredentialEndpoint(username, password) + endpoint.start() + + try: + + # Keep track of our starting time + startTime = time.time() + + # If we're copying Dockerfiles to an output directory then make sure it exists and is empty + if config.layoutDir is not None: + if os.path.exists(config.layoutDir): + shutil.rmtree(config.layoutDir) + os.makedirs(config.layoutDir) + + # Keep track of the images we've built + builtImages = [] + + commonArgs = [ + "--build-arg", + "NAMESPACE={}".format(GlobalConfiguration.getTagNamespace()), + ] + + # Compute the build options for the UE4 build prerequisites image + # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) + prereqsArgs = ["--build-arg", "BASEIMAGE=" + config.baseImage] + if config.containerPlatform == "windows": + prereqsArgs = prereqsArgs + [ + "--build-arg", + "DLLSRCIMAGE=" + config.dllSrcImage, + "--build-arg", + "VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber, + ] + + # Build the UE4 build prerequisites image + builder.build( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + ) + builtImages.append("ue4-build-prerequisites") + + # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets + secrets = {} + if config.opts.get("use_build_secrets", False) == True: + secrets = {"username": username, "password": password} + + # Build the UE4 source image + prereqConsumerArgs = [ + "--build-arg", + "PREREQS_TAG={}".format(config.prereqsTag), + ] + credentialArgs = [] if len(secrets) > 0 else endpoint.args() + ue4SourceArgs = prereqConsumerArgs + [ + "--build-arg", + "GIT_REPO={}".format(config.repository), + "--build-arg", + "GIT_BRANCH={}".format(config.branch), + "--build-arg", + "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), + ] + builder.build( + "ue4-source", + mainTags, + commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, + secrets, + ) + builtImages.append("ue4-source") + + # Build the UE4 Engine source build image, unless requested otherwise by the user + ue4BuildArgs = prereqConsumerArgs + [ + "--build-arg", + "TAG={}".format(mainTags[1]), + ] + if config.noEngine == False: + builder.build( + "ue4-engine", + mainTags, + commonArgs + config.platformArgs + ue4BuildArgs, + ) + builtImages.append("ue4-engine") + else: + logger.info( + "User specified `--no-engine`, skipping ue4-engine image build." + ) + + # Build the minimal UE4 CI image, unless requested otherwise by the user + buildUe4Minimal = config.noMinimal == False + if buildUe4Minimal == True: + builder.build( + "ue4-minimal", + mainTags, + commonArgs + config.platformArgs + ue4BuildArgs, + ) + builtImages.append("ue4-minimal") + else: + logger.info( + "User specified `--no-minimal`, skipping ue4-minimal image build." + ) + + # Build the full UE4 CI image, unless requested otherwise by the user + buildUe4Full = buildUe4Minimal == True and config.noFull == False + if buildUe4Full == True: + + # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them + infrastructureFlags = [] + if config.ue4cliVersion is not None: + infrastructureFlags.extend( + [ + "--build-arg", + "UE4CLI_VERSION={}".format(config.ue4cliVersion), + ] + ) + if config.conanUe4cliVersion is not None: + infrastructureFlags.extend( + [ + "--build-arg", + "CONAN_UE4CLI_VERSION={}".format(config.conanUe4cliVersion), + ] + ) + + # Build the image + builder.build( + "ue4-full", + mainTags, + commonArgs + + config.platformArgs + + ue4BuildArgs + + infrastructureFlags, + ) + builtImages.append("ue4-full") + else: + logger.info( + "Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build." + ) + + # If we are generating Dockerfiles then include information about the options used to generate them + if config.layoutDir is not None: + + # Determine whether we generated a single combined Dockerfile or a set of Dockerfiles + if config.combine == True: + + # Generate a comment to place at the top of the single combined Dockerfile + lines = [ + "This file was generated by ue4-docker version {} with the following options:".format( + __version__ + ), + "", + ] + lines.extend( + [ + "- {}: {}".format(key, json.dumps(value)) + for key, value in sorted(config.opts.items()) + ] + ) + lines.extend( + [ + "", + "This Dockerfile combines the steps for the following images:", + "", + ] + ) + lines.extend(["- {}".format(image) for image in builtImages]) + comment = "\n".join(["# {}".format(line) for line in lines]) + + # Inject the comment at the top of the Dockerfile, being sure to place it after any `escape` parser directive + dockerfile = join(config.layoutDir, "combined", "Dockerfile") + dockerfileContents = FilesystemUtils.readFile(dockerfile) + if dockerfileContents.startswith("# escape"): + newline = dockerfileContents.index("\n") + dockerfileContents = ( + dockerfileContents[0 : newline + 1] + + "\n" + + comment + + "\n\n" + + dockerfileContents[newline + 1 :] + ) + else: + dockerfileContents = comment + "\n\n" + dockerfileContents + FilesystemUtils.writeFile(dockerfile, dockerfileContents) + + else: + + # Create a JSON file to accompany the set of generated Dockerfiles + FilesystemUtils.writeFile( + join(config.layoutDir, "generated.json"), + json.dumps( + { + "version": __version__, + "images": builtImages, + "opts": config.opts, + }, + indent=4, + sort_keys=True, + ), + ) + + # Report the total execution time + endTime = time.time() + logger.action( + "Total execution time: {}".format( + humanfriendly.format_timespan(endTime - startTime) + ) + ) + + # Stop the resource monitoring background thread if it is running + resourceMonitor.stop() + + # Stop the HTTP server + endpoint.stop() + + except (Exception, KeyboardInterrupt) as e: + + # One of the images failed to build + logger.error("Error: {}".format(e)) + resourceMonitor.stop() + endpoint.stop() + sys.exit(1) diff --git a/ue4docker/clean.py b/ue4docker/clean.py index dcd19529..377f0860 100644 --- a/ue4docker/clean.py +++ b/ue4docker/clean.py @@ -1,63 +1,87 @@ import argparse, subprocess, sys from .infrastructure import * + def _isIntermediateImage(image): - sentinel = 'com.adamrehn.ue4-docker.sentinel' - labels = image.attrs['ContainerConfig']['Labels'] - return labels is not None and sentinel in labels + sentinel = "com.adamrehn.ue4-docker.sentinel" + labels = image.attrs["ContainerConfig"]["Labels"] + return labels is not None and sentinel in labels + def _cleanMatching(cleaner, filter, tag, dryRun): - tagSuffix = ':{}'.format(tag) if tag is not None else '*' - matching = DockerUtils.listImages(tagFilter = filter + tagSuffix) - cleaner.cleanMultiple([image.tags[0] for image in matching], dryRun) + tagSuffix = ":{}".format(tag) if tag is not None else "*" + matching = DockerUtils.listImages(tagFilter=filter + tagSuffix) + cleaner.cleanMultiple([image.tags[0] for image in matching], dryRun) + def clean(): - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} clean] '.format(sys.argv[0])) - - # Our supported command-line arguments - parser = argparse.ArgumentParser( - prog='{} clean'.format(sys.argv[0]), - description = - 'Cleans built container images. ' + - 'By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed.' - ) - parser.add_argument('-tag', default=None, help='Only clean images with the specified tag') - parser.add_argument('--source', action='store_true', help='Clean ue4-source images') - parser.add_argument('--engine', action='store_true', help='Clean ue4-engine images') - parser.add_argument('--all', action='store_true', help='Clean all ue4-docker images') - parser.add_argument('--dry-run', action='store_true', help='Print docker commands instead of running them') - parser.add_argument('--prune', action='store_true', help='Run `docker system prune` after cleaning') - - # Parse the supplied command-line arguments - args = parser.parse_args() - - # Create our image cleaner - cleaner = ImageCleaner(logger) - - # Remove any intermediate images leftover from our multi-stage builds - dangling = DockerUtils.listImages(filters = {'dangling': True}) - dangling = [image.id for image in dangling if _isIntermediateImage(image)] - cleaner.cleanMultiple(dangling, args.dry_run) - - # If requested, remove ue4-source images - if args.source == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-source'), args.tag, args.dry_run) - - # If requested, remove ue4-engine images - if args.engine == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-engine'), args.tag, args.dry_run) - - # If requested, remove everything - if args.all == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-*'), args.tag, args.dry_run) - - # If requested, run `docker system prune` - if args.prune == True: - logger.action('Running `docker system prune`...') - pruneCommand = ['docker', 'system', 'prune', '-f'] - if args.dry_run == True: - print(pruneCommand) - else: - subprocess.call(pruneCommand) + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} clean] ".format(sys.argv[0])) + + # Our supported command-line arguments + parser = argparse.ArgumentParser( + prog="{} clean".format(sys.argv[0]), + description="Cleans built container images. " + + "By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed.", + ) + parser.add_argument( + "-tag", default=None, help="Only clean images with the specified tag" + ) + parser.add_argument("--source", action="store_true", help="Clean ue4-source images") + parser.add_argument("--engine", action="store_true", help="Clean ue4-engine images") + parser.add_argument( + "--all", action="store_true", help="Clean all ue4-docker images" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print docker commands instead of running them", + ) + parser.add_argument( + "--prune", action="store_true", help="Run `docker system prune` after cleaning" + ) + + # Parse the supplied command-line arguments + args = parser.parse_args() + + # Create our image cleaner + cleaner = ImageCleaner(logger) + + # Remove any intermediate images leftover from our multi-stage builds + dangling = DockerUtils.listImages(filters={"dangling": True}) + dangling = [image.id for image in dangling if _isIntermediateImage(image)] + cleaner.cleanMultiple(dangling, args.dry_run) + + # If requested, remove ue4-source images + if args.source == True: + _cleanMatching( + cleaner, + GlobalConfiguration.resolveTag("ue4-source"), + args.tag, + args.dry_run, + ) + + # If requested, remove ue4-engine images + if args.engine == True: + _cleanMatching( + cleaner, + GlobalConfiguration.resolveTag("ue4-engine"), + args.tag, + args.dry_run, + ) + + # If requested, remove everything + if args.all == True: + _cleanMatching( + cleaner, GlobalConfiguration.resolveTag("ue4-*"), args.tag, args.dry_run + ) + + # If requested, run `docker system prune` + if args.prune == True: + logger.action("Running `docker system prune`...") + pruneCommand = ["docker", "system", "prune", "-f"] + if args.dry_run == True: + print(pruneCommand) + else: + subprocess.call(pruneCommand) diff --git a/ue4docker/diagnostics/base.py b/ue4docker/diagnostics/base.py index 6575b0c3..e538dcd1 100644 --- a/ue4docker/diagnostics/base.py +++ b/ue4docker/diagnostics/base.py @@ -2,111 +2,156 @@ from os.path import abspath, dirname, join import subprocess + class DiagnosticBase(object): - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - raise NotImplementedError - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - raise NotImplementedError - - def getPrefix(self): - ''' - Returns the short name of the diagnostic for use in log output - ''' - raise NotImplementedError - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - raise NotImplementedError - - - # Helper functionality for derived classes - - def _printAndRun(self, logger, prefix, command, check=False): - ''' - Prints a command and then executes it - ''' - logger.info(prefix + 'Run: {}'.format(command), False) - subprocess.run(command, check=check) - - def _checkPlatformMistmatch(self, logger, containerPlatform): - ''' - Verifies that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) - ''' - prefix = self.getPrefix() - dockerInfo = DockerUtils.info() - dockerPlatform = dockerInfo['OSType'].lower() - if containerPlatform == 'windows' and dockerPlatform == 'linux': - logger.error('[{}] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.'.format(prefix), False) - logger.error('[{}] Use the --linux flag if you want to test Linux containers instead.'.format(prefix), False) - raise RuntimeError - elif containerPlatform == 'linux' and dockerPlatform == 'windows': - logger.error('[{}] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.'.format(prefix), False) - logger.error('[{}] Remove the --linux flag if you want to test Windows containers instead.'.format(prefix), False) - raise RuntimeError - - def _generateWindowsBuildArgs(self, logger, basetagOverride=None, isolationOverride=None): - ''' - Generates the build arguments for testing Windows containers, with optional overrides for base tag and isolation mode - ''' - - # Determine the appropriate container image base tag for the host system release unless the user specified a base tag - buildArgs = [] - hostBaseTag = WindowsUtils.getHostBaseTag() - baseTag = basetagOverride if basetagOverride is not None else hostBaseTag - - if baseTag is None: - raise RuntimeError('unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag') - - buildArgs = ['--build-arg', 'BASETAG={}'.format(baseTag)] - - # Use the default isolation mode unless requested otherwise - dockerInfo = DockerUtils.info() - isolation = isolationOverride if isolationOverride is not None else dockerInfo['Isolation'] - buildArgs += ['--isolation={}'.format(isolation)] - - # If the user specified process isolation mode and a different base tag to the host system then warn them - prefix = self.getPrefix() - if isolation == 'process' and baseTag != hostBaseTag: - logger.info('[{}] Warning: attempting to use different Windows container/host versions'.format(prefix), False) - logger.info('[{}] when running in process isolation mode, this will usually break!'.format(prefix), False) - - # Set a sensible memory limit when using Hyper-V isolation mode - if isolation == 'hyperv': - buildArgs += ['-m', '4GiB'] - - return buildArgs - - def _buildDockerfile(self, logger, containerPlatform, tag, buildArgs): - ''' - Attempts to build the diagnostic's Dockerfile for the specified container platform, with the specified parameters - ''' - - # Attempt to build the Dockerfile - prefix = self.getPrefix() - contextDir = join(dirname(dirname(abspath(__file__))), 'dockerfiles', 'diagnostics', prefix, containerPlatform) - try: - command = ['docker', 'build', '-t', tag, contextDir] + buildArgs - self._printAndRun(logger, '[{}] '.format(prefix), command, check=True) - built = True - except: - logger.error('[{}] Build failed!'.format(prefix)) - built = False - - # Remove any built images, including intermediate images - logger.action('[{}] Cleaning up...'.format(prefix), False) - if built == True: - self._printAndRun(logger, '[{}] '.format(prefix), ['docker', 'rmi', tag]) - self._printAndRun(logger, '[{}] '.format(prefix), ['docker', 'system', 'prune', '-f']) - - # Report the success or failure of the build - return built + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + raise NotImplementedError + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + raise NotImplementedError + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + raise NotImplementedError + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + raise NotImplementedError + + # Helper functionality for derived classes + + def _printAndRun(self, logger, prefix, command, check=False): + """ + Prints a command and then executes it + """ + logger.info(prefix + "Run: {}".format(command), False) + subprocess.run(command, check=check) + + def _checkPlatformMistmatch(self, logger, containerPlatform): + """ + Verifies that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + """ + prefix = self.getPrefix() + dockerInfo = DockerUtils.info() + dockerPlatform = dockerInfo["OSType"].lower() + if containerPlatform == "windows" and dockerPlatform == "linux": + logger.error( + "[{}] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.".format( + prefix + ), + False, + ) + logger.error( + "[{}] Use the --linux flag if you want to test Linux containers instead.".format( + prefix + ), + False, + ) + raise RuntimeError + elif containerPlatform == "linux" and dockerPlatform == "windows": + logger.error( + "[{}] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.".format( + prefix + ), + False, + ) + logger.error( + "[{}] Remove the --linux flag if you want to test Windows containers instead.".format( + prefix + ), + False, + ) + raise RuntimeError + + def _generateWindowsBuildArgs( + self, logger, basetagOverride=None, isolationOverride=None + ): + """ + Generates the build arguments for testing Windows containers, with optional overrides for base tag and isolation mode + """ + + # Determine the appropriate container image base tag for the host system release unless the user specified a base tag + buildArgs = [] + hostBaseTag = WindowsUtils.getHostBaseTag() + baseTag = basetagOverride if basetagOverride is not None else hostBaseTag + + if baseTag is None: + raise RuntimeError( + "unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag" + ) + + buildArgs = ["--build-arg", "BASETAG={}".format(baseTag)] + + # Use the default isolation mode unless requested otherwise + dockerInfo = DockerUtils.info() + isolation = ( + isolationOverride + if isolationOverride is not None + else dockerInfo["Isolation"] + ) + buildArgs += ["--isolation={}".format(isolation)] + + # If the user specified process isolation mode and a different base tag to the host system then warn them + prefix = self.getPrefix() + if isolation == "process" and baseTag != hostBaseTag: + logger.info( + "[{}] Warning: attempting to use different Windows container/host versions".format( + prefix + ), + False, + ) + logger.info( + "[{}] when running in process isolation mode, this will usually break!".format( + prefix + ), + False, + ) + + # Set a sensible memory limit when using Hyper-V isolation mode + if isolation == "hyperv": + buildArgs += ["-m", "4GiB"] + + return buildArgs + + def _buildDockerfile(self, logger, containerPlatform, tag, buildArgs): + """ + Attempts to build the diagnostic's Dockerfile for the specified container platform, with the specified parameters + """ + + # Attempt to build the Dockerfile + prefix = self.getPrefix() + contextDir = join( + dirname(dirname(abspath(__file__))), + "dockerfiles", + "diagnostics", + prefix, + containerPlatform, + ) + try: + command = ["docker", "build", "-t", tag, contextDir] + buildArgs + self._printAndRun(logger, "[{}] ".format(prefix), command, check=True) + built = True + except: + logger.error("[{}] Build failed!".format(prefix)) + built = False + + # Remove any built images, including intermediate images + logger.action("[{}] Cleaning up...".format(prefix), False) + if built == True: + self._printAndRun(logger, "[{}] ".format(prefix), ["docker", "rmi", tag]) + self._printAndRun( + logger, "[{}] ".format(prefix), ["docker", "system", "prune", "-f"] + ) + + # Report the success or failure of the build + return built diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/ue4docker/diagnostics/diagnostic_20gig.py index 97572c22..b7372565 100644 --- a/ue4docker/diagnostics/diagnostic_20gig.py +++ b/ue4docker/diagnostics/diagnostic_20gig.py @@ -4,68 +4,92 @@ import argparse, os, platform from os.path import abspath, dirname, join + class diagnostic20Gig(DiagnosticBase): - - # The tag we use for built images - IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:20gig' - - def __init__(self): - - # Setup our argument parser so we can use its help message output in our description text - self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 20gig') - self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - return 'Check for Docker 20GiB COPY bug' - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - return '\n'.join([ - 'This diagnostic determines if the Docker daemon suffers from 20GiB COPY bug', - 'reported at https://github.com/moby/moby/issues/37352 (affects Windows containers only)', - '', - '#37352 was fixed in https://github.com/moby/moby/pull/41636 but that fix was not released yet', - '', - self._parser.format_help() - ]) - - def getPrefix(self): - ''' - Returns the short name of the diagnostic for use in log output - ''' - return '20gig' - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - - # Parse our supplied arguments - args = self._parser.parse_args(args) - - # Determine which platform we are running on - containerPlatform = platform.system().lower() - - if containerPlatform != 'windows': - logger.action('[20gig] Diagnostic skipped. Current platform is not affected by the bug this diagnostic checks\n') - return True - - buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) - - # Attempt to build the Dockerfile - logger.action('[20gig] Attempting to COPY more than 20GiB between layers...', False) - built = self._buildDockerfile(logger, containerPlatform, diagnostic20Gig.IMAGE_TAG, buildArgs) - - # Inform the user of the outcome of the diagnostic - if built == True: - logger.action('[20gig] Diagnostic succeeded! The Docker daemon can COPY more than 20GiB between layers.\n') - else: - logger.error('[20gig] Diagnostic failed! The Docker daemon cannot COPY more than 20GiB between layers.\n', True) - - return built + + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:20gig" + + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 20gig") + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check for Docker 20GiB COPY bug" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if the Docker daemon suffers from 20GiB COPY bug", + "reported at https://github.com/moby/moby/issues/37352 (affects Windows containers only)", + "", + "#37352 was fixed in https://github.com/moby/moby/pull/41636 but that fix was not released yet", + "", + self._parser.format_help(), + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "20gig" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which platform we are running on + containerPlatform = platform.system().lower() + + if containerPlatform != "windows": + logger.action( + "[20gig] Diagnostic skipped. Current platform is not affected by the bug this diagnostic checks\n" + ) + return True + + buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + + # Attempt to build the Dockerfile + logger.action( + "[20gig] Attempting to COPY more than 20GiB between layers...", False + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnostic20Gig.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[20gig] Diagnostic succeeded! The Docker daemon can COPY more than 20GiB between layers.\n" + ) + else: + logger.error( + "[20gig] Diagnostic failed! The Docker daemon cannot COPY more than 20GiB between layers.\n", + True, + ) + + return built diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index 2fb53322..f4e50e4a 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -4,74 +4,113 @@ import argparse, os, platform from os.path import abspath, dirname, join + class diagnostic8Gig(DiagnosticBase): - - # The tag we use for built images - IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:8gig' - - def __init__(self): - - # Setup our argument parser so we can use its help message output in our description text - self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics 8gig') - self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") - self._parser.add_argument('--random', action='store_true', help="Create a file filled with random bytes instead of zeroes under Windows") - self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - return 'Check for Docker 8GiB filesystem layer bug' - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - return '\n'.join([ - 'This diagnostic determines if the Docker daemon suffers from one of the 8GiB filesystem', - 'layer bugs reported at https://github.com/moby/moby/issues/37581 (affects all platforms)', - 'or https://github.com/moby/moby/issues/40444 (affects Windows containers only)', - '', - '#37581 was fixed in Docker CE 18.09.0 and #40444 was fixed in Docker CE 20.10.0', - '', - self._parser.format_help() - ]) - - def getPrefix(self): - ''' - Returns the short name of the diagnostic for use in log output - ''' - return '8gig' - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - - # Parse our supplied arguments - args = self._parser.parse_args(args) - - # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) - containerPlatform = 'linux' if args.linux == True or platform.system().lower() != 'windows' else 'windows' - - # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) - try: - self._checkPlatformMistmatch(logger, containerPlatform) - except RuntimeError: - return False - - # Set our build arguments when testing Windows containers - buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) if containerPlatform == 'windows' else [] - - # Attempt to build the Dockerfile - logger.action('[8gig] Attempting to build an image with an 8GiB filesystem layer...', False) - built = self._buildDockerfile(logger, containerPlatform, diagnostic8Gig.IMAGE_TAG, buildArgs) - - # Inform the user of the outcome of the diagnostic - if built == True: - logger.action('[8gig] Diagnostic succeeded! The Docker daemon can build images with 8GiB filesystem layers.\n') - else: - logger.error('[8gig] Diagnostic failed! The Docker daemon cannot build images with 8GiB filesystem layers.\n', True) - - return built + + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:8gig" + + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 8gig") + self._parser.add_argument( + "--linux", + action="store_true", + help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)", + ) + self._parser.add_argument( + "--random", + action="store_true", + help="Create a file filled with random bytes instead of zeroes under Windows", + ) + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check for Docker 8GiB filesystem layer bug" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if the Docker daemon suffers from one of the 8GiB filesystem", + "layer bugs reported at https://github.com/moby/moby/issues/37581 (affects all platforms)", + "or https://github.com/moby/moby/issues/40444 (affects Windows containers only)", + "", + "#37581 was fixed in Docker CE 18.09.0 and #40444 was fixed in Docker CE 20.10.0", + "", + self._parser.format_help(), + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "8gig" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = ( + "linux" + if args.linux == True or platform.system().lower() != "windows" + else "windows" + ) + + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + try: + self._checkPlatformMistmatch(logger, containerPlatform) + except RuntimeError: + return False + + # Set our build arguments when testing Windows containers + buildArgs = ( + self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + if containerPlatform == "windows" + else [] + ) + + # Attempt to build the Dockerfile + logger.action( + "[8gig] Attempting to build an image with an 8GiB filesystem layer...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnostic8Gig.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[8gig] Diagnostic succeeded! The Docker daemon can build images with 8GiB filesystem layers.\n" + ) + else: + logger.error( + "[8gig] Diagnostic failed! The Docker daemon cannot build images with 8GiB filesystem layers.\n", + True, + ) + + return built diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index 2101aa93..1a5dbef3 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -4,38 +4,53 @@ from .diagnostic_maxsize import diagnosticMaxSize from .diagnostic_network import diagnosticNetwork + class allDiagnostics(DiagnosticBase): - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - return 'Run all available diagnostics' - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - return 'This diagnostic runs all available diagnostics in sequence.' - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - - # Run all available diagnostics in turn, storing the results - results = [] - diagnostics = [diagnostic8Gig(), diagnostic20Gig(), diagnosticMaxSize(), diagnosticNetwork()] - for index, diagnostic in enumerate(diagnostics): - - # Run the diagnostic and report its result - logger.info('[all] Running individual diagnostic: "{}"'.format(diagnostic.getName()), True) - results.append(diagnostic.run(logger)) - logger.info('[all] Individual diagnostic result: {}'.format('passed' if results[-1] == True else 'failed'), False) - - # Print a newline after the last diagnostic has run - if index == len(diagnostics) - 1: - print() - - # Only report success if all diagnostics succeeded - return False not in results + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Run all available diagnostics" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "This diagnostic runs all available diagnostics in sequence." + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Run all available diagnostics in turn, storing the results + results = [] + diagnostics = [ + diagnostic8Gig(), + diagnostic20Gig(), + diagnosticMaxSize(), + diagnosticNetwork(), + ] + for index, diagnostic in enumerate(diagnostics): + + # Run the diagnostic and report its result + logger.info( + '[all] Running individual diagnostic: "{}"'.format( + diagnostic.getName() + ), + True, + ) + results.append(diagnostic.run(logger)) + logger.info( + "[all] Individual diagnostic result: {}".format( + "passed" if results[-1] == True else "failed" + ), + False, + ) + + # Print a newline after the last diagnostic has run + if index == len(diagnostics) - 1: + print() + + # Only report success if all diagnostics succeeded + return False not in results diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py index 0b831eb7..787d32de 100644 --- a/ue4docker/diagnostics/diagnostic_maxsize.py +++ b/ue4docker/diagnostics/diagnostic_maxsize.py @@ -2,62 +2,89 @@ from .base import DiagnosticBase import platform + class diagnosticMaxSize(DiagnosticBase): - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - return 'Check for Windows storage-opt bug' - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - return '\n'.join([ - 'This diagnostic determines if the Windows host OS suffers from the bug that', - 'prevents users from increasing the maximum allowed image size using Docker\'s', - '`storage-opt` configuration key, as reported here:', - 'https://github.com/docker/for-win/issues/4100', - '', - 'This bug is present in Windows Server and Windows 10 versions 1903 and 1909,', - 'but a workaround was introduced in Docker CE version 19.03.6.', - ]) - - def getPrefix(self): - ''' - Returns the short name of the diagnostic for use in log output - ''' - return 'maxsize' - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - - # Verify that we are running under Windows and are in Windows container mode if using Docker Desktop - dockerPlatform = DockerUtils.info()['OSType'].lower() - hostPlatform = platform.system().lower() - if hostPlatform != 'windows': - logger.info('[maxsize] This diagnostic only applies to Windows host systems.', False) - return True - elif dockerPlatform != 'windows': - logger.error('[maxsize] Error: Docker Desktop is currently in Linux container mode.', False) - logger.error('[maxsize] Please switch to Windows container mode to run this diagnostic.', False) - return False - - # Verify that we are running Windows Server or Windows 10 version 1903 or newer - if WindowsUtils.getWindowsBuild() < 18362: - logger.info('[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.', False) - return True - - # Attempt to run a Windows Nanoserver 1903 container with the `storage-mode` configuration options set - # (The bug doesn't seem to be triggered when using older Windows images, presumably because they use an older host kernel) - try: - command = ['docker', 'run', '--rm', '--storage-opt', 'size=200GB', 'mcr.microsoft.com/windows/nanoserver:1903', 'cmd', 'exit'] - self._printAndRun(logger, '[maxsize] ', command, check=True) - logger.action('[maxsize] Diagnostic succeeded! The host platform can specify maximum image sizes using Docker\'s `storage-opt` configuration key.\n') - return True - except: - logger.error('[maxsize] Diagnostic failed! The host platform cannot specify maximum image sizes using Docker\'s `storage-opt` configuration key.\n', True) - return False + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check for Windows storage-opt bug" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if the Windows host OS suffers from the bug that", + "prevents users from increasing the maximum allowed image size using Docker's", + "`storage-opt` configuration key, as reported here:", + "https://github.com/docker/for-win/issues/4100", + "", + "This bug is present in Windows Server and Windows 10 versions 1903 and 1909,", + "but a workaround was introduced in Docker CE version 19.03.6.", + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "maxsize" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Verify that we are running under Windows and are in Windows container mode if using Docker Desktop + dockerPlatform = DockerUtils.info()["OSType"].lower() + hostPlatform = platform.system().lower() + if hostPlatform != "windows": + logger.info( + "[maxsize] This diagnostic only applies to Windows host systems.", False + ) + return True + elif dockerPlatform != "windows": + logger.error( + "[maxsize] Error: Docker Desktop is currently in Linux container mode.", + False, + ) + logger.error( + "[maxsize] Please switch to Windows container mode to run this diagnostic.", + False, + ) + return False + + # Verify that we are running Windows Server or Windows 10 version 1903 or newer + if WindowsUtils.getWindowsBuild() < 18362: + logger.info( + "[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.", + False, + ) + return True + + # Attempt to run a Windows Nanoserver 1903 container with the `storage-mode` configuration options set + # (The bug doesn't seem to be triggered when using older Windows images, presumably because they use an older host kernel) + try: + command = [ + "docker", + "run", + "--rm", + "--storage-opt", + "size=200GB", + "mcr.microsoft.com/windows/nanoserver:1903", + "cmd", + "exit", + ] + self._printAndRun(logger, "[maxsize] ", command, check=True) + logger.action( + "[maxsize] Diagnostic succeeded! The host platform can specify maximum image sizes using Docker's `storage-opt` configuration key.\n" + ) + return True + except: + logger.error( + "[maxsize] Diagnostic failed! The host platform cannot specify maximum image sizes using Docker's `storage-opt` configuration key.\n", + True, + ) + return False diff --git a/ue4docker/diagnostics/diagnostic_network.py b/ue4docker/diagnostics/diagnostic_network.py index 35338f13..b4e00d1a 100644 --- a/ue4docker/diagnostics/diagnostic_network.py +++ b/ue4docker/diagnostics/diagnostic_network.py @@ -2,71 +2,109 @@ from .base import DiagnosticBase import argparse, platform + class diagnosticNetwork(DiagnosticBase): - - # The tag we use for built images - IMAGE_TAG = 'adamrehn/ue4-docker/diagnostics:network' - - def __init__(self): - - # Setup our argument parser so we can use its help message output in our description text - self._parser = argparse.ArgumentParser(prog='ue4-docker diagnostics network') - self._parser.add_argument('--linux', action='store_true', help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)") - self._parser.add_argument('--isolation', default=None, choices=['hyperv', 'process'], help="Override the default isolation mode when testing Windows containers") - self._parser.add_argument('-basetag', default=None, choices=WindowsUtils.getKnownBaseTags(), help="Override the default base image tag when testing Windows containers") - - def getName(self): - ''' - Returns the human-readable name of the diagnostic - ''' - return 'Check that containers can access the internet correctly' - - def getDescription(self): - ''' - Returns a description of what the diagnostic does - ''' - return '\n'.join([ - 'This diagnostic determines if running containers are able to access the internet,', - 'resolve DNS entries, and download remote files.', - '', - 'This is primarily useful in troubleshooting network connectivity and proxy issues.' - ]) - - def getPrefix(self): - ''' - Returns the short name of the diagnostic for use in log output - ''' - return 'network' - - def run(self, logger, args=[]): - ''' - Runs the diagnostic - ''' - - # Parse our supplied arguments - args = self._parser.parse_args(args) - - # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) - containerPlatform = 'linux' if args.linux == True or platform.system().lower() != 'windows' else 'windows' - - # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) - try: - self._checkPlatformMistmatch(logger, containerPlatform) - except RuntimeError: - return False - - # Set our build arguments when testing Windows containers - buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) if containerPlatform == 'windows' else [] - - # Attempt to build the Dockerfile - logger.action('[network] Attempting to build an image that accesses network resources...', False) - built = self._buildDockerfile(logger, containerPlatform, diagnosticNetwork.IMAGE_TAG, buildArgs) - - # Inform the user of the outcome of the diagnostic - if built == True: - logger.action('[network] Diagnostic succeeded! Running containers can access network resources without any issues.\n') - else: - logger.error('[network] Diagnostic failed! Running containers cannot access network resources. See the docs for troubleshooting tips:', True) - logger.error('[network] https://docs.adamrehn.com/ue4-docker/building-images/troubleshooting-build-issues#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n', False) - - return built + + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:network" + + def __init__(self): + + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics network") + self._parser.add_argument( + "--linux", + action="store_true", + help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)", + ) + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check that containers can access the internet correctly" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if running containers are able to access the internet,", + "resolve DNS entries, and download remote files.", + "", + "This is primarily useful in troubleshooting network connectivity and proxy issues.", + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "network" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = ( + "linux" + if args.linux == True or platform.system().lower() != "windows" + else "windows" + ) + + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + try: + self._checkPlatformMistmatch(logger, containerPlatform) + except RuntimeError: + return False + + # Set our build arguments when testing Windows containers + buildArgs = ( + self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + if containerPlatform == "windows" + else [] + ) + + # Attempt to build the Dockerfile + logger.action( + "[network] Attempting to build an image that accesses network resources...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnosticNetwork.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[network] Diagnostic succeeded! Running containers can access network resources without any issues.\n" + ) + else: + logger.error( + "[network] Diagnostic failed! Running containers cannot access network resources. See the docs for troubleshooting tips:", + True, + ) + logger.error( + "[network] https://docs.adamrehn.com/ue4-docker/building-images/troubleshooting-build-issues#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n", + False, + ) + + return built diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index ba3e3ebb..829ebbc9 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -2,63 +2,72 @@ from .diagnostics import * import sys + def diagnostics(): - - # The diagnostics that can be run - DIAGNOSTICS = { - 'all': allDiagnostics(), - '8gig': diagnostic8Gig(), - '20gig': diagnostic20Gig(), - 'maxsize': diagnosticMaxSize(), - 'network': diagnosticNetwork() - } - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} diagnostics] '.format(sys.argv[0])) - - # Parse the supplied command-line arguments - stripped = list([arg for arg in sys.argv if arg.strip('-') not in ['h', 'help']]) - args = { - 'help': len(sys.argv) > len(stripped), - 'diagnostic': stripped[1] if len(stripped) > 1 else None, - } - - # If a diagnostic name has been specified, verify that it is valid - if args['diagnostic'] is not None and args['diagnostic'] not in DIAGNOSTICS: - logger.error('Error: unrecognised diagnostic "{}".'.format(args['diagnostic']), False) - sys.exit(1) - - # Determine if we are running a diagnostic - if args['help'] == False and args['diagnostic'] is not None: - - # Run the diagnostic - diagnostic = DIAGNOSTICS[args['diagnostic']] - logger.action('Running diagnostic: "{}"'.format(diagnostic.getName()), False) - passed = diagnostic.run(logger, stripped[2:]) - - # Print the result - if passed == True: - logger.action('Diagnostic result: passed', False) - else: - logger.error('Diagnostic result: failed', False) - - # Determine if we are displaying the help for a specific diagnostic - elif args['help'] == True and args['diagnostic'] is not None: - - # Display the help for the diagnostic - diagnostic = DIAGNOSTICS[args['diagnostic']] - print('{} diagnostics {}'.format(sys.argv[0], args['diagnostic'])) - print(diagnostic.getName() + '\n') - print(diagnostic.getDescription()) - - else: - - # Print usage syntax - print('Usage: {} diagnostics DIAGNOSTIC\n'.format(sys.argv[0])) - print('Runs diagnostics to detect issues with the host system configuration\n') - print('Available diagnostics:') - PrettyPrinting.printColumns([ - (diagnostic, DIAGNOSTICS[diagnostic].getName()) - for diagnostic in DIAGNOSTICS - ]) - print('\nRun `{} diagnostics DIAGNOSTIC --help` for more information on a diagnostic.'.format(sys.argv[0])) + + # The diagnostics that can be run + DIAGNOSTICS = { + "all": allDiagnostics(), + "8gig": diagnostic8Gig(), + "20gig": diagnostic20Gig(), + "maxsize": diagnosticMaxSize(), + "network": diagnosticNetwork(), + } + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} diagnostics] ".format(sys.argv[0])) + + # Parse the supplied command-line arguments + stripped = list([arg for arg in sys.argv if arg.strip("-") not in ["h", "help"]]) + args = { + "help": len(sys.argv) > len(stripped), + "diagnostic": stripped[1] if len(stripped) > 1 else None, + } + + # If a diagnostic name has been specified, verify that it is valid + if args["diagnostic"] is not None and args["diagnostic"] not in DIAGNOSTICS: + logger.error( + 'Error: unrecognised diagnostic "{}".'.format(args["diagnostic"]), False + ) + sys.exit(1) + + # Determine if we are running a diagnostic + if args["help"] == False and args["diagnostic"] is not None: + + # Run the diagnostic + diagnostic = DIAGNOSTICS[args["diagnostic"]] + logger.action('Running diagnostic: "{}"'.format(diagnostic.getName()), False) + passed = diagnostic.run(logger, stripped[2:]) + + # Print the result + if passed == True: + logger.action("Diagnostic result: passed", False) + else: + logger.error("Diagnostic result: failed", False) + + # Determine if we are displaying the help for a specific diagnostic + elif args["help"] == True and args["diagnostic"] is not None: + + # Display the help for the diagnostic + diagnostic = DIAGNOSTICS[args["diagnostic"]] + print("{} diagnostics {}".format(sys.argv[0], args["diagnostic"])) + print(diagnostic.getName() + "\n") + print(diagnostic.getDescription()) + + else: + + # Print usage syntax + print("Usage: {} diagnostics DIAGNOSTIC\n".format(sys.argv[0])) + print("Runs diagnostics to detect issues with the host system configuration\n") + print("Available diagnostics:") + PrettyPrinting.printColumns( + [ + (diagnostic, DIAGNOSTICS[diagnostic].getName()) + for diagnostic in DIAGNOSTICS + ] + ) + print( + "\nRun `{} diagnostics DIAGNOSTIC --help` for more information on a diagnostic.".format( + sys.argv[0] + ) + ) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py b/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py index 04c4add4..c8a797dd 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py @@ -4,22 +4,41 @@ # Determine the root directory for the source build and the Installed Build sourceRoot = sys.argv[1] -installedRoot = join(sourceRoot, 'LocalBuilds', 'Engine', 'Linux') +installedRoot = join(sourceRoot, "LocalBuilds", "Engine", "Linux") # Locate the bundled toolchain and copy it to the Installed Build -sdkGlob = join(sourceRoot, 'Engine', 'Extras', 'ThirdPartyNotUE', 'SDKs', 'HostLinux', 'Linux_x64', '*', 'x86_64-unknown-linux-gnu') +sdkGlob = join( + sourceRoot, + "Engine", + "Extras", + "ThirdPartyNotUE", + "SDKs", + "HostLinux", + "Linux_x64", + "*", + "x86_64-unknown-linux-gnu", +) for bundled in glob.glob(sdkGlob): - - # Extract the root path for the toolchain - toolchain = dirname(bundled) - - # Print progress output - print('Copying bundled toolchain "{}" to Installed Build...'.format(basename(toolchain)), file=sys.stderr) - sys.stderr.flush() - - # Perform the copy - dest = join(installedRoot, relpath(toolchain, sourceRoot)) - if exists(dest) == True: - print('Destination toolchain already exists: {}'.format(dest), file=sys.stderr, flush=True) - else: - shutil.copytree(toolchain, dest) + + # Extract the root path for the toolchain + toolchain = dirname(bundled) + + # Print progress output + print( + 'Copying bundled toolchain "{}" to Installed Build...'.format( + basename(toolchain) + ), + file=sys.stderr, + ) + sys.stderr.flush() + + # Perform the copy + dest = join(installedRoot, relpath(toolchain, sourceRoot)) + if exists(dest) == True: + print( + "Destination toolchain already exists: {}".format(dest), + file=sys.stderr, + flush=True, + ) + else: + shutil.copytree(toolchain, dest) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py b/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py index 87d4670f..597992ed 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py @@ -1,19 +1,19 @@ #!/usr/bin/env python3 import os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Enable the OpenGL RHI for Engine versions where it is present but deprecated iniFile = sys.argv[1] config = readFile(iniFile) -config = config.replace( - '; +TargetedRHIs=GLSL_430', - '+TargetedRHIs=GLSL_430' -) +config = config.replace("; +TargetedRHIs=GLSL_430", "+TargetedRHIs=GLSL_430") writeFile(iniFile, config) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py b/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py index f573d403..6ce9a427 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 import os, re, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini iniFile = sys.argv[1] config = readFile(iniFile) -config = re.sub('PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', 'PlatformType="\\2", RequiredFile="\\1"', config) +config = re.sub( + 'PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', + 'PlatformType="\\2", RequiredFile="\\1"', + config, +) writeFile(iniFile, config) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index b47bface..f281b800 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 import json, os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Read the build graph XML buildXml = sys.argv[1] @@ -19,27 +22,31 @@ def writeFile(filename, data): # Add verbose output flags to the `BuildDerivedDataCache` command code = code.replace( - 'Command Name="BuildDerivedDataCache" Arguments="', - 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ' + 'Command Name="BuildDerivedDataCache" Arguments="', + 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ', ) # Disable building for AArch64 by default (enabled by default since 4.24.0) code = code.replace( - 'Property Name="DefaultWithLinuxAArch64" Value="true"', - 'Property Name="DefaultWithLinuxAArch64" Value="false"' + 'Property Name="DefaultWithLinuxAArch64" Value="true"', + 'Property Name="DefaultWithLinuxAArch64" Value="false"', ) # Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail # (See for details of the bug and its fix) -if versionData['MajorVersion'] != 4 or versionData['MinorVersion'] != 24 or versionData['PatchVersion'] >= 3: - code = code.replace( - 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' - ) - code = code.replace( - 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' - ) +if ( + versionData["MajorVersion"] != 4 + or versionData["MinorVersion"] != 24 + or versionData["PatchVersion"] >= 3 +): + code = code.replace( + 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"', + ) + code = code.replace( + 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"', + ) # Write the modified XML back to disk writeFile(buildXml, code) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py index c6afa462..cc2f1de7 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 import os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + filtersXml = sys.argv[1] code = readFile(filtersXml) @@ -15,9 +18,9 @@ def writeFile(filename, data): # Add missing SetupDotnet.sh in Unreal Engine 5.0.0-early-access-1 # See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 # and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 -dotnet = 'Engine/Binaries/ThirdParty/DotNet/Linux/...' -setup_dotnet = 'Engine/Build/BatchFiles/Linux/SetupDotnet.sh' +dotnet = "Engine/Binaries/ThirdParty/DotNet/Linux/..." +setup_dotnet = "Engine/Build/BatchFiles/Linux/SetupDotnet.sh" if dotnet in code and setup_dotnet not in code: - code = code.replace(dotnet, f'{dotnet}\n{setup_dotnet}') + code = code.replace(dotnet, f"{dotnet}\n{setup_dotnet}") writeFile(filtersXml, code) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py index 7535f234..fc5aa38a 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 import json, os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file versionFile = sys.argv[1] details = json.loads(readFile(versionFile)) -details['Changelist'] = details['CompatibleChangelist'] -details['IsPromotedBuild'] = 1 +details["Changelist"] = details["CompatibleChangelist"] +details["IsPromotedBuild"] = 1 writeFile(versionFile, json.dumps(details, indent=4)) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py index 4807e06b..5ec6243b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py @@ -4,39 +4,38 @@ # Logs a message to stderr def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() + print(message, file=sys.stderr) + sys.stderr.flush() + # Extracts the files and directories for the specified component and moves them to a separate output directory def extractComponent(inputDir, outputDir, component, description, items): - - # Print progress output - log('\nExtracting {}...'.format(description)) - - # Create the output directory for the component if it doesn't already exist - componentDir = join(outputDir, component) - os.makedirs(outputDir, exist_ok=True) - - # Move each file and directory for the component to the output directory - for item in items: - - # Verify that the item exists - if not exists(item): - log('Skipping non-existent item: {}'.format(item)) - continue - - # Print progress output - log('Moving: {}'.format(item)) - - # Ensure the parent directory for the item exists in the output directory - parent = dirname(item).replace(inputDir, componentDir) - os.makedirs(parent, exist_ok=True) - - # Perform the move - shutil.move( - item, - join(parent, basename(item)) - ) + + # Print progress output + log("\nExtracting {}...".format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + + # Verify that the item exists + if not exists(item): + log("Skipping non-existent item: {}".format(item)) + continue + + # Print progress output + log("Moving: {}".format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move(item, join(parent, basename(item))) + # Retrieve the path to the root directory of the Installed Build rootDir = sys.argv[1] @@ -46,13 +45,17 @@ def extractComponent(inputDir, outputDir, component, description, items): os.makedirs(outputDir, exist_ok=True) # Extract the DDC -ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')] -extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc) +ddc = [join(rootDir, "Engine", "DerivedDataCache", "Compressed.ddp")] +extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) # Extract debug symbols -symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True) -extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles) +symbolFiles = glob.glob(join(rootDir, "**", "*.debug"), recursive=True) + glob.glob( + join(rootDir, "**", "*.sym"), recursive=True +) +extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) # Extract template projects and samples -subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']] -extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs) +subdirs = [join(rootDir, subdir) for subdir in ["FeaturePacks", "Samples", "Templates"]] +extractComponent( + rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs +) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py b/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py index f573d403..6ce9a427 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py @@ -1,16 +1,23 @@ #!/usr/bin/env python3 import os, re, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini iniFile = sys.argv[1] config = readFile(iniFile) -config = re.sub('PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', 'PlatformType="\\2", RequiredFile="\\1"', config) +config = re.sub( + 'PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', + 'PlatformType="\\2", RequiredFile="\\1"', + config, +) writeFile(iniFile, config) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py index 8b30764f..06de3a2a 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 import os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Read the build graph XML buildXml = sys.argv[1] @@ -15,18 +18,18 @@ def writeFile(filename, data): # Prevent HoloLens support from being enabled by default under Windows in 4.23.0 onwards code = code.replace( - 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', - 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"' + 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', + 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"', ) # Enable client and server targets by default in 4.23.0 onwards code = code.replace( - 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"' + 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"', ) code = code.replace( - 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"' + 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', + 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"', ) # Write the modified XML back to disk diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py index af7de086..20c9e254 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 import os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Remove the dependency on Linux cross-compilation debug tools introduced in UE4.20.0 filtersXml = sys.argv[1] code = readFile(filtersXml) -code = code.replace('Engine/Binaries/Linux/dump_syms.exe', '') -code = code.replace('Engine/Binaries/Linux/BreakpadSymbolEncoder.exe', '') +code = code.replace("Engine/Binaries/Linux/dump_syms.exe", "") +code = code.replace("Engine/Binaries/Linux/BreakpadSymbolEncoder.exe", "") writeFile(filtersXml, code) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index 7535f234..fc5aa38a 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -1,17 +1,20 @@ #!/usr/bin/env python3 import json, os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file versionFile = sys.argv[1] details = json.loads(readFile(versionFile)) -details['Changelist'] = details['CompatibleChangelist'] -details['IsPromotedBuild'] = 1 +details["Changelist"] = details["CompatibleChangelist"] +details["IsPromotedBuild"] = 1 writeFile(versionFile, json.dumps(details, indent=4)) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py index 116e7fc2..b3bcf03e 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -4,39 +4,38 @@ # Logs a message to stderr def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() + print(message, file=sys.stderr) + sys.stderr.flush() + # Extracts the files and directories for the specified component and moves them to a separate output directory def extractComponent(inputDir, outputDir, component, description, items): - - # Print progress output - log('\nExtracting {}...'.format(description)) - - # Create the output directory for the component if it doesn't already exist - componentDir = join(outputDir, component) - os.makedirs(outputDir, exist_ok=True) - - # Move each file and directory for the component to the output directory - for item in items: - - # Verify that the item exists - if not exists(item): - log('Skipping non-existent item: {}'.format(item)) - continue - - # Print progress output - log('Moving: {}'.format(item)) - - # Ensure the parent directory for the item exists in the output directory - parent = dirname(item).replace(inputDir, componentDir) - os.makedirs(parent, exist_ok=True) - - # Perform the move - shutil.move( - item, - join(parent, basename(item)) - ) + + # Print progress output + log("\nExtracting {}...".format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + + # Verify that the item exists + if not exists(item): + log("Skipping non-existent item: {}".format(item)) + continue + + # Print progress output + log("Moving: {}".format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move(item, join(parent, basename(item))) + # Retrieve the path to the root directory of the Installed Build rootDir = sys.argv[1] @@ -46,13 +45,15 @@ def extractComponent(inputDir, outputDir, component, description, items): os.makedirs(outputDir, exist_ok=True) # Extract the DDC -ddc = [join(rootDir, 'Engine', 'DerivedDataCache', 'Compressed.ddp')] -extractComponent(rootDir, outputDir, 'DDC', 'Derived Data Cache (DDC)', ddc) +ddc = [join(rootDir, "Engine", "DerivedDataCache", "Compressed.ddp")] +extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) # Extract debug symbols -symbolFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True) -extractComponent(rootDir, outputDir, 'DebugSymbols', 'debug symbols', symbolFiles) +symbolFiles = glob.glob(join(rootDir, "**", "*.pdb"), recursive=True) +extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) # Extract template projects and samples -subdirs = [join(rootDir, subdir) for subdir in ['FeaturePacks', 'Samples', 'Templates']] -extractComponent(rootDir, outputDir, 'TemplatesAndSamples', 'template projects and samples', subdirs) +subdirs = [join(rootDir, subdir) for subdir in ["FeaturePacks", "Samples", "Templates"]] +extractComponent( + rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs +) diff --git a/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py b/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py index ff75f591..f63a12b4 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py +++ b/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py @@ -6,7 +6,11 @@ systemLd = sys.argv[2] # Locate the bundled version(s) of ld and replace them with symlinks to the system ld -for bundled in glob.glob(os.path.join(sdkRoot, '*', 'x86_64-unknown-linux-gnu', 'bin', 'x86_64-unknown-linux-gnu-ld')): - os.unlink(bundled) - os.symlink(systemLd, bundled) - print('{} => {}'.format(bundled, systemLd), file=sys.stderr) +for bundled in glob.glob( + os.path.join( + sdkRoot, "*", "x86_64-unknown-linux-gnu", "bin", "x86_64-unknown-linux-gnu-ld" + ) +): + os.unlink(bundled) + os.symlink(systemLd, bundled) + print("{} => {}".format(bundled, systemLd), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py index 9919defb..177f8247 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -2,45 +2,54 @@ import json, os, subprocess, sys from os.path import join + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] -verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' -versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) -if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: - - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them - # (See: ) - gitdepsFile = join(engineRoot, 'Engine', 'Build', 'Commit.gitdeps.xml') - gitdepsXml = readFile(gitdepsFile) - if '', - '\n ' + - '' - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' + - '' - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' + - '' - ) - - writeFile(gitdepsFile, gitdepsXml) - - if verboseOutput == True: - print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) - else: - print('PATCHED {}'.format(gitdepsFile), file=sys.stderr) +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" +versionDetails = json.loads( + readFile(join(engineRoot, "Engine", "Build", "Build.version")) +) +if ( + versionDetails["MajorVersion"] == 4 + and versionDetails["MinorVersion"] == 25 + and versionDetails["PatchVersion"] == 4 +): + + # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them + # (See: ) + gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") + gitdepsXml = readFile(gitdepsFile) + if '', + '\n ' + + '', + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '', + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '', + ) + + writeFile(gitdepsFile, gitdepsXml) + + if verboseOutput == True: + print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) + else: + print("PATCHED {}".format(gitdepsFile), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py b/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py index 2d6c8053..435ddd90 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py @@ -1,25 +1,29 @@ #!/usr/bin/env python3 import os, re, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + def patchFile(filename, search, replace): - contents = readFile(filename) - patched = contents.replace(search, replace) - writeFile(filename, patched) + contents = readFile(filename) + patched = contents.replace(search, replace) + writeFile(filename, patched) + # Apply our bugfixes to UnrealBuildTool (UBT) # Ensure modules always have a binary output directory value set when exporting JSON # (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) patchFile( - os.path.join(sys.argv[1], 'Configuration', 'UEBuildTarget.cs'), - 'Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);', - 'Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);' + os.path.join(sys.argv[1], "Configuration", "UEBuildTarget.cs"), + "Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);", + "Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);", ) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index 9919defb..177f8247 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -2,45 +2,54 @@ import json, os, subprocess, sys from os.path import join + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] -verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' -versionDetails = json.loads(readFile(join(engineRoot, 'Engine', 'Build', 'Build.version'))) -if versionDetails['MajorVersion'] == 4 and versionDetails['MinorVersion'] == 25 and versionDetails['PatchVersion'] == 4: - - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them - # (See: ) - gitdepsFile = join(engineRoot, 'Engine', 'Build', 'Commit.gitdeps.xml') - gitdepsXml = readFile(gitdepsFile) - if '', - '\n ' + - '' - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' + - '' - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' + - '' - ) - - writeFile(gitdepsFile, gitdepsXml) - - if verboseOutput == True: - print('PATCHED {}:\n\n{}'.format(gitdepsFile, gitdepsXml), file=sys.stderr) - else: - print('PATCHED {}'.format(gitdepsFile), file=sys.stderr) +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" +versionDetails = json.loads( + readFile(join(engineRoot, "Engine", "Build", "Build.version")) +) +if ( + versionDetails["MajorVersion"] == 4 + and versionDetails["MinorVersion"] == 25 + and versionDetails["PatchVersion"] == 4 +): + + # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them + # (See: ) + gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") + gitdepsXml = readFile(gitdepsFile) + if '', + '\n ' + + '', + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '', + ) + gitdepsXml = gitdepsXml.replace( + '', + '\n ' + + '', + ) + + writeFile(gitdepsFile, gitdepsXml) + + if verboseOutput == True: + print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) + else: + print("PATCHED {}".format(gitdepsFile), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py b/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py index 2415a446..5ba8d924 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py @@ -1,33 +1,40 @@ #!/usr/bin/env python3 import os, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + # Comment out the call to the UE4 prereqs installer in Setup.bat -PREREQ_CALL = 'start /wait Engine\\Extras\\Redist\\en-us\\UE4PrereqSetup_x64.exe' +PREREQ_CALL = "start /wait Engine\\Extras\\Redist\\en-us\\UE4PrereqSetup_x64.exe" setupScript = sys.argv[1] -verboseOutput = len(sys.argv) > 2 and sys.argv[2] == '1' +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" code = readFile(setupScript) -code = code.replace('echo Installing prerequisites...', 'echo (Skipping installation of prerequisites)') -code = code.replace(PREREQ_CALL, '@rem ' + PREREQ_CALL) +code = code.replace( + "echo Installing prerequisites...", "echo (Skipping installation of prerequisites)" +) +code = code.replace(PREREQ_CALL, "@rem " + PREREQ_CALL) # Also comment out the version selector call, since we don't need shell integration -SELECTOR_CALL = '.\\Engine\\Binaries\\Win64\\UnrealVersionSelector-Win64-Shipping.exe /register' -code = code.replace(SELECTOR_CALL, '@rem ' + SELECTOR_CALL) +SELECTOR_CALL = ( + ".\\Engine\\Binaries\\Win64\\UnrealVersionSelector-Win64-Shipping.exe /register" +) +code = code.replace(SELECTOR_CALL, "@rem " + SELECTOR_CALL) # Add output so we can see when script execution is complete, and ensure `pause` is not called on error -code = code.replace('rem Done!', 'echo Done!\r\nexit /b 0') -code = code.replace('pause', '@rem pause') +code = code.replace("rem Done!", "echo Done!\r\nexit /b 0") +code = code.replace("pause", "@rem pause") writeFile(setupScript, code) # Print the patched code to stderr for debug purposes if verboseOutput == True: - print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) + print("PATCHED {}:\n\n{}".format(setupScript, code), file=sys.stderr) else: - print('PATCHED {}'.format(setupScript), file=sys.stderr) + print("PATCHED {}".format(setupScript), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py b/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py index 2d6c8053..435ddd90 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py @@ -1,25 +1,29 @@ #!/usr/bin/env python3 import os, re, sys + def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + def patchFile(filename, search, replace): - contents = readFile(filename) - patched = contents.replace(search, replace) - writeFile(filename, patched) + contents = readFile(filename) + patched = contents.replace(search, replace) + writeFile(filename, patched) + # Apply our bugfixes to UnrealBuildTool (UBT) # Ensure modules always have a binary output directory value set when exporting JSON # (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) patchFile( - os.path.join(sys.argv[1], 'Configuration', 'UEBuildTarget.cs'), - 'Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);', - 'Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);' + os.path.join(sys.argv[1], "Configuration", "UEBuildTarget.cs"), + "Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);", + "Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);", ) diff --git a/ue4docker/export.py b/ue4docker/export.py index c34e6ee4..3e98fcda 100644 --- a/ue4docker/export.py +++ b/ue4docker/export.py @@ -2,88 +2,108 @@ from .exports import * import sys + def _notNone(items): - return len([i for i in items if i is not None]) == len(items) + return len([i for i in items if i is not None]) == len(items) + def _extractArg(args, index): - return args[index] if len(args) > index else None + return args[index] if len(args) > index else None + def _isHelpFlag(arg): - return (arg.strip('-') in ['h', 'help']) == True + return (arg.strip("-") in ["h", "help"]) == True + def _stripHelpFlags(args): - return list([a for a in args if _isHelpFlag(a) == False]) + return list([a for a in args if _isHelpFlag(a) == False]) + def export(): - - # The components that can be exported - COMPONENTS = { - 'installed': { - 'function': exportInstalledBuild, - 'description': 'Exports an Installed Build of the Engine', - 'image': GlobalConfiguration.resolveTag('ue4-full'), - 'help': 'Copies the Installed Build from a container to the host system.\nOnly supported under Linux for UE 4.21.0 and newer.' - }, - 'packages': { - 'function': exportPackages, - 'description': 'Exports conan-ue4cli wrapper packages', - 'image': GlobalConfiguration.resolveTag('ue4-full'), - 'help': - 'Runs a temporary conan server inside a container and uses it to export the\ngenerated conan-ue4cli wrapper packages.\n\n' + - 'Currently the only supported destination value is "cache", which exports\nthe packages to the Conan local cache on the host system.' - } - } - - # Parse the supplied command-line arguments - stripped = _stripHelpFlags(sys.argv) - args = { - 'help': len(stripped) < len(sys.argv), - 'component': _extractArg(stripped, 1), - 'tag': _extractArg(stripped, 2), - 'destination': _extractArg(stripped, 3) - } - - # If a component name has been specified, verify that it is valid - if args['component'] is not None and args['component'] not in COMPONENTS: - print('Error: unrecognised component "{}".'.format(args['component']), file=sys.stderr) - sys.exit(1) - - # Determine if we are performing an export - if args['help'] == False and _notNone([args['component'], args['tag'], args['destination']]): - - # Determine if the user specified an image and a tag or just a tag - tag = args['tag'] - details = COMPONENTS[ args['component'] ] - requiredImage = '{}:{}'.format(details['image'], tag) if ':' not in tag else tag - - # Verify that the required container image exists - if DockerUtils.exists(requiredImage) == False: - print('Error: the specified container image "{}" does not exist.'.format(requiredImage), file=sys.stderr) - sys.exit(1) - - # Attempt to perform the export - details['function'](requiredImage, args['destination'], stripped[4:]) - print('Export complete.') - - # Determine if we are displaying the help for a specific component - elif args['help'] == True and args['component'] is not None: - - # Display the help for the component - component = sys.argv[1] - details = COMPONENTS[component] - print('{} export {}'.format(sys.argv[0], component)) - print(details['description'] + '\n') - print('Exports from image: {}:TAG\n'.format(details['image'])) - print(details['help']) - - else: - - # Print usage syntax - print('Usage: {} export COMPONENT TAG DESTINATION\n'.format(sys.argv[0])) - print('Exports components from built container images to the host system\n') - print('Components:') - PrettyPrinting.printColumns([ - (component, COMPONENTS[component]['description']) - for component in COMPONENTS - ]) - print('\nRun `{} export COMPONENT --help` for more information on a component.'.format(sys.argv[0])) + + # The components that can be exported + COMPONENTS = { + "installed": { + "function": exportInstalledBuild, + "description": "Exports an Installed Build of the Engine", + "image": GlobalConfiguration.resolveTag("ue4-full"), + "help": "Copies the Installed Build from a container to the host system.\nOnly supported under Linux for UE 4.21.0 and newer.", + }, + "packages": { + "function": exportPackages, + "description": "Exports conan-ue4cli wrapper packages", + "image": GlobalConfiguration.resolveTag("ue4-full"), + "help": "Runs a temporary conan server inside a container and uses it to export the\ngenerated conan-ue4cli wrapper packages.\n\n" + + 'Currently the only supported destination value is "cache", which exports\nthe packages to the Conan local cache on the host system.', + }, + } + + # Parse the supplied command-line arguments + stripped = _stripHelpFlags(sys.argv) + args = { + "help": len(stripped) < len(sys.argv), + "component": _extractArg(stripped, 1), + "tag": _extractArg(stripped, 2), + "destination": _extractArg(stripped, 3), + } + + # If a component name has been specified, verify that it is valid + if args["component"] is not None and args["component"] not in COMPONENTS: + print( + 'Error: unrecognised component "{}".'.format(args["component"]), + file=sys.stderr, + ) + sys.exit(1) + + # Determine if we are performing an export + if args["help"] == False and _notNone( + [args["component"], args["tag"], args["destination"]] + ): + + # Determine if the user specified an image and a tag or just a tag + tag = args["tag"] + details = COMPONENTS[args["component"]] + requiredImage = "{}:{}".format(details["image"], tag) if ":" not in tag else tag + + # Verify that the required container image exists + if DockerUtils.exists(requiredImage) == False: + print( + 'Error: the specified container image "{}" does not exist.'.format( + requiredImage + ), + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to perform the export + details["function"](requiredImage, args["destination"], stripped[4:]) + print("Export complete.") + + # Determine if we are displaying the help for a specific component + elif args["help"] == True and args["component"] is not None: + + # Display the help for the component + component = sys.argv[1] + details = COMPONENTS[component] + print("{} export {}".format(sys.argv[0], component)) + print(details["description"] + "\n") + print("Exports from image: {}:TAG\n".format(details["image"])) + print(details["help"]) + + else: + + # Print usage syntax + print("Usage: {} export COMPONENT TAG DESTINATION\n".format(sys.argv[0])) + print("Exports components from built container images to the host system\n") + print("Components:") + PrettyPrinting.printColumns( + [ + (component, COMPONENTS[component]["description"]) + for component in COMPONENTS + ] + ) + print( + "\nRun `{} export COMPONENT --help` for more information on a component.".format( + sys.argv[0] + ) + ) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py index b072eb0f..e078d05d 100644 --- a/ue4docker/exports/export_installed.py +++ b/ue4docker/exports/export_installed.py @@ -2,52 +2,73 @@ import os, platform, shutil, subprocess, sys # Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) +semver = PackageUtils.importFile( + "semver", os.path.join(PackageUtils.getPackageLocation("semver"), "semver.py") +) + def exportInstalledBuild(image, destination, extraArgs): - - # Verify that we are running under Linux - if platform.system() != 'Linux': - print('Error: Installed Builds can only be exported under Linux.', file=sys.stderr) - sys.exit(1) - - # Verify that the destination directory does not already exist - if os.path.exists(destination) == True: - print('Error: the destination directory already exists.', file=sys.stderr) - sys.exit(1) - - # Verify that the Installed Build in the specified image is at least 4.21.0 - versionResult = SubprocessUtils.capture(['docker', 'run', '--rm', '-ti', image, 'ue4', 'version']) - try: - version = semver.parse(SubprocessUtils.extractLines(versionResult.stdout)[-1]) - if version['minor'] < 21: - raise Exception() - except: - print('Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.', file=sys.stderr) - sys.exit(1) - - # Start a container from which we will copy files - container = DockerUtils.start(image, 'bash') - - # Attempt to perform the export - print('Exporting to {}...'.format(destination)) - containerPath = '{}:/home/ue4/UnrealEngine'.format(container.name) - exportResult = subprocess.call(['docker', 'cp', containerPath, destination]) - - # Stop the container, irrespective of whether or not the export succeeded - container.stop() - - # If the export succeeded, regenerate the linker symlinks on the host system - if exportResult == 0: - print('Performing linker symlink fixup...') - subprocess.call([ - sys.executable, - os.path.join(os.path.dirname(os.path.dirname(__file__)), 'dockerfiles', 'ue4-source', 'linux', 'linker-fixup.py'), - os.path.join(destination, 'Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64'), - shutil.which('ld') - ]) - - # Report any failures - if exportResult != 0: - print('Error: failed to export Installed Build.', file=sys.stderr) - sys.exit(1) + + # Verify that we are running under Linux + if platform.system() != "Linux": + print( + "Error: Installed Builds can only be exported under Linux.", file=sys.stderr + ) + sys.exit(1) + + # Verify that the destination directory does not already exist + if os.path.exists(destination) == True: + print("Error: the destination directory already exists.", file=sys.stderr) + sys.exit(1) + + # Verify that the Installed Build in the specified image is at least 4.21.0 + versionResult = SubprocessUtils.capture( + ["docker", "run", "--rm", "-ti", image, "ue4", "version"] + ) + try: + version = semver.parse(SubprocessUtils.extractLines(versionResult.stdout)[-1]) + if version["minor"] < 21: + raise Exception() + except: + print( + "Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.", + file=sys.stderr, + ) + sys.exit(1) + + # Start a container from which we will copy files + container = DockerUtils.start(image, "bash") + + # Attempt to perform the export + print("Exporting to {}...".format(destination)) + containerPath = "{}:/home/ue4/UnrealEngine".format(container.name) + exportResult = subprocess.call(["docker", "cp", containerPath, destination]) + + # Stop the container, irrespective of whether or not the export succeeded + container.stop() + + # If the export succeeded, regenerate the linker symlinks on the host system + if exportResult == 0: + print("Performing linker symlink fixup...") + subprocess.call( + [ + sys.executable, + os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "dockerfiles", + "ue4-source", + "linux", + "linker-fixup.py", + ), + os.path.join( + destination, + "Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64", + ), + shutil.which("ld"), + ] + ) + + # Report any failures + if exportResult != 0: + print("Error: failed to export Installed Build.", file=sys.stderr) + sys.exit(1) diff --git a/ue4docker/exports/export_packages.py b/ue4docker/exports/export_packages.py index 5328ef01..c78de730 100644 --- a/ue4docker/exports/export_packages.py +++ b/ue4docker/exports/export_packages.py @@ -2,10 +2,10 @@ import docker, os, subprocess, sys, tempfile # The name we use for our temporary Conan remote -REMOTE_NAME = '_ue4docker_export_temp' +REMOTE_NAME = "_ue4docker_export_temp" # Our conan_server config file data -CONAN_SERVER_CONFIG = ''' +CONAN_SERVER_CONFIG = """ [server] jwt_secret: jwt_secret jwt_expire_minutes: 120 @@ -26,132 +26,166 @@ [users] user: password -''' +""" def exportPackages(image, destination, extraArgs): - - # Create our logger to generate coloured output on stderr - logger = Logger() - - # Verify that the destination is "cache" - if destination.lower() != 'cache': - logger.error('Error: the only supported package export destination is "cache".') - sys.exit(1) - - # Verify that Conan is installed on the host - try: - SubprocessUtils.run(['conan', '--version']) - except: - logger.error('Error: Conan must be installed on the host system to export packages.') - sys.exit(1) - - # Determine if the container image is a Windows image or a Linux image - imageOS = DockerUtils.listImages(image)[0].attrs['Os'] - - # Use the appropriate commands and paths for the container platform - cmdsAndPaths = { - - 'linux': { - 'rootCommand': ['bash', '-c', 'sleep infinity'], - 'mkdirCommand': ['mkdir'], - 'copyCommand': ['cp', '-f'], - - 'dataDir': '/home/ue4/.conan_server/data', - 'configDir': '/home/ue4/.conan_server/', - 'bindMount': '/hostdir/' - }, - - 'windows': { - 'rootCommand': ['timeout', '/t', '99999', '/nobreak'], - 'mkdirCommand': ['cmd', '/S', '/C', 'mkdir'], - 'copyCommand': ['xcopy', '/y'], - - 'dataDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\data', - 'configDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\', - 'bindMount': 'C:\\hostdir\\' - } - - }[imageOS] - - # Create an auto-deleting temporary directory to hold our server config file - with tempfile.TemporaryDirectory() as tempDir: - - # Progress output - print('Starting conan_server in a container...') - - # Start a container from which we will export packages, bind-mounting our temp directory - container = DockerUtils.start( - image, - cmdsAndPaths['rootCommand'], - ports = {'9300/tcp': 9300}, - mounts = [docker.types.Mount(cmdsAndPaths['bindMount'], tempDir, 'bind')], - stdin_open = imageOS == 'windows', - tty = imageOS == 'windows', - remove = True - ) - - # Reload the container attributes from the Docker daemon to ensure the networking fields are populated - container.reload() - - # Under Linux we can simply access the container from the host over the loopback address, but this doesn't work under Windows - # (See ) - externalAddress = '127.0.0.1' if imageOS == 'linux' else container.attrs['NetworkSettings']['Networks']['nat']['IPAddress'] - - # Generate our server config file in the temp directory - FilesystemUtils.writeFile(os.path.join(tempDir, 'server.conf'), CONAN_SERVER_CONFIG.format(externalAddress, cmdsAndPaths['dataDir'])) - - # Keep track of the `conan_server` log output so we can display it in case of an error - serverOutput = None - - try: - - # Copy the server config file to the expected location inside the container - DockerUtils.execMultiple(container, [ - cmdsAndPaths['mkdirCommand'] + [cmdsAndPaths['configDir']], - cmdsAndPaths['copyCommand'] + [cmdsAndPaths['bindMount'] + 'server.conf', cmdsAndPaths['configDir']] - ]) - - # Start `conan_server` - serverOutput = DockerUtils.exec(container, ['conan_server'], stream = True) - - # Progress output - print('Uploading packages to the server...') - - # Upload all of the packages in the container's local cache to the server - DockerUtils.execMultiple(container, [ - ['conan', 'remote', 'add', 'localhost', 'http://127.0.0.1:9300'], - ['conan', 'user', 'user', '-r', 'localhost', '-p', 'password'], - ['conan', 'upload', '*/4.*', '--all', '--confirm', '-r=localhost'] - ]) - - # Configure the server as a temporary remote on the host system - SubprocessUtils.run(['conan', 'remote', 'add', REMOTE_NAME, 'http://{}:9300'.format(externalAddress)]) - SubprocessUtils.run(['conan', 'user', 'user', '-r', REMOTE_NAME, '-p', 'password']) - - # Retrieve the list of packages that were uploaded to the server - packages = SubprocessUtils.extractLines(SubprocessUtils.capture(['conan', 'search', '-r', REMOTE_NAME, '*']).stdout) - packages = [package for package in packages if '/' in package and '@' in package] - - # Download each package in turn - for package in packages: - print('Downloading package {} to host system local cache...'.format(package)) - SubprocessUtils.run(['conan', 'download', '-r', REMOTE_NAME, package]) - - # Once we reach this point, everything has worked and we don't need to output any logs - serverOutput = None - - finally: - - # Stop the container, irrespective of whether or not the export succeeded - print('Stopping conan_server...') - container.stop() - - # If something went wrong then output the logs from `conan_server` to assist in diagnosing the failure - if serverOutput is not None: - print('Log output from conan_server:') - for chunk in serverOutput: - logger.error(chunk.decode('utf-8')) - - # Remove the temporary remote if it was created successfully - SubprocessUtils.run(['conan', 'remote', 'remove', REMOTE_NAME], check = False) + + # Create our logger to generate coloured output on stderr + logger = Logger() + + # Verify that the destination is "cache" + if destination.lower() != "cache": + logger.error('Error: the only supported package export destination is "cache".') + sys.exit(1) + + # Verify that Conan is installed on the host + try: + SubprocessUtils.run(["conan", "--version"]) + except: + logger.error( + "Error: Conan must be installed on the host system to export packages." + ) + sys.exit(1) + + # Determine if the container image is a Windows image or a Linux image + imageOS = DockerUtils.listImages(image)[0].attrs["Os"] + + # Use the appropriate commands and paths for the container platform + cmdsAndPaths = { + "linux": { + "rootCommand": ["bash", "-c", "sleep infinity"], + "mkdirCommand": ["mkdir"], + "copyCommand": ["cp", "-f"], + "dataDir": "/home/ue4/.conan_server/data", + "configDir": "/home/ue4/.conan_server/", + "bindMount": "/hostdir/", + }, + "windows": { + "rootCommand": ["timeout", "/t", "99999", "/nobreak"], + "mkdirCommand": ["cmd", "/S", "/C", "mkdir"], + "copyCommand": ["xcopy", "/y"], + "dataDir": "C:\\Users\\ContainerAdministrator\\.conan_server\\data", + "configDir": "C:\\Users\\ContainerAdministrator\\.conan_server\\", + "bindMount": "C:\\hostdir\\", + }, + }[imageOS] + + # Create an auto-deleting temporary directory to hold our server config file + with tempfile.TemporaryDirectory() as tempDir: + + # Progress output + print("Starting conan_server in a container...") + + # Start a container from which we will export packages, bind-mounting our temp directory + container = DockerUtils.start( + image, + cmdsAndPaths["rootCommand"], + ports={"9300/tcp": 9300}, + mounts=[docker.types.Mount(cmdsAndPaths["bindMount"], tempDir, "bind")], + stdin_open=imageOS == "windows", + tty=imageOS == "windows", + remove=True, + ) + + # Reload the container attributes from the Docker daemon to ensure the networking fields are populated + container.reload() + + # Under Linux we can simply access the container from the host over the loopback address, but this doesn't work under Windows + # (See ) + externalAddress = ( + "127.0.0.1" + if imageOS == "linux" + else container.attrs["NetworkSettings"]["Networks"]["nat"]["IPAddress"] + ) + + # Generate our server config file in the temp directory + FilesystemUtils.writeFile( + os.path.join(tempDir, "server.conf"), + CONAN_SERVER_CONFIG.format(externalAddress, cmdsAndPaths["dataDir"]), + ) + + # Keep track of the `conan_server` log output so we can display it in case of an error + serverOutput = None + + try: + + # Copy the server config file to the expected location inside the container + DockerUtils.execMultiple( + container, + [ + cmdsAndPaths["mkdirCommand"] + [cmdsAndPaths["configDir"]], + cmdsAndPaths["copyCommand"] + + [ + cmdsAndPaths["bindMount"] + "server.conf", + cmdsAndPaths["configDir"], + ], + ], + ) + + # Start `conan_server` + serverOutput = DockerUtils.exec(container, ["conan_server"], stream=True) + + # Progress output + print("Uploading packages to the server...") + + # Upload all of the packages in the container's local cache to the server + DockerUtils.execMultiple( + container, + [ + ["conan", "remote", "add", "localhost", "http://127.0.0.1:9300"], + ["conan", "user", "user", "-r", "localhost", "-p", "password"], + ["conan", "upload", "*/4.*", "--all", "--confirm", "-r=localhost"], + ], + ) + + # Configure the server as a temporary remote on the host system + SubprocessUtils.run( + [ + "conan", + "remote", + "add", + REMOTE_NAME, + "http://{}:9300".format(externalAddress), + ] + ) + SubprocessUtils.run( + ["conan", "user", "user", "-r", REMOTE_NAME, "-p", "password"] + ) + + # Retrieve the list of packages that were uploaded to the server + packages = SubprocessUtils.extractLines( + SubprocessUtils.capture( + ["conan", "search", "-r", REMOTE_NAME, "*"] + ).stdout + ) + packages = [ + package for package in packages if "/" in package and "@" in package + ] + + # Download each package in turn + for package in packages: + print( + "Downloading package {} to host system local cache...".format( + package + ) + ) + SubprocessUtils.run(["conan", "download", "-r", REMOTE_NAME, package]) + + # Once we reach this point, everything has worked and we don't need to output any logs + serverOutput = None + + finally: + + # Stop the container, irrespective of whether or not the export succeeded + print("Stopping conan_server...") + container.stop() + + # If something went wrong then output the logs from `conan_server` to assist in diagnosing the failure + if serverOutput is not None: + print("Log output from conan_server:") + for chunk in serverOutput: + logger.error(chunk.decode("utf-8")) + + # Remove the temporary remote if it was created successfully + SubprocessUtils.run(["conan", "remote", "remove", REMOTE_NAME], check=False) diff --git a/ue4docker/info.py b/ue4docker/info.py index 03f95df3..ceb57459 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -2,67 +2,98 @@ from .version import __version__ from .infrastructure import * + def _osName(dockerInfo): - if platform.system() == 'Windows': - return WindowsUtils.systemString() - elif platform.system() == 'Darwin': - return DarwinUtils.systemString() - else: - return 'Linux ({}, {})'.format(dockerInfo['OperatingSystem'], dockerInfo['KernelVersion']) + if platform.system() == "Windows": + return WindowsUtils.systemString() + elif platform.system() == "Darwin": + return DarwinUtils.systemString() + else: + return "Linux ({}, {})".format( + dockerInfo["OperatingSystem"], dockerInfo["KernelVersion"] + ) + def _formatSize(size): - return humanfriendly.format_size(size, binary=True) + return humanfriendly.format_size(size, binary=True) + def info(): - - # Verify that Docker is installed - if DockerUtils.installed() == False: - print('Error: could not detect Docker version. Please ensure Docker is installed.', file=sys.stderr) - sys.exit(1) - - # Gather our information about the Docker daemon - dockerInfo = DockerUtils.info() - nvidiaDocker = platform.system() == 'Linux' and 'nvidia' in dockerInfo['Runtimes'] - maxSize = DockerUtils.maxsize() - rootDir = dockerInfo['DockerRootDir'] - - # If we are communicating with a Linux Docker daemon under Windows or macOS then we can't query the available disk space - canQueryDisk = dockerInfo['OSType'].lower() == platform.system().lower() - - # Gather our information about the host system - diskSpace = _formatSize(shutil.disk_usage(rootDir).free) if canQueryDisk == True else 'Unknown (typically means the Docker daemon is running in a Moby VM, e.g. Docker Desktop)' - memPhysical = psutil.virtual_memory().total - memVirtual = psutil.swap_memory().total - cpuPhysical = psutil.cpu_count(False) - cpuLogical = psutil.cpu_count() - - # Attempt to query PyPI to determine the latest version of ue4-docker - # (We ignore any errors here to ensure the `ue4-docker info` command still works without network access) - try: - latestVersion = GlobalConfiguration.getLatestVersion() - except: - latestVersion = None - - # Prepare our report items - items = [ - ('ue4-docker version', '{}{}'.format(__version__, '' if latestVersion is None else ' (latest available version is {})'.format(latestVersion))), - ('Operating system', _osName(dockerInfo)), - ('Docker daemon version', dockerInfo['ServerVersion']), - ('NVIDIA Docker supported', 'Yes' if nvidiaDocker == True else 'No'), - ('Maximum image size', '{:.0f}GB'.format(maxSize) if maxSize != -1 else 'No limit detected'), - ('Available disk space', diskSpace), - ('Total system memory', '{} physical, {} virtual'.format(_formatSize(memPhysical), _formatSize(memVirtual))), - ('Number of processors', '{} physical, {} logical'.format(cpuPhysical, cpuLogical)) - ] - - # Determine the longest item name so we can format our list in nice columns - longestName = max([len(i[0]) for i in items]) - minSpaces = 4 - - # Print our report - for item in items: - print('{}:{}{}'.format( - item[0], - ' ' * ((longestName + minSpaces) - len(item[0])), - item[1] - )) + + # Verify that Docker is installed + if DockerUtils.installed() == False: + print( + "Error: could not detect Docker version. Please ensure Docker is installed.", + file=sys.stderr, + ) + sys.exit(1) + + # Gather our information about the Docker daemon + dockerInfo = DockerUtils.info() + nvidiaDocker = platform.system() == "Linux" and "nvidia" in dockerInfo["Runtimes"] + maxSize = DockerUtils.maxsize() + rootDir = dockerInfo["DockerRootDir"] + + # If we are communicating with a Linux Docker daemon under Windows or macOS then we can't query the available disk space + canQueryDisk = dockerInfo["OSType"].lower() == platform.system().lower() + + # Gather our information about the host system + diskSpace = ( + _formatSize(shutil.disk_usage(rootDir).free) + if canQueryDisk == True + else "Unknown (typically means the Docker daemon is running in a Moby VM, e.g. Docker Desktop)" + ) + memPhysical = psutil.virtual_memory().total + memVirtual = psutil.swap_memory().total + cpuPhysical = psutil.cpu_count(False) + cpuLogical = psutil.cpu_count() + + # Attempt to query PyPI to determine the latest version of ue4-docker + # (We ignore any errors here to ensure the `ue4-docker info` command still works without network access) + try: + latestVersion = GlobalConfiguration.getLatestVersion() + except: + latestVersion = None + + # Prepare our report items + items = [ + ( + "ue4-docker version", + "{}{}".format( + __version__, + "" + if latestVersion is None + else " (latest available version is {})".format(latestVersion), + ), + ), + ("Operating system", _osName(dockerInfo)), + ("Docker daemon version", dockerInfo["ServerVersion"]), + ("NVIDIA Docker supported", "Yes" if nvidiaDocker == True else "No"), + ( + "Maximum image size", + "{:.0f}GB".format(maxSize) if maxSize != -1 else "No limit detected", + ), + ("Available disk space", diskSpace), + ( + "Total system memory", + "{} physical, {} virtual".format( + _formatSize(memPhysical), _formatSize(memVirtual) + ), + ), + ( + "Number of processors", + "{} physical, {} logical".format(cpuPhysical, cpuLogical), + ), + ] + + # Determine the longest item name so we can format our list in nice columns + longestName = max([len(i[0]) for i in items]) + minSpaces = 4 + + # Print our report + for item in items: + print( + "{}:{}{}".format( + item[0], " " * ((longestName + minSpaces) - len(item[0])), item[1] + ) + ) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index da3eda1b..1590512a 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -5,348 +5,538 @@ from pkg_resources import parse_version # Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) +semver = PackageUtils.importFile( + "semver", os.path.join(PackageUtils.getPackageLocation("semver"), "semver.py") +) # The default Unreal Engine git repository -DEFAULT_GIT_REPO = 'https://github.com/EpicGames/UnrealEngine.git' +DEFAULT_GIT_REPO = "https://github.com/EpicGames/UnrealEngine.git" # The base images for Linux containers LINUX_BASE_IMAGES = { - 'opengl': 'nvidia/opengl:1.0-glvnd-devel-ubuntu18.04', - 'cudagl': { - '9.2': 'nvidia/cudagl:9.2-devel-ubuntu18.04', - '10.0': 'nvidia/cudagl:10.0-devel-ubuntu18.04', - '10.1': 'nvidia/cudagl:10.1-devel-ubuntu18.04', - '10.2': 'nvidia/cudagl:10.2-devel-ubuntu18.04' - } + "opengl": "nvidia/opengl:1.0-glvnd-devel-ubuntu18.04", + "cudagl": { + "9.2": "nvidia/cudagl:9.2-devel-ubuntu18.04", + "10.0": "nvidia/cudagl:10.0-devel-ubuntu18.04", + "10.1": "nvidia/cudagl:10.1-devel-ubuntu18.04", + "10.2": "nvidia/cudagl:10.2-devel-ubuntu18.04", + }, } # The default CUDA version to use when `--cuda` is specified without a value -DEFAULT_CUDA_VERSION = '9.2' +DEFAULT_CUDA_VERSION = "9.2" # The default memory limit (in GB) under Windows DEFAULT_MEMORY_LIMIT = 10.0 + class VisualStudio(object): - VS2017 = '2017' - VS2019 = '2019' - - BuildNumbers = { - VS2017 : '15', - VS2019 : '16', - } - - MinSupportedUnreal = { - # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 - # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa - # - # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ - # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 - VS2019 : semver.VersionInfo(4, 25) - } + VS2017 = "2017" + VS2019 = "2019" + + BuildNumbers = { + VS2017: "15", + VS2019: "16", + } + + MinSupportedUnreal = { + # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 + # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa + # + # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ + # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 + VS2019: semver.VersionInfo(4, 25) + } + class ExcludedComponent(object): - ''' - The different components that we support excluding from the built images - ''' - - # Engine Derived Data Cache (DDC) - DDC = 'ddc' - - # Engine debug symbols - Debug = 'debug' - - # Template projects and samples - Templates = 'templates' - - - @staticmethod - def description(component): - ''' - Returns a human-readable description of the specified component - ''' - return { - - ExcludedComponent.DDC: 'Derived Data Cache (DDC)', - ExcludedComponent.Debug: 'Debug symbols', - ExcludedComponent.Templates: 'Template projects and samples' - - }.get(component, '[Unknown component]') + """ + The different components that we support excluding from the built images + """ + + # Engine Derived Data Cache (DDC) + DDC = "ddc" + + # Engine debug symbols + Debug = "debug" + + # Template projects and samples + Templates = "templates" + + @staticmethod + def description(component): + """ + Returns a human-readable description of the specified component + """ + return { + ExcludedComponent.DDC: "Derived Data Cache (DDC)", + ExcludedComponent.Debug: "Debug symbols", + ExcludedComponent.Templates: "Template projects and samples", + }.get(component, "[Unknown component]") class BuildConfiguration(object): - - @staticmethod - def addArguments(parser): - ''' - Registers our supported command-line arguments with the supplied argument parser - ''' - parser.add_argument('release', help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch') - parser.add_argument('--linux', action='store_true', help='Build Linux container images under Windows') - parser.add_argument('--rebuild', action='store_true', help='Rebuild images even if they already exist') - parser.add_argument('--dry-run', action='store_true', help='Print `docker build` commands instead of running them') - parser.add_argument('--no-engine', action='store_true', help='Don\'t build the ue4-engine image') - parser.add_argument('--no-minimal', action='store_true', help='Don\'t build the ue4-minimal image') - parser.add_argument('--no-full', action='store_true', help='Don\'t build the ue4-full image') - parser.add_argument('--no-cache', action='store_true', help='Disable Docker build cache') - parser.add_argument('--random-memory', action='store_true', help='Use a random memory limit for Windows containers') - parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.DDC, ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') - parser.add_argument('--opt', action='append', default=[], help='Set an advanced configuration option (can be specified multiple times to specify multiple options)') - parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') - parser.add_argument('--visual-studio', default=VisualStudio.VS2017, choices=VisualStudio.BuildNumbers.keys(), help='Specify Visual Studio Build Tools version to use for Windows containers') - parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') - parser.add_argument('-password', default=None, help='Specify the password or access token to use when cloning the git repository') - parser.add_argument('-repo', default=None, help='Set the custom git repository to clone when "custom" is specified as the release value') - parser.add_argument('-branch', default=None, help='Set the custom branch/tag to clone when "custom" is specified as the release value') - parser.add_argument('-isolation', default=None, help='Set the isolation mode to use for Windows containers (process or hyperv)') - parser.add_argument('-basetag', default=None, help='Windows Server Core base image tag to use for Windows containers (default is the host OS version)') - parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images') - parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') - parser.add_argument('-ue4cli', default=None, help='Override the default version of ue4cli installed in the ue4-full image') - parser.add_argument('-conan-ue4cli', default=None, help='Override the default version of conan-ue4cli installed in the ue4-full image') - parser.add_argument('-layout', default=None, help='Copy generated Dockerfiles to the specified directory and don\'t build the images') - parser.add_argument('--combine', action='store_true', help='Combine generated Dockerfiles into a single multi-stage build Dockerfile') - parser.add_argument('--monitor', action='store_true', help='Monitor resource usage during builds (useful for debugging)') - parser.add_argument('-interval', type=float, default=20.0, help='Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)') - parser.add_argument('--ignore-blacklist', action='store_true', help='Run builds even on blacklisted versions of Windows (advanced use only)') - parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose output during builds (useful for debugging)') - - def __init__(self, parser, argv): - ''' - Creates a new build configuration based on the supplied arguments object - ''' - - # If the user has specified `--cuda` without a version value, treat the value as an empty string - argv = [arg + '=' if arg == '--cuda' else arg for arg in argv] - - # Parse the supplied command-line arguments - self.args = parser.parse_args(argv) - - # Determine if we are building a custom version of UE4 rather than an official release - self.args.release = self.args.release.lower() - if self.args.release == 'custom' or self.args.release.startswith('custom:'): - - # Both a custom repository and a custom branch/tag must be specified - if self.args.repo is None or self.args.branch is None: - raise RuntimeError('both a repository and branch/tag must be specified when building a custom version of the Engine') - - # Use the specified repository and branch/tag - customName = self.args.release.split(':', 2)[1].strip() if ':' in self.args.release else '' - self.release = customName if len(customName) > 0 else 'custom' - self.repository = self.args.repo - self.branch = self.args.branch - self.custom = True - - else: - - # Validate the specified version string - try: - ue4Version = semver.parse(self.args.release) - if ue4Version['major'] != 4 or ue4Version['prerelease'] != None: - raise Exception() - self.release = semver.format_version(ue4Version['major'], ue4Version['minor'], ue4Version['patch']) - except: - raise RuntimeError('invalid UE4 release number "{}", full semver format required (e.g. "4.20.0")'.format(self.args.release)) - - # Use the default repository and the release tag for the specified version - self.repository = DEFAULT_GIT_REPO - self.branch = '{}-release'.format(self.release) - self.custom = False - - # Store our common configuration settings - self.containerPlatform = 'windows' if platform.system() == 'Windows' and self.args.linux == False else 'linux' - self.dryRun = self.args.dry_run - self.rebuild = self.args.rebuild - self.noEngine = self.args.no_engine - self.noMinimal = self.args.no_minimal - self.noFull = self.args.no_full - self.suffix = self.args.suffix - self.platformArgs = ['--no-cache'] if self.args.no_cache == True else [] - self.excludedComponents = set(self.args.exclude) - self.baseImage = None - self.prereqsTag = None - self.ignoreBlacklist = self.args.ignore_blacklist - self.verbose = self.args.verbose - self.layoutDir = self.args.layout - self.combine = self.args.combine - - # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them - self.ue4cliVersion = self._processPackageVersion('ue4cli', self.args.ue4cli) - self.conanUe4cliVersion = self._processPackageVersion('conan-ue4cli', self.args.conan_ue4cli) - - # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) - self.opts = {} - for o in self.args.opt: - if '=' in o: - key, value = o.split('=', 1) - self.opts[key.replace('-', '_')] = self._processTemplateValue(value) - else: - self.opts[o.replace('-', '_')] = True - - # If we are generating Dockerfiles then generate them for all images that have not been explicitly excluded - if self.layoutDir is not None: - self.rebuild = True - - # If we are generating Dockerfiles and combining them then set the corresponding Jinja context value - if self.layoutDir is not None and self.combine == True: - self.opts['combine'] = True - - # If the user requested an option that is only compatible with generated Dockerfiles then ensure `-layout` was specified - if self.layoutDir is None and self.opts.get('source_mode', 'git') != 'git': - raise RuntimeError('the `-layout` flag must be used when specifying a non-default value for the `source_mode` option') - if self.layoutDir is None and self.combine == True: - raise RuntimeError('the `-layout` flag must be used when specifying the `--combine` flag') - - # Verify that the value for `source_mode` is valid if specified - validSourceModes = ['git', 'copy'] - if self.opts.get('source_mode', 'git') not in validSourceModes: - raise RuntimeError('invalid value specified for the `source_mode` option, valid values are {}'.format(validSourceModes)) - - # Verify that the value for `credential_mode` is valid if specified - validCredentialModes = ['endpoint', 'secrets'] if self.containerPlatform == 'linux' else ['endpoint'] - if self.opts.get('credential_mode', 'endpoint') not in validCredentialModes: - raise RuntimeError('invalid value specified for the `credential_mode` option, valid values are {} when building {} containers'.format(validCredentialModes, self.containerPlatform.title())) - - # Generate Jinja context values for keeping or excluding components - self.opts['excluded_components'] = { - 'ddc': ExcludedComponent.DDC in self.excludedComponents, - 'debug': ExcludedComponent.Debug in self.excludedComponents, - 'templates': ExcludedComponent.Templates in self.excludedComponents - } - - # If we're building Windows containers, generate our Windows-specific configuration settings - if self.containerPlatform == 'windows': - self._generateWindowsConfig() - - # If we're building Linux containers, generate our Linux-specific configuration settings - if self.containerPlatform == 'linux': - self._generateLinuxConfig() - - # If the user-specified suffix passed validation, prefix it with a dash - self.suffix = '-{}'.format(self.suffix) if self.suffix != '' else '' - - def describeExcludedComponents(self): - ''' - Returns a list of strings describing the components that will be excluded (if any.) - ''' - return sorted([ExcludedComponent.description(component) for component in self.excludedComponents]) - - def _generateWindowsConfig(self): - self.visualStudio = self.args.visual_studio - - if not self.custom: - # Check whether specified Unreal Engine release is compatible with specified Visual Studio - vsMinSupportedUnreal = VisualStudio.MinSupportedUnreal.get(self.visualStudio, None) - if vsMinSupportedUnreal is not None and semver.VersionInfo.parse(self.release) < vsMinSupportedUnreal: - raise RuntimeError('specified version of Unreal Engine cannot be built with Visual Studio {}, oldest supported is {}'.format(self.visualStudio, vsMinSupportedUnreal)) - - self.visualStudioBuildNumber = VisualStudio.BuildNumbers[self.visualStudio] - # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac - # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. - # Otherwise, UAT errors out with "Unknown argument: VS2019". - if self.visualStudio != VisualStudio.VS2017: - self.opts['buildgraph_args'] = self.opts.get('buildgraph_args', '') + f' -set:VS{self.visualStudio}=true' - - # Determine base tag for the Windows release of the host system - self.hostBasetag = WindowsUtils.getHostBaseTag() - - # Store the tag for the base Windows Server Core image - self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag - - if self.basetag is None: - raise RuntimeError('unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag') - - self.baseImage = 'mcr.microsoft.com/windows/servercore:' + self.basetag - self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) - self.prereqsTag = self.basetag + '-vs' + self.visualStudio - - # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect - if self.args.isolation is not None: - self.isolation = self.args.isolation - else: - - # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode - differentKernels = self.basetag != self.hostBasetag - dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') - if not differentKernels and dockerSupportsProcess: - self.isolation = 'process' - else: - self.isolation = DockerUtils.info()['Isolation'] - - # Set the isolation mode Docker flag - self.platformArgs.append('--isolation=' + self.isolation) - - # If the user has explicitly specified a memory limit then use it, otherwise auto-detect - self.memLimit = None - if self.args.m is not None: - try: - self.memLimit = humanfriendly.parse_size(self.args.m) / (1000*1000*1000) - except: - raise RuntimeError('invalid memory limit "{}"'.format(self.args.m)) - else: - - # Only specify a memory limit when using Hyper-V isolation mode, in order to override the 1GB default limit - # (Process isolation mode does not impose any memory limits by default) - if self.isolation == 'hyperv': - self.memLimit = DEFAULT_MEMORY_LIMIT if self.args.random_memory == False else random.uniform(DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT + 2.0) - - # Set the memory limit Docker flag - if self.memLimit is not None: - self.platformArgs.extend(['-m', '{:.2f}GB'.format(self.memLimit)]) - - def _generateLinuxConfig(self): - - # Verify that any user-specified tag suffix does not collide with our base tags - if self.suffix.startswith('opengl') or self.suffix.startswith('cudagl'): - raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') - - # Determine if we are building CUDA-enabled container images - self.cuda = None - if self.args.cuda is not None: - - # Verify that the specified CUDA version is valid - self.cuda = self.args.cuda if self.args.cuda != '' else DEFAULT_CUDA_VERSION - if self.cuda not in LINUX_BASE_IMAGES['cudagl']: - raise RuntimeError('unsupported CUDA version "{}", supported versions are: {}'.format( - self.cuda, - ', '.join([v for v in LINUX_BASE_IMAGES['cudagl']]) - )) - - # Use the appropriate base image for the specified CUDA version - self.baseImage = LINUX_BASE_IMAGES['cudagl'][self.cuda] - self.prereqsTag = 'cudagl{}'.format(self.cuda) - else: - self.baseImage = LINUX_BASE_IMAGES['opengl'] - self.prereqsTag = 'opengl' - - def _processPackageVersion(self, package, version): - - # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified - # (e.g. package==X.X.X, package>=X.X.X, git+https://url/for/package/repo.git, etc.) - if version is None or '/' in version or version.lower().startswith(package): - return version - - # If a version specifier (e.g. ==X.X.X, >=X.X.X, etc.) was specified, prefix it with the package name - if '=' in version: - return package + version - - # If a raw version number was specified, prefix the package name and a strict equality specifier - return '{}=={}'.format(package, version) - - def _processTemplateValue(self, value): - - # If the value is a boolean (either raw or represented by zero or one) then parse it - if value.lower() in ['true', '1']: - return True - elif value.lower() in ['false', '0']: - return False - - # If the value is a JSON object or array then attempt to parse it - if (value.startswith('{') and value.endswith('}')) or (value.startswith('[') and value.endswith(']')): - try: - return json.loads(value) - except: - print('Warning: could not parse option value "{}" as JSON, treating value as a string'.format(value)) - - # Treat all other values as strings - return value + @staticmethod + def addArguments(parser): + """ + Registers our supported command-line arguments with the supplied argument parser + """ + parser.add_argument( + "release", + help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch', + ) + parser.add_argument( + "--linux", + action="store_true", + help="Build Linux container images under Windows", + ) + parser.add_argument( + "--rebuild", + action="store_true", + help="Rebuild images even if they already exist", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print `docker build` commands instead of running them", + ) + parser.add_argument( + "--no-engine", action="store_true", help="Don't build the ue4-engine image" + ) + parser.add_argument( + "--no-minimal", + action="store_true", + help="Don't build the ue4-minimal image", + ) + parser.add_argument( + "--no-full", action="store_true", help="Don't build the ue4-full image" + ) + parser.add_argument( + "--no-cache", action="store_true", help="Disable Docker build cache" + ) + parser.add_argument( + "--random-memory", + action="store_true", + help="Use a random memory limit for Windows containers", + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + choices=[ + ExcludedComponent.DDC, + ExcludedComponent.Debug, + ExcludedComponent.Templates, + ], + help="Exclude the specified component (can be specified multiple times to exclude multiple components)", + ) + parser.add_argument( + "--opt", + action="append", + default=[], + help="Set an advanced configuration option (can be specified multiple times to specify multiple options)", + ) + parser.add_argument( + "--cuda", + default=None, + metavar="VERSION", + help="Add CUDA support as well as OpenGL support when building Linux containers", + ) + parser.add_argument( + "--visual-studio", + default=VisualStudio.VS2017, + choices=VisualStudio.BuildNumbers.keys(), + help="Specify Visual Studio Build Tools version to use for Windows containers", + ) + parser.add_argument( + "-username", + default=None, + help="Specify the username to use when cloning the git repository", + ) + parser.add_argument( + "-password", + default=None, + help="Specify the password or access token to use when cloning the git repository", + ) + parser.add_argument( + "-repo", + default=None, + help='Set the custom git repository to clone when "custom" is specified as the release value', + ) + parser.add_argument( + "-branch", + default=None, + help='Set the custom branch/tag to clone when "custom" is specified as the release value', + ) + parser.add_argument( + "-isolation", + default=None, + help="Set the isolation mode to use for Windows containers (process or hyperv)", + ) + parser.add_argument( + "-basetag", + default=None, + help="Windows Server Core base image tag to use for Windows containers (default is the host OS version)", + ) + parser.add_argument( + "-suffix", default="", help="Add a suffix to the tags of the built images" + ) + parser.add_argument( + "-m", + default=None, + help="Override the default memory limit under Windows (also overrides --random-memory)", + ) + parser.add_argument( + "-ue4cli", + default=None, + help="Override the default version of ue4cli installed in the ue4-full image", + ) + parser.add_argument( + "-conan-ue4cli", + default=None, + help="Override the default version of conan-ue4cli installed in the ue4-full image", + ) + parser.add_argument( + "-layout", + default=None, + help="Copy generated Dockerfiles to the specified directory and don't build the images", + ) + parser.add_argument( + "--combine", + action="store_true", + help="Combine generated Dockerfiles into a single multi-stage build Dockerfile", + ) + parser.add_argument( + "--monitor", + action="store_true", + help="Monitor resource usage during builds (useful for debugging)", + ) + parser.add_argument( + "-interval", + type=float, + default=20.0, + help="Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)", + ) + parser.add_argument( + "--ignore-blacklist", + action="store_true", + help="Run builds even on blacklisted versions of Windows (advanced use only)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output during builds (useful for debugging)", + ) + + def __init__(self, parser, argv): + """ + Creates a new build configuration based on the supplied arguments object + """ + + # If the user has specified `--cuda` without a version value, treat the value as an empty string + argv = [arg + "=" if arg == "--cuda" else arg for arg in argv] + + # Parse the supplied command-line arguments + self.args = parser.parse_args(argv) + + # Determine if we are building a custom version of UE4 rather than an official release + self.args.release = self.args.release.lower() + if self.args.release == "custom" or self.args.release.startswith("custom:"): + + # Both a custom repository and a custom branch/tag must be specified + if self.args.repo is None or self.args.branch is None: + raise RuntimeError( + "both a repository and branch/tag must be specified when building a custom version of the Engine" + ) + + # Use the specified repository and branch/tag + customName = ( + self.args.release.split(":", 2)[1].strip() + if ":" in self.args.release + else "" + ) + self.release = customName if len(customName) > 0 else "custom" + self.repository = self.args.repo + self.branch = self.args.branch + self.custom = True + + else: + + # Validate the specified version string + try: + ue4Version = semver.parse(self.args.release) + if ue4Version["major"] != 4 or ue4Version["prerelease"] != None: + raise Exception() + self.release = semver.format_version( + ue4Version["major"], ue4Version["minor"], ue4Version["patch"] + ) + except: + raise RuntimeError( + 'invalid UE4 release number "{}", full semver format required (e.g. "4.20.0")'.format( + self.args.release + ) + ) + + # Use the default repository and the release tag for the specified version + self.repository = DEFAULT_GIT_REPO + self.branch = "{}-release".format(self.release) + self.custom = False + + # Store our common configuration settings + self.containerPlatform = ( + "windows" + if platform.system() == "Windows" and self.args.linux == False + else "linux" + ) + self.dryRun = self.args.dry_run + self.rebuild = self.args.rebuild + self.noEngine = self.args.no_engine + self.noMinimal = self.args.no_minimal + self.noFull = self.args.no_full + self.suffix = self.args.suffix + self.platformArgs = ["--no-cache"] if self.args.no_cache == True else [] + self.excludedComponents = set(self.args.exclude) + self.baseImage = None + self.prereqsTag = None + self.ignoreBlacklist = self.args.ignore_blacklist + self.verbose = self.args.verbose + self.layoutDir = self.args.layout + self.combine = self.args.combine + + # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them + self.ue4cliVersion = self._processPackageVersion("ue4cli", self.args.ue4cli) + self.conanUe4cliVersion = self._processPackageVersion( + "conan-ue4cli", self.args.conan_ue4cli + ) + + # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) + self.opts = {} + for o in self.args.opt: + if "=" in o: + key, value = o.split("=", 1) + self.opts[key.replace("-", "_")] = self._processTemplateValue(value) + else: + self.opts[o.replace("-", "_")] = True + + # If we are generating Dockerfiles then generate them for all images that have not been explicitly excluded + if self.layoutDir is not None: + self.rebuild = True + + # If we are generating Dockerfiles and combining them then set the corresponding Jinja context value + if self.layoutDir is not None and self.combine == True: + self.opts["combine"] = True + + # If the user requested an option that is only compatible with generated Dockerfiles then ensure `-layout` was specified + if self.layoutDir is None and self.opts.get("source_mode", "git") != "git": + raise RuntimeError( + "the `-layout` flag must be used when specifying a non-default value for the `source_mode` option" + ) + if self.layoutDir is None and self.combine == True: + raise RuntimeError( + "the `-layout` flag must be used when specifying the `--combine` flag" + ) + + # Verify that the value for `source_mode` is valid if specified + validSourceModes = ["git", "copy"] + if self.opts.get("source_mode", "git") not in validSourceModes: + raise RuntimeError( + "invalid value specified for the `source_mode` option, valid values are {}".format( + validSourceModes + ) + ) + + # Verify that the value for `credential_mode` is valid if specified + validCredentialModes = ( + ["endpoint", "secrets"] + if self.containerPlatform == "linux" + else ["endpoint"] + ) + if self.opts.get("credential_mode", "endpoint") not in validCredentialModes: + raise RuntimeError( + "invalid value specified for the `credential_mode` option, valid values are {} when building {} containers".format( + validCredentialModes, self.containerPlatform.title() + ) + ) + + # Generate Jinja context values for keeping or excluding components + self.opts["excluded_components"] = { + "ddc": ExcludedComponent.DDC in self.excludedComponents, + "debug": ExcludedComponent.Debug in self.excludedComponents, + "templates": ExcludedComponent.Templates in self.excludedComponents, + } + + # If we're building Windows containers, generate our Windows-specific configuration settings + if self.containerPlatform == "windows": + self._generateWindowsConfig() + + # If we're building Linux containers, generate our Linux-specific configuration settings + if self.containerPlatform == "linux": + self._generateLinuxConfig() + + # If the user-specified suffix passed validation, prefix it with a dash + self.suffix = "-{}".format(self.suffix) if self.suffix != "" else "" + + def describeExcludedComponents(self): + """ + Returns a list of strings describing the components that will be excluded (if any.) + """ + return sorted( + [ + ExcludedComponent.description(component) + for component in self.excludedComponents + ] + ) + + def _generateWindowsConfig(self): + self.visualStudio = self.args.visual_studio + + if not self.custom: + # Check whether specified Unreal Engine release is compatible with specified Visual Studio + vsMinSupportedUnreal = VisualStudio.MinSupportedUnreal.get( + self.visualStudio, None + ) + if ( + vsMinSupportedUnreal is not None + and semver.VersionInfo.parse(self.release) < vsMinSupportedUnreal + ): + raise RuntimeError( + "specified version of Unreal Engine cannot be built with Visual Studio {}, oldest supported is {}".format( + self.visualStudio, vsMinSupportedUnreal + ) + ) + + self.visualStudioBuildNumber = VisualStudio.BuildNumbers[self.visualStudio] + # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac + # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. + # Otherwise, UAT errors out with "Unknown argument: VS2019". + if self.visualStudio != VisualStudio.VS2017: + self.opts["buildgraph_args"] = ( + self.opts.get("buildgraph_args", "") + + f" -set:VS{self.visualStudio}=true" + ) + + # Determine base tag for the Windows release of the host system + self.hostBasetag = WindowsUtils.getHostBaseTag() + + # Store the tag for the base Windows Server Core image + self.basetag = ( + self.args.basetag if self.args.basetag is not None else self.hostBasetag + ) + + if self.basetag is None: + raise RuntimeError( + "unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag" + ) + + self.baseImage = "mcr.microsoft.com/windows/servercore:" + self.basetag + self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) + self.prereqsTag = self.basetag + "-vs" + self.visualStudio + + # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect + if self.args.isolation is not None: + self.isolation = self.args.isolation + else: + + # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode + differentKernels = self.basetag != self.hostBasetag + dockerSupportsProcess = parse_version( + DockerUtils.version()["Version"] + ) >= parse_version("18.09.0") + if not differentKernels and dockerSupportsProcess: + self.isolation = "process" + else: + self.isolation = DockerUtils.info()["Isolation"] + + # Set the isolation mode Docker flag + self.platformArgs.append("--isolation=" + self.isolation) + + # If the user has explicitly specified a memory limit then use it, otherwise auto-detect + self.memLimit = None + if self.args.m is not None: + try: + self.memLimit = humanfriendly.parse_size(self.args.m) / ( + 1000 * 1000 * 1000 + ) + except: + raise RuntimeError('invalid memory limit "{}"'.format(self.args.m)) + else: + + # Only specify a memory limit when using Hyper-V isolation mode, in order to override the 1GB default limit + # (Process isolation mode does not impose any memory limits by default) + if self.isolation == "hyperv": + self.memLimit = ( + DEFAULT_MEMORY_LIMIT + if self.args.random_memory == False + else random.uniform( + DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT + 2.0 + ) + ) + + # Set the memory limit Docker flag + if self.memLimit is not None: + self.platformArgs.extend(["-m", "{:.2f}GB".format(self.memLimit)]) + + def _generateLinuxConfig(self): + + # Verify that any user-specified tag suffix does not collide with our base tags + if self.suffix.startswith("opengl") or self.suffix.startswith("cudagl"): + raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') + + # Determine if we are building CUDA-enabled container images + self.cuda = None + if self.args.cuda is not None: + + # Verify that the specified CUDA version is valid + self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION + if self.cuda not in LINUX_BASE_IMAGES["cudagl"]: + raise RuntimeError( + 'unsupported CUDA version "{}", supported versions are: {}'.format( + self.cuda, ", ".join([v for v in LINUX_BASE_IMAGES["cudagl"]]) + ) + ) + + # Use the appropriate base image for the specified CUDA version + self.baseImage = LINUX_BASE_IMAGES["cudagl"][self.cuda] + self.prereqsTag = "cudagl{}".format(self.cuda) + else: + self.baseImage = LINUX_BASE_IMAGES["opengl"] + self.prereqsTag = "opengl" + + def _processPackageVersion(self, package, version): + + # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified + # (e.g. package==X.X.X, package>=X.X.X, git+https://url/for/package/repo.git, etc.) + if version is None or "/" in version or version.lower().startswith(package): + return version + + # If a version specifier (e.g. ==X.X.X, >=X.X.X, etc.) was specified, prefix it with the package name + if "=" in version: + return package + version + + # If a raw version number was specified, prefix the package name and a strict equality specifier + return "{}=={}".format(package, version) + + def _processTemplateValue(self, value): + + # If the value is a boolean (either raw or represented by zero or one) then parse it + if value.lower() in ["true", "1"]: + return True + elif value.lower() in ["false", "0"]: + return False + + # If the value is a JSON object or array then attempt to parse it + if (value.startswith("{") and value.endswith("}")) or ( + value.startswith("[") and value.endswith("]") + ): + try: + return json.loads(value) + except: + print( + 'Warning: could not parse option value "{}" as JSON, treating value as a string'.format( + value + ) + ) + + # Treat all other values as strings + return value diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/ue4docker/infrastructure/CredentialEndpoint.py index 312ae3a8..87156324 100644 --- a/ue4docker/infrastructure/CredentialEndpoint.py +++ b/ue4docker/infrastructure/CredentialEndpoint.py @@ -2,81 +2,83 @@ from .NetworkUtils import NetworkUtils from flask import Flask, request + class CredentialEndpoint(object): - - def __init__(self, username, password): - ''' - Creates an endpoint manager for the supplied credentials - ''' - - # Make sure neither our username or password are blank, since that can cause `git clone` to hang indefinitely - self.username = username if username is not None and len(username) > 0 else ' ' - self.password = password if password is not None and len(password) > 0 else ' ' - self.endpoint = None - - # Generate a security token to require when requesting credentials - self.token = secrets.token_hex(16) - - def args(self): - ''' - Returns the Docker build arguments for creating containers that require Git credentials - ''' - - # Resolve the IP address for the host system - hostAddress = NetworkUtils.hostIP() - - # Provide the host address and security token to the container - return [ - '--build-arg', 'HOST_ADDRESS_ARG=' + urllib.parse.quote_plus(hostAddress), - '--build-arg', 'HOST_TOKEN_ARG=' + urllib.parse.quote_plus(self.token) - ] - - def start(self): - ''' - Starts the HTTP endpoint as a child process - ''' - - # Create a child process to run the credential endpoint - self.endpoint = multiprocessing.Process( - target = CredentialEndpoint._endpoint, - args=(self.username, self.password, self.token) - ) - - # Spawn the child process and give the endpoint time to start - self.endpoint.start() - time.sleep(2) - - # Verify that the endpoint started correctly - if self.endpoint.is_alive() == False: - raise RuntimeError('failed to start the credential endpoint') - - def stop(self): - ''' - Stops the HTTP endpoint child process - ''' - self.endpoint.terminate() - self.endpoint.join() - - @staticmethod - def _endpoint(username, password, token): - ''' - Implements a HTTP endpoint to provide Git credentials to Docker containers - ''' - server = Flask(__name__) - - # Disable the first-run banner message - os.environ['WERKZEUG_RUN_MAIN'] = 'true' - - # Disable Flask log output (from ) - log = logging.getLogger('werkzeug') - log.setLevel(logging.ERROR) - - @server.route('/', methods=['POST']) - def credentials(): - if 'token' in request.args and request.args['token'] == token: - prompt = request.data.decode('utf-8') - return password if "Password for" in prompt else username - else: - return 'Invalid security token' - - server.run(host='0.0.0.0', port=9876) + def __init__(self, username, password): + """ + Creates an endpoint manager for the supplied credentials + """ + + # Make sure neither our username or password are blank, since that can cause `git clone` to hang indefinitely + self.username = username if username is not None and len(username) > 0 else " " + self.password = password if password is not None and len(password) > 0 else " " + self.endpoint = None + + # Generate a security token to require when requesting credentials + self.token = secrets.token_hex(16) + + def args(self): + """ + Returns the Docker build arguments for creating containers that require Git credentials + """ + + # Resolve the IP address for the host system + hostAddress = NetworkUtils.hostIP() + + # Provide the host address and security token to the container + return [ + "--build-arg", + "HOST_ADDRESS_ARG=" + urllib.parse.quote_plus(hostAddress), + "--build-arg", + "HOST_TOKEN_ARG=" + urllib.parse.quote_plus(self.token), + ] + + def start(self): + """ + Starts the HTTP endpoint as a child process + """ + + # Create a child process to run the credential endpoint + self.endpoint = multiprocessing.Process( + target=CredentialEndpoint._endpoint, + args=(self.username, self.password, self.token), + ) + + # Spawn the child process and give the endpoint time to start + self.endpoint.start() + time.sleep(2) + + # Verify that the endpoint started correctly + if self.endpoint.is_alive() == False: + raise RuntimeError("failed to start the credential endpoint") + + def stop(self): + """ + Stops the HTTP endpoint child process + """ + self.endpoint.terminate() + self.endpoint.join() + + @staticmethod + def _endpoint(username, password, token): + """ + Implements a HTTP endpoint to provide Git credentials to Docker containers + """ + server = Flask(__name__) + + # Disable the first-run banner message + os.environ["WERKZEUG_RUN_MAIN"] = "true" + + # Disable Flask log output (from ) + log = logging.getLogger("werkzeug") + log.setLevel(logging.ERROR) + + @server.route("/", methods=["POST"]) + def credentials(): + if "token" in request.args and request.args["token"] == token: + prompt = request.data.decode("utf-8") + return password if "Password for" in prompt else username + else: + return "Invalid security token" + + server.run(host="0.0.0.0", port=9876) diff --git a/ue4docker/infrastructure/DarwinUtils.py b/ue4docker/infrastructure/DarwinUtils.py index 7c054693..bd9ec61f 100644 --- a/ue4docker/infrastructure/DarwinUtils.py +++ b/ue4docker/infrastructure/DarwinUtils.py @@ -1,38 +1,39 @@ from packaging import version import os, platform + class DarwinUtils(object): - - @staticmethod - def minimumRequiredVersion(): - ''' - Returns the minimum required version of macOS, which is 10.10.3 Yosemite - - (10.10.3 is the minimum required version for Docker for Mac, as per: - ) - ''' - return '10.10.3' - - @staticmethod - def systemString(): - ''' - Generates a human-readable version string for the macOS host system - ''' - return 'macOS {} (Kernel Version {})'.format( - platform.mac_ver()[0], - platform.uname().release - ) - - @staticmethod - def getMacOsVersion(): - ''' - Returns the version number for the macOS host system - ''' - return platform.mac_ver()[0] - - @staticmethod - def isSupportedMacOsVersion(): - ''' - Verifies that the macOS host system meets our minimum version requirements - ''' - return version.parse(DarwinUtils.getMacOsVersion()) >= version.parse(DarwinUtils.minimumRequiredVersion()) + @staticmethod + def minimumRequiredVersion(): + """ + Returns the minimum required version of macOS, which is 10.10.3 Yosemite + + (10.10.3 is the minimum required version for Docker for Mac, as per: + ) + """ + return "10.10.3" + + @staticmethod + def systemString(): + """ + Generates a human-readable version string for the macOS host system + """ + return "macOS {} (Kernel Version {})".format( + platform.mac_ver()[0], platform.uname().release + ) + + @staticmethod + def getMacOsVersion(): + """ + Returns the version number for the macOS host system + """ + return platform.mac_ver()[0] + + @staticmethod + def isSupportedMacOsVersion(): + """ + Verifies that the macOS host system meets our minimum version requirements + """ + return version.parse(DarwinUtils.getMacOsVersion()) >= version.parse( + DarwinUtils.minimumRequiredVersion() + ) diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index b4f798a6..536db54c 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -1,192 +1,218 @@ import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re from .FilesystemUtils import FilesystemUtils + class DockerUtils(object): - - @staticmethod - def installed(): - ''' - Determines if Docker is installed - ''' - try: - return (DockerUtils.version() is not None), None - except Exception as e: - logging.debug(str(e)) - return False, e - - @staticmethod - def version(): - ''' - Retrieves the version information for the Docker daemon - ''' - client = docker.from_env() - return client.version() - - @staticmethod - def info(): - ''' - Retrieves the system information as produced by `docker info` - ''' - client = docker.from_env() - return client.info() - - @staticmethod - def exists(name): - ''' - Determines if the specified image exists - ''' - client = docker.from_env() - try: - image = client.images.get(name) - return True - except: - return False - - @staticmethod - def build(tags, context, args): - ''' - Returns the `docker build` command to build an image - ''' - tagArgs = [['-t', tag] for tag in tags] - return ['docker', 'build'] + list(itertools.chain.from_iterable(tagArgs)) + [context] + args - - @staticmethod - def buildx(tags, context, args, secrets): - ''' - Returns the `docker buildx` command to build an image with the BuildKit backend - ''' - tagArgs = [['-t', tag] for tag in tags] - return ['docker', 'buildx', 'build'] + list(itertools.chain.from_iterable(tagArgs)) + [context] + ['--progress=plain'] + args + list(itertools.chain.from_iterable([['--secret', s] for s in secrets])) - - @staticmethod - def pull(image): - ''' - Returns the `docker pull` command to pull an image from a remote registry - ''' - return ['docker', 'pull', image] - - @staticmethod - def start(image, command, **kwargs): - ''' - Starts a container in a detached state and returns the container handle - ''' - client = docker.from_env() - return client.containers.run(image, command, detach=True, **kwargs) - - @staticmethod - def configFilePath(): - ''' - Returns the path to the Docker daemon configuration file under Windows - ''' - return '{}\\Docker\\config\\daemon.json'.format(os.environ['ProgramData']) - - @staticmethod - def getConfig(): - ''' - Retrieves and parses the Docker daemon configuration file under Windows - ''' - configPath = DockerUtils.configFilePath() - if os.path.exists(configPath) == True: - with open(configPath) as configFile: - return json.load(configFile) - - return {} - - @staticmethod - def setConfig(config): - ''' - Writes new values to the Docker daemon configuration file under Windows - ''' - configPath = DockerUtils.configFilePath() - with open(configPath, 'w') as configFile: - configFile.write(json.dumps(config)) - - @staticmethod - def maxsize(): - ''' - Determines the configured size limit (in GB) for Windows containers - ''' - if platform.system() != 'Windows': - return -1 - - config = DockerUtils.getConfig() - if 'storage-opts' in config: - sizes = [opt.replace('size=', '') for opt in config['storage-opts'] if 'size=' in opt] - if len(sizes) > 0: - return humanfriendly.parse_size(sizes[0]) / 1000000000 - - # The default limit on image size is 20GB - # (https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container-issues) - return 20.0 - - @staticmethod - def listImages(tagFilter = None, filters = {}, all=False): - ''' - Retrieves the details for each image matching the specified filters - ''' - - # Retrieve the list of images matching the specified filters - client = docker.from_env() - images = client.images.list(filters=filters, all=all) - - # Apply our tag filter if one was specified - if tagFilter is not None: - images = [i for i in images if len(i.tags) > 0 and len(fnmatch.filter(i.tags, tagFilter)) > 0] - - return images - - @staticmethod - def exec(container, command, **kwargs): - ''' - Executes a command in a container returned by `DockerUtils.start()` and returns the output - ''' - result, output = container.exec_run(command, **kwargs) - if result is not None and result != 0: - container.stop() - raise RuntimeError( - 'Failed to run command {} in container. Process returned exit code {} with output: {}'.format( - command, - result, - output - ) - ) - - return output - - @staticmethod - def execMultiple(container, commands, **kwargs): - ''' - Executes multiple commands in a container returned by `DockerUtils.start()` - ''' - for command in commands: - DockerUtils.exec(container, command, **kwargs) - - @staticmethod - def injectPostRunMessage(dockerfile, platform, messageLines): - ''' - Injects the supplied message at the end of each RUN directive in the specified Dockerfile - ''' - - # Generate the `echo` command for each line of the message - prefix = 'echo.' if platform == 'windows' else "echo '" - suffix = '' if platform == 'windows' else "'" - echoCommands = ''.join([' && {}{}{}'.format(prefix, line, suffix) for line in messageLines]) - - # Read the Dockerfile contents and convert all line endings to \n - contents = FilesystemUtils.readFile(dockerfile) - contents = contents.replace('\r\n', '\n') - - # Determine the escape character for the Dockerfile - escapeMatch = re.search('#[\\s]*escape[\\s]*=[\\s]*([^\n])\n', contents) - escape = escapeMatch[1] if escapeMatch is not None else '\\' - - # Identify each RUN directive in the Dockerfile - runMatches = re.finditer('^RUN(.+?[^{}])\n'.format(re.escape(escape)), contents, re.DOTALL | re.MULTILINE) - if runMatches is not None: - for match in runMatches: - - # Append the `echo` commands to the directive - contents = contents.replace(match[0], 'RUN{}{}\n'.format(match[1], echoCommands)) - - # Write the modified contents back to the Dockerfile - FilesystemUtils.writeFile(dockerfile, contents) + @staticmethod + def installed(): + """ + Determines if Docker is installed + """ + try: + return (DockerUtils.version() is not None), None + except Exception as e: + logging.debug(str(e)) + return False, e + + @staticmethod + def version(): + """ + Retrieves the version information for the Docker daemon + """ + client = docker.from_env() + return client.version() + + @staticmethod + def info(): + """ + Retrieves the system information as produced by `docker info` + """ + client = docker.from_env() + return client.info() + + @staticmethod + def exists(name): + """ + Determines if the specified image exists + """ + client = docker.from_env() + try: + image = client.images.get(name) + return True + except: + return False + + @staticmethod + def build(tags, context, args): + """ + Returns the `docker build` command to build an image + """ + tagArgs = [["-t", tag] for tag in tags] + return ( + ["docker", "build"] + + list(itertools.chain.from_iterable(tagArgs)) + + [context] + + args + ) + + @staticmethod + def buildx(tags, context, args, secrets): + """ + Returns the `docker buildx` command to build an image with the BuildKit backend + """ + tagArgs = [["-t", tag] for tag in tags] + return ( + ["docker", "buildx", "build"] + + list(itertools.chain.from_iterable(tagArgs)) + + [context] + + ["--progress=plain"] + + args + + list(itertools.chain.from_iterable([["--secret", s] for s in secrets])) + ) + + @staticmethod + def pull(image): + """ + Returns the `docker pull` command to pull an image from a remote registry + """ + return ["docker", "pull", image] + + @staticmethod + def start(image, command, **kwargs): + """ + Starts a container in a detached state and returns the container handle + """ + client = docker.from_env() + return client.containers.run(image, command, detach=True, **kwargs) + + @staticmethod + def configFilePath(): + """ + Returns the path to the Docker daemon configuration file under Windows + """ + return "{}\\Docker\\config\\daemon.json".format(os.environ["ProgramData"]) + + @staticmethod + def getConfig(): + """ + Retrieves and parses the Docker daemon configuration file under Windows + """ + configPath = DockerUtils.configFilePath() + if os.path.exists(configPath) == True: + with open(configPath) as configFile: + return json.load(configFile) + + return {} + + @staticmethod + def setConfig(config): + """ + Writes new values to the Docker daemon configuration file under Windows + """ + configPath = DockerUtils.configFilePath() + with open(configPath, "w") as configFile: + configFile.write(json.dumps(config)) + + @staticmethod + def maxsize(): + """ + Determines the configured size limit (in GB) for Windows containers + """ + if platform.system() != "Windows": + return -1 + + config = DockerUtils.getConfig() + if "storage-opts" in config: + sizes = [ + opt.replace("size=", "") + for opt in config["storage-opts"] + if "size=" in opt + ] + if len(sizes) > 0: + return humanfriendly.parse_size(sizes[0]) / 1000000000 + + # The default limit on image size is 20GB + # (https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container-issues) + return 20.0 + + @staticmethod + def listImages(tagFilter=None, filters={}, all=False): + """ + Retrieves the details for each image matching the specified filters + """ + + # Retrieve the list of images matching the specified filters + client = docker.from_env() + images = client.images.list(filters=filters, all=all) + + # Apply our tag filter if one was specified + if tagFilter is not None: + images = [ + i + for i in images + if len(i.tags) > 0 and len(fnmatch.filter(i.tags, tagFilter)) > 0 + ] + + return images + + @staticmethod + def exec(container, command, **kwargs): + """ + Executes a command in a container returned by `DockerUtils.start()` and returns the output + """ + result, output = container.exec_run(command, **kwargs) + if result is not None and result != 0: + container.stop() + raise RuntimeError( + "Failed to run command {} in container. Process returned exit code {} with output: {}".format( + command, result, output + ) + ) + + return output + + @staticmethod + def execMultiple(container, commands, **kwargs): + """ + Executes multiple commands in a container returned by `DockerUtils.start()` + """ + for command in commands: + DockerUtils.exec(container, command, **kwargs) + + @staticmethod + def injectPostRunMessage(dockerfile, platform, messageLines): + """ + Injects the supplied message at the end of each RUN directive in the specified Dockerfile + """ + + # Generate the `echo` command for each line of the message + prefix = "echo." if platform == "windows" else "echo '" + suffix = "" if platform == "windows" else "'" + echoCommands = "".join( + [" && {}{}{}".format(prefix, line, suffix) for line in messageLines] + ) + + # Read the Dockerfile contents and convert all line endings to \n + contents = FilesystemUtils.readFile(dockerfile) + contents = contents.replace("\r\n", "\n") + + # Determine the escape character for the Dockerfile + escapeMatch = re.search("#[\\s]*escape[\\s]*=[\\s]*([^\n])\n", contents) + escape = escapeMatch[1] if escapeMatch is not None else "\\" + + # Identify each RUN directive in the Dockerfile + runMatches = re.finditer( + "^RUN(.+?[^{}])\n".format(re.escape(escape)), + contents, + re.DOTALL | re.MULTILINE, + ) + if runMatches is not None: + for match in runMatches: + + # Append the `echo` commands to the directive + contents = contents.replace( + match[0], "RUN{}{}\n".format(match[1], echoCommands) + ) + + # Write the modified contents back to the Dockerfile + FilesystemUtils.writeFile(dockerfile, contents) diff --git a/ue4docker/infrastructure/FilesystemUtils.py b/ue4docker/infrastructure/FilesystemUtils.py index 51a59e49..c7a2d6ef 100644 --- a/ue4docker/infrastructure/FilesystemUtils.py +++ b/ue4docker/infrastructure/FilesystemUtils.py @@ -1,17 +1,16 @@ class FilesystemUtils(object): - - @staticmethod - def readFile(filename): - ''' - Reads data from a file - ''' - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - - @staticmethod - def writeFile(filename, data): - ''' - Writes data to a file - ''' - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + @staticmethod + def readFile(filename): + """ + Reads data from a file + """ + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + @staticmethod + def writeFile(filename, data): + """ + Writes data to a file + """ + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) diff --git a/ue4docker/infrastructure/GlobalConfiguration.py b/ue4docker/infrastructure/GlobalConfiguration.py index a9b43577..58e9b86f 100644 --- a/ue4docker/infrastructure/GlobalConfiguration.py +++ b/ue4docker/infrastructure/GlobalConfiguration.py @@ -3,34 +3,43 @@ # The default namespace for our tagged container images -DEFAULT_TAG_NAMESPACE = 'adamrehn' +DEFAULT_TAG_NAMESPACE = "adamrehn" class GlobalConfiguration(object): - ''' - Manages access to the global configuration settings for ue4-docker itself - ''' - - @staticmethod - def getLatestVersion(): - ''' - Queries PyPI to determine the latest available release of ue4-docker - ''' - releases = [parse_version(release) for release in requests.get('https://pypi.org/pypi/ue4-docker/json').json()['releases']] - return sorted(releases)[-1] - - @staticmethod - def getTagNamespace(): - ''' - Returns the currently-configured namespace for container image tags - ''' - return os.environ.get('UE4DOCKER_TAG_NAMESPACE', DEFAULT_TAG_NAMESPACE) - - @staticmethod - def resolveTag(tag): - ''' - Resolves a Docker image tag with respect to our currently-configured namespace - ''' - - # If the specified tag already includes a namespace, simply return it unmodified - return tag if '/' in tag else '{}/{}'.format(GlobalConfiguration.getTagNamespace(), tag) + """ + Manages access to the global configuration settings for ue4-docker itself + """ + + @staticmethod + def getLatestVersion(): + """ + Queries PyPI to determine the latest available release of ue4-docker + """ + releases = [ + parse_version(release) + for release in requests.get("https://pypi.org/pypi/ue4-docker/json").json()[ + "releases" + ] + ] + return sorted(releases)[-1] + + @staticmethod + def getTagNamespace(): + """ + Returns the currently-configured namespace for container image tags + """ + return os.environ.get("UE4DOCKER_TAG_NAMESPACE", DEFAULT_TAG_NAMESPACE) + + @staticmethod + def resolveTag(tag): + """ + Resolves a Docker image tag with respect to our currently-configured namespace + """ + + # If the specified tag already includes a namespace, simply return it unmodified + return ( + tag + if "/" in tag + else "{}/{}".format(GlobalConfiguration.getTagNamespace(), tag) + ) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index bb36764e..19284961 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -5,180 +5,217 @@ from os.path import basename, exists, join from jinja2 import Environment, Template + class ImageBuilder(object): - - def __init__(self, root, platform, logger, rebuild=False, dryRun=False, layoutDir=None, templateContext=None, combine=False): - ''' - Creates an ImageBuilder for the specified build parameters - ''' - self.root = root - self.platform = platform - self.logger = logger - self.rebuild = rebuild - self.dryRun = dryRun - self.layoutDir = layoutDir - self.templateContext = templateContext if templateContext is not None else {} - self.combine = combine - - def build(self, name, tags, args, secrets=None): - ''' - Builds the specified image if it doesn't exist or if we're forcing a rebuild - ''' - - # Create a Jinja template environment and render the Dockerfile template - environment = Environment(autoescape=False, trim_blocks=True, lstrip_blocks=True) - dockerfile = join(self.context(name), 'Dockerfile') - templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) - rendered = templateInstance.render(self.templateContext) - - # Compress excess whitespace introduced during Jinja rendering and save the contents back to disk - # (Ensure that we still have a single trailing newline at the end of the Dockerfile) - while '\n\n\n' in rendered: - rendered = rendered.replace('\n\n\n', '\n\n') - rendered = rendered.strip('\n') + '\n' - FilesystemUtils.writeFile(dockerfile, rendered) - - # Inject our filesystem layer commit message after each RUN directive in the Dockerfile - DockerUtils.injectPostRunMessage(dockerfile, self.platform, [ - '', - 'RUN directive complete. Docker will now commit the filesystem layer to disk.', - 'Note that for large filesystem layers this can take quite some time.', - 'Performing filesystem layer commit...', - '' - ]) - - # Create a temporary directory to hold any files needed for the build - with tempfile.TemporaryDirectory() as tempDir: - - # Determine whether we are building using `docker buildx` with build secrets - imageTags = self._formatTags(name, tags) - command = DockerUtils.build(imageTags, self.context(name), args) - if self.platform == 'linux' and secrets is not None and len(secrets) > 0: - - # Create temporary files to store the contents of each of our secrets - secretFlags = [] - for secret, contents in secrets.items(): - secretFile = join(tempDir, secret) - FilesystemUtils.writeFile(secretFile, contents) - secretFlags.append('id={},src={}'.format(secret, secretFile)) - - # Generate the `docker buildx` command to use our build secrets - command = DockerUtils.buildx(imageTags, self.context(name), args, secretFlags) - - # Build the image if it doesn't already exist - self._processImage( - imageTags[0], - name, - command, - 'build', - 'built' - ) - - def context(self, name): - ''' - Resolve the full path to the build context for the specified image - ''' - return join(self.root, basename(name), self.platform) - - def pull(self, image): - ''' - Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version - ''' - self._processImage( - image, - None, - DockerUtils.pull(image), - 'pull', - 'pulled' - ) - - def willBuild(self, name, tags): - ''' - Determines if we will build the specified image, based on our build settings - ''' - imageTags = self._formatTags(name, tags) - return self._willProcess(imageTags[0]) - - def _formatTags(self, name, tags): - ''' - Generates the list of fully-qualified tags that we will use when building an image - ''' - return ['{}:{}'.format(GlobalConfiguration.resolveTag(name), tag) for tag in tags] - - def _willProcess(self, image): - ''' - Determines if we will build or pull the specified image, based on our build settings - ''' - return self.rebuild == True or DockerUtils.exists(image) == False - - def _processImage(self, image, name, command, actionPresentTense, actionPastTense): - ''' - Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) - ''' - - # Determine if we are processing the image - if self._willProcess(image) == False: - self.logger.info('Image "{}" exists and rebuild not requested, skipping {}.'.format(image, actionPresentTense)) - return - - # Determine if we are running in "dry run" mode - self.logger.action('{}ing image "{}"...'.format(actionPresentTense.capitalize(), image)) - if self.dryRun == True: - print(command) - self.logger.action('Completed dry run for image "{}".'.format(image), newline=False) - return - - # Determine if we're just copying the Dockerfile to an output directory - if self.layoutDir is not None: - - # Determine whether we're performing a simple copy or combining generated Dockerfiles - source = self.context(name) - if self.combine == True: - - # Ensure the destination directory exists - dest = join(self.layoutDir, 'combined') - self.logger.action('Merging "{}" into "{}"...'.format(source, dest), newline=False) - os.makedirs(dest, exist_ok=True) - - # Merge the source Dockerfile with any existing Dockerfile contents in the destination directory - # (Insert a single newline between merged file contents and ensure we have a single trailing newline) - sourceDockerfile = join(source, 'Dockerfile') - destDockerfile = join(dest, 'Dockerfile') - dockerfileContents = FilesystemUtils.readFile(destDockerfile) if exists(destDockerfile) else '' - dockerfileContents = dockerfileContents + '\n' + FilesystemUtils.readFile(sourceDockerfile) - dockerfileContents = dockerfileContents.strip('\n') + '\n' - FilesystemUtils.writeFile(destDockerfile, dockerfileContents) - - # Copy any supplemental files from the source directory to the destination directory - # (Exclude any extraneous files which are not referenced in the Dockerfile contents) - for file in glob.glob(join(source, '*.*')): - if basename(file) in dockerfileContents: - shutil.copy(file, join(dest, basename(file))) - - # Report our success - self.logger.action('Merged Dockerfile for image "{}".'.format(image), newline=False) - - else: - - # Copy the source directory to the destination - dest = join(self.layoutDir, basename(name)) - self.logger.action('Copying "{}" to "{}"...'.format(source, dest), newline=False) - shutil.copytree(source, dest) - self.logger.action('Copied Dockerfile for image "{}".'.format(image), newline=False) - - return - - # Attempt to process the image using the supplied command - startTime = time.time() - exitCode = subprocess.call(command) - endTime = time.time() - - # Determine if processing succeeded - if exitCode == 0: - self.logger.action('{} image "{}" in {}'.format( - actionPastTense.capitalize(), - image, - humanfriendly.format_timespan(endTime - startTime) - ), newline=False) - else: - raise RuntimeError('failed to {} image "{}".'.format(actionPresentTense, image)) + def __init__( + self, + root, + platform, + logger, + rebuild=False, + dryRun=False, + layoutDir=None, + templateContext=None, + combine=False, + ): + """ + Creates an ImageBuilder for the specified build parameters + """ + self.root = root + self.platform = platform + self.logger = logger + self.rebuild = rebuild + self.dryRun = dryRun + self.layoutDir = layoutDir + self.templateContext = templateContext if templateContext is not None else {} + self.combine = combine + + def build(self, name, tags, args, secrets=None): + """ + Builds the specified image if it doesn't exist or if we're forcing a rebuild + """ + + # Create a Jinja template environment and render the Dockerfile template + environment = Environment( + autoescape=False, trim_blocks=True, lstrip_blocks=True + ) + dockerfile = join(self.context(name), "Dockerfile") + templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) + rendered = templateInstance.render(self.templateContext) + + # Compress excess whitespace introduced during Jinja rendering and save the contents back to disk + # (Ensure that we still have a single trailing newline at the end of the Dockerfile) + while "\n\n\n" in rendered: + rendered = rendered.replace("\n\n\n", "\n\n") + rendered = rendered.strip("\n") + "\n" + FilesystemUtils.writeFile(dockerfile, rendered) + + # Inject our filesystem layer commit message after each RUN directive in the Dockerfile + DockerUtils.injectPostRunMessage( + dockerfile, + self.platform, + [ + "", + "RUN directive complete. Docker will now commit the filesystem layer to disk.", + "Note that for large filesystem layers this can take quite some time.", + "Performing filesystem layer commit...", + "", + ], + ) + + # Create a temporary directory to hold any files needed for the build + with tempfile.TemporaryDirectory() as tempDir: + + # Determine whether we are building using `docker buildx` with build secrets + imageTags = self._formatTags(name, tags) + command = DockerUtils.build(imageTags, self.context(name), args) + if self.platform == "linux" and secrets is not None and len(secrets) > 0: + + # Create temporary files to store the contents of each of our secrets + secretFlags = [] + for secret, contents in secrets.items(): + secretFile = join(tempDir, secret) + FilesystemUtils.writeFile(secretFile, contents) + secretFlags.append("id={},src={}".format(secret, secretFile)) + + # Generate the `docker buildx` command to use our build secrets + command = DockerUtils.buildx( + imageTags, self.context(name), args, secretFlags + ) + + # Build the image if it doesn't already exist + self._processImage(imageTags[0], name, command, "build", "built") + + def context(self, name): + """ + Resolve the full path to the build context for the specified image + """ + return join(self.root, basename(name), self.platform) + + def pull(self, image): + """ + Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version + """ + self._processImage(image, None, DockerUtils.pull(image), "pull", "pulled") + + def willBuild(self, name, tags): + """ + Determines if we will build the specified image, based on our build settings + """ + imageTags = self._formatTags(name, tags) + return self._willProcess(imageTags[0]) + + def _formatTags(self, name, tags): + """ + Generates the list of fully-qualified tags that we will use when building an image + """ + return [ + "{}:{}".format(GlobalConfiguration.resolveTag(name), tag) for tag in tags + ] + + def _willProcess(self, image): + """ + Determines if we will build or pull the specified image, based on our build settings + """ + return self.rebuild == True or DockerUtils.exists(image) == False + + def _processImage(self, image, name, command, actionPresentTense, actionPastTense): + """ + Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) + """ + + # Determine if we are processing the image + if self._willProcess(image) == False: + self.logger.info( + 'Image "{}" exists and rebuild not requested, skipping {}.'.format( + image, actionPresentTense + ) + ) + return + + # Determine if we are running in "dry run" mode + self.logger.action( + '{}ing image "{}"...'.format(actionPresentTense.capitalize(), image) + ) + if self.dryRun == True: + print(command) + self.logger.action( + 'Completed dry run for image "{}".'.format(image), newline=False + ) + return + + # Determine if we're just copying the Dockerfile to an output directory + if self.layoutDir is not None: + + # Determine whether we're performing a simple copy or combining generated Dockerfiles + source = self.context(name) + if self.combine == True: + + # Ensure the destination directory exists + dest = join(self.layoutDir, "combined") + self.logger.action( + 'Merging "{}" into "{}"...'.format(source, dest), newline=False + ) + os.makedirs(dest, exist_ok=True) + + # Merge the source Dockerfile with any existing Dockerfile contents in the destination directory + # (Insert a single newline between merged file contents and ensure we have a single trailing newline) + sourceDockerfile = join(source, "Dockerfile") + destDockerfile = join(dest, "Dockerfile") + dockerfileContents = ( + FilesystemUtils.readFile(destDockerfile) + if exists(destDockerfile) + else "" + ) + dockerfileContents = ( + dockerfileContents + + "\n" + + FilesystemUtils.readFile(sourceDockerfile) + ) + dockerfileContents = dockerfileContents.strip("\n") + "\n" + FilesystemUtils.writeFile(destDockerfile, dockerfileContents) + + # Copy any supplemental files from the source directory to the destination directory + # (Exclude any extraneous files which are not referenced in the Dockerfile contents) + for file in glob.glob(join(source, "*.*")): + if basename(file) in dockerfileContents: + shutil.copy(file, join(dest, basename(file))) + + # Report our success + self.logger.action( + 'Merged Dockerfile for image "{}".'.format(image), newline=False + ) + + else: + + # Copy the source directory to the destination + dest = join(self.layoutDir, basename(name)) + self.logger.action( + 'Copying "{}" to "{}"...'.format(source, dest), newline=False + ) + shutil.copytree(source, dest) + self.logger.action( + 'Copied Dockerfile for image "{}".'.format(image), newline=False + ) + + return + + # Attempt to process the image using the supplied command + startTime = time.time() + exitCode = subprocess.call(command) + endTime = time.time() + + # Determine if processing succeeded + if exitCode == 0: + self.logger.action( + '{} image "{}" in {}'.format( + actionPastTense.capitalize(), + image, + humanfriendly.format_timespan(endTime - startTime), + ), + newline=False, + ) + else: + raise RuntimeError( + 'failed to {} image "{}".'.format(actionPresentTense, image) + ) diff --git a/ue4docker/infrastructure/ImageCleaner.py b/ue4docker/infrastructure/ImageCleaner.py index d876b518..b29cccb6 100644 --- a/ue4docker/infrastructure/ImageCleaner.py +++ b/ue4docker/infrastructure/ImageCleaner.py @@ -1,27 +1,27 @@ from .DockerUtils import DockerUtils import humanfriendly, os, subprocess, time + class ImageCleaner(object): - - def __init__(self, logger): - self.logger = logger - - def clean(self, image, dryRun=False): - ''' - Removes the specified image - ''' - - # Determine if we are running in "dry run" mode - self.logger.action('Removing image "{}"...'.format(image)) - cleanCommand = ['docker', 'rmi', image] - if dryRun == True: - print(cleanCommand) - else: - subprocess.call(cleanCommand) - - def cleanMultiple(self, images, dryRun=False): - ''' - Removes all of the images in the supplied list - ''' - for image in images: - self.clean(image, dryRun) + def __init__(self, logger): + self.logger = logger + + def clean(self, image, dryRun=False): + """ + Removes the specified image + """ + + # Determine if we are running in "dry run" mode + self.logger.action('Removing image "{}"...'.format(image)) + cleanCommand = ["docker", "rmi", image] + if dryRun == True: + print(cleanCommand) + else: + subprocess.call(cleanCommand) + + def cleanMultiple(self, images, dryRun=False): + """ + Removes all of the images in the supplied list + """ + for image in images: + self.clean(image, dryRun) diff --git a/ue4docker/infrastructure/Logger.py b/ue4docker/infrastructure/Logger.py index fd9db238..512892e7 100644 --- a/ue4docker/infrastructure/Logger.py +++ b/ue4docker/infrastructure/Logger.py @@ -1,33 +1,33 @@ from termcolor import colored import colorama, sys + class Logger(object): - - def __init__(self, prefix=''): - ''' - Creates a logger that will print coloured output to stderr - ''' - colorama.init() - self.prefix = prefix - - def action(self, output, newline=True): - ''' - Prints information about an action that is being performed - ''' - self._print('green', output, newline) - - def error(self, output, newline=False): - ''' - Prints information about an error that has occurred - ''' - self._print('red', output, newline) - - def info(self, output, newline=True): - ''' - Prints information that does not pertain to an action or an error - ''' - self._print('yellow', output, newline) - - def _print(self, colour, output, newline): - whitespace = '\n' if newline == True else '' - print(colored(whitespace + self.prefix + output, color=colour), file=sys.stderr) + def __init__(self, prefix=""): + """ + Creates a logger that will print coloured output to stderr + """ + colorama.init() + self.prefix = prefix + + def action(self, output, newline=True): + """ + Prints information about an action that is being performed + """ + self._print("green", output, newline) + + def error(self, output, newline=False): + """ + Prints information about an error that has occurred + """ + self._print("red", output, newline) + + def info(self, output, newline=True): + """ + Prints information that does not pertain to an action or an error + """ + self._print("yellow", output, newline) + + def _print(self, colour, output, newline): + whitespace = "\n" if newline == True else "" + print(colored(whitespace + self.prefix + output, color=colour), file=sys.stderr) diff --git a/ue4docker/infrastructure/NetworkUtils.py b/ue4docker/infrastructure/NetworkUtils.py index acba2116..210b775d 100644 --- a/ue4docker/infrastructure/NetworkUtils.py +++ b/ue4docker/infrastructure/NetworkUtils.py @@ -1,19 +1,19 @@ import socket + class NetworkUtils(object): - - @staticmethod - def hostIP(): - ''' - Determines the IP address of the host - ''' - # Code from - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except: - IP = '127.0.0.1' - finally: - s.close() - return IP + @staticmethod + def hostIP(): + """ + Determines the IP address of the host + """ + # Code from + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("10.255.255.255", 1)) + IP = s.getsockname()[0] + except: + IP = "127.0.0.1" + finally: + s.close() + return IP diff --git a/ue4docker/infrastructure/PackageUtils.py b/ue4docker/infrastructure/PackageUtils.py index 0948e3ad..a63a9cdd 100644 --- a/ue4docker/infrastructure/PackageUtils.py +++ b/ue4docker/infrastructure/PackageUtils.py @@ -1,20 +1,20 @@ import importlib.util, pkg_resources + class PackageUtils(object): - - @staticmethod - def getPackageLocation(package): - ''' - Attempts to retrieve the filesystem location for the specified Python package - ''' - return pkg_resources.get_distribution(package).location - - @staticmethod - def importFile(moduleName, filePath): - ''' - Directly imports a Python module from a source file - ''' - spec = importlib.util.spec_from_file_location(moduleName, filePath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module + @staticmethod + def getPackageLocation(package): + """ + Attempts to retrieve the filesystem location for the specified Python package + """ + return pkg_resources.get_distribution(package).location + + @staticmethod + def importFile(moduleName, filePath): + """ + Directly imports a Python module from a source file + """ + spec = importlib.util.spec_from_file_location(moduleName, filePath) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module diff --git a/ue4docker/infrastructure/PrettyPrinting.py b/ue4docker/infrastructure/PrettyPrinting.py index 03a5a54d..2c1745d8 100644 --- a/ue4docker/infrastructure/PrettyPrinting.py +++ b/ue4docker/infrastructure/PrettyPrinting.py @@ -1,15 +1,14 @@ class PrettyPrinting(object): - - @staticmethod - def printColumns(pairs, indent = 2, minSpaces = 6): - ''' - Prints a list of paired values in two nicely aligned columns - ''' - - # Determine the length of the longest item in the left-hand column - longestName = max([len(pair[0]) for pair in pairs]) - - # Print the two columns - for pair in pairs: - whitespace = ' ' * ((longestName + minSpaces) - len(pair[0])) - print('{}{}{}{}'.format(' ' * indent, pair[0], whitespace, pair[1])) + @staticmethod + def printColumns(pairs, indent=2, minSpaces=6): + """ + Prints a list of paired values in two nicely aligned columns + """ + + # Determine the length of the longest item in the left-hand column + longestName = max([len(pair[0]) for pair in pairs]) + + # Print the two columns + for pair in pairs: + whitespace = " " * ((longestName + minSpaces) - len(pair[0])) + print("{}{}{}{}".format(" " * indent, pair[0], whitespace, pair[1])) diff --git a/ue4docker/infrastructure/ResourceMonitor.py b/ue4docker/infrastructure/ResourceMonitor.py index bb225ff4..48e5fc24 100644 --- a/ue4docker/infrastructure/ResourceMonitor.py +++ b/ue4docker/infrastructure/ResourceMonitor.py @@ -1,79 +1,84 @@ import datetime, humanfriendly, os, psutil, shutil, threading, time from .DockerUtils import DockerUtils + class ResourceMonitor(threading.Thread): - - def __init__(self, logger, interval): - ''' - Creates a resource monitor with the specified configuration - ''' - super().__init__() - self._logger = logger - self._interval = interval - self._lock = threading.Lock() - self._shouldStop = False - - def stop(self): - ''' - Stops the resource monitor thread - ''' - - # Set the flag to instruct the resource monitor loop to stop - with self._lock: - self._shouldStop = True - - # Wait for the resource monitor thread to complete - if self.is_alive() == True: - self.join() - - def run(self): - ''' - The resource monitor loop itself - ''' - - # Determine which filesystem the Docker daemon uses for storing its data directory - dockerInfo = DockerUtils.info() - rootDir = dockerInfo['DockerRootDir'] - - # If we cannot access the Docker data directory (e.g. when the daemon is in a Moby VM), don't report disk space - reportDisk = os.path.exists(rootDir) - - # Sample the CPU usage using an interval of 1 second the first time to prime the system - # (See: ) - psutil.cpu_percent(1.0) - - # Loop until asked to stop - while True: - - # Check that the thread has not been asked to stop - with self._lock: - if self._shouldStop == True: - return - - # Format the timestamp for the current time in ISO 8601 format (albeit without the "T" separator) - isoTime = datetime.datetime.now().replace(microsecond=0).isoformat(' ') - - # We format data sizes using binary units (KiB, MiB, GiB, etc.) - formatSize = lambda size: humanfriendly.format_size(size, binary=True, keep_width=True) - - # Format the current quantity of available disk space on the Docker data directory's filesystem - diskSpace = formatSize(shutil.disk_usage(rootDir).free) if reportDisk == True else 'Unknown' - - # Format the current quantity of available system memory - physicalMemory = formatSize(psutil.virtual_memory().free) - virtualMemory = formatSize(psutil.swap_memory().free) - - # Format the current CPU usage levels - cpu = psutil.cpu_percent() - - # Report the current levels of our available resources - self._logger.info('[{}] [Available disk: {}] [Available memory: {} physical, {} virtual] [CPU usage: {:.2f}%]'.format( - isoTime, - diskSpace, - physicalMemory, - virtualMemory, - cpu - ), False) - - # Sleep until the next sampling interval - time.sleep(self._interval) + def __init__(self, logger, interval): + """ + Creates a resource monitor with the specified configuration + """ + super().__init__() + self._logger = logger + self._interval = interval + self._lock = threading.Lock() + self._shouldStop = False + + def stop(self): + """ + Stops the resource monitor thread + """ + + # Set the flag to instruct the resource monitor loop to stop + with self._lock: + self._shouldStop = True + + # Wait for the resource monitor thread to complete + if self.is_alive() == True: + self.join() + + def run(self): + """ + The resource monitor loop itself + """ + + # Determine which filesystem the Docker daemon uses for storing its data directory + dockerInfo = DockerUtils.info() + rootDir = dockerInfo["DockerRootDir"] + + # If we cannot access the Docker data directory (e.g. when the daemon is in a Moby VM), don't report disk space + reportDisk = os.path.exists(rootDir) + + # Sample the CPU usage using an interval of 1 second the first time to prime the system + # (See: ) + psutil.cpu_percent(1.0) + + # Loop until asked to stop + while True: + + # Check that the thread has not been asked to stop + with self._lock: + if self._shouldStop == True: + return + + # Format the timestamp for the current time in ISO 8601 format (albeit without the "T" separator) + isoTime = datetime.datetime.now().replace(microsecond=0).isoformat(" ") + + # We format data sizes using binary units (KiB, MiB, GiB, etc.) + formatSize = lambda size: humanfriendly.format_size( + size, binary=True, keep_width=True + ) + + # Format the current quantity of available disk space on the Docker data directory's filesystem + diskSpace = ( + formatSize(shutil.disk_usage(rootDir).free) + if reportDisk == True + else "Unknown" + ) + + # Format the current quantity of available system memory + physicalMemory = formatSize(psutil.virtual_memory().free) + virtualMemory = formatSize(psutil.swap_memory().free) + + # Format the current CPU usage levels + cpu = psutil.cpu_percent() + + # Report the current levels of our available resources + self._logger.info( + "[{}] [Available disk: {}] [Available memory: {} physical, {} virtual] [CPU usage: {:.2f}%]".format( + isoTime, diskSpace, physicalMemory, virtualMemory, cpu + ), + False, + ) + + # Sleep until the next sampling interval + time.sleep(self._interval) diff --git a/ue4docker/infrastructure/SubprocessUtils.py b/ue4docker/infrastructure/SubprocessUtils.py index 00805628..429b6ea5 100644 --- a/ue4docker/infrastructure/SubprocessUtils.py +++ b/ue4docker/infrastructure/SubprocessUtils.py @@ -2,53 +2,50 @@ class VerboseCalledProcessError(RuntimeError): - '''' - A verbose wrapper for `subprocess.CalledProcessError` that prints stdout and stderr - ''' - - def __init__(self, wrapped): - self.wrapped = wrapped - - def __str__(self): - return '{}\nstdout: {}\nstderr: {}'.format( - self.wrapped, - self.wrapped.output, - self.wrapped.stderr - ) + """' + A verbose wrapper for `subprocess.CalledProcessError` that prints stdout and stderr + """ + + def __init__(self, wrapped): + self.wrapped = wrapped + + def __str__(self): + return "{}\nstdout: {}\nstderr: {}".format( + self.wrapped, self.wrapped.output, self.wrapped.stderr + ) class SubprocessUtils(object): - - @staticmethod - def extractLines(output): - ''' - Extracts the individual lines from the output of a child process - ''' - return output.decode('utf-8').replace('\r\n', '\n').strip().split('\n') - - @staticmethod - def capture(command, check = True, **kwargs): - ''' - Executes a child process and captures its output. - - If the child process fails and `check` is True then a verbose exception will be raised. - ''' - try: - return subprocess.run( - command, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - check = check, - **kwargs - ) - except subprocess.CalledProcessError as e: - raise VerboseCalledProcessError(e) from None - - @staticmethod - def run(command, check = True, **kwargs): - ''' - Executes a child process. - - If the child process fails and `check` is True then a verbose exception will be raised. - ''' - return SubprocessUtils.capture(command, check, **kwargs) + @staticmethod + def extractLines(output): + """ + Extracts the individual lines from the output of a child process + """ + return output.decode("utf-8").replace("\r\n", "\n").strip().split("\n") + + @staticmethod + def capture(command, check=True, **kwargs): + """ + Executes a child process and captures its output. + + If the child process fails and `check` is True then a verbose exception will be raised. + """ + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=check, + **kwargs + ) + except subprocess.CalledProcessError as e: + raise VerboseCalledProcessError(e) from None + + @staticmethod + def run(command, check=True, **kwargs): + """ + Executes a child process. + + If the child process fails and `check` is True then a verbose exception will be raised. + """ + return SubprocessUtils.capture(command, check, **kwargs) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index a68732ec..36021ea0 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -3,127 +3,135 @@ import platform, sys from typing import Optional -if platform.system() == 'Windows': - import winreg +if platform.system() == "Windows": + import winreg + class WindowsUtils(object): - # The oldest Windows build we support - _minimumRequiredBuild = 17763 - - # This lookup table is based on the list of valid tags from - # and list of build-to-release mapping from https://docs.microsoft.com/en-us/windows/release-health/release-information - _knownTagsByBuildNumber = { - 17763: 'ltsc2019', - 18362: '1903', - 18363: '1909', - 19041: '2004', - 19042: '20H2', - 19043: '21H1', - } - - _knownTags = list(_knownTagsByBuildNumber.values()) - - # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs - # (See: ) - _blacklistedHosts = [18362, 18363] - - @staticmethod - def _getVersionRegKey(subkey : str) -> str: - ''' - Retrieves the specified Windows version key from the registry - - @raises FileNotFoundError if registry key doesn't exist - ''' - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') - value = winreg.QueryValueEx(key, subkey) - winreg.CloseKey(key) - return value[0] - - @staticmethod - def requiredSizeLimit() -> float: - ''' - Returns the minimum required image size limit (in GB) for Windows containers - ''' - return 400.0 - - @staticmethod - def minimumRequiredBuild() -> int: - ''' - Returns the minimum required version of Windows 10 / Windows Server - ''' - return WindowsUtils._minimumRequiredBuild - - @staticmethod - def systemString() -> str: - ''' - Generates a verbose human-readable version string for the Windows host system - ''' - return '{} (Build {}.{})'.format( - WindowsUtils._getVersionRegKey('ProductName'), - WindowsUtils.getWindowsBuild(), - WindowsUtils._getVersionRegKey('UBR') - ) - - @staticmethod - def getHostBaseTag() -> Optional[str]: - ''' - Retrieves the tag for the Windows Server Core base image matching the host Windows system - ''' - - hostBuild = WindowsUtils.getWindowsBuild() - - return WindowsUtils._knownTagsByBuildNumber.get(hostBuild) - - @staticmethod - def getWindowsBuild() -> int: - ''' - Returns build number for the Windows host system - ''' - return sys.getwindowsversion().build - - @staticmethod - def isBlacklistedWindowsHost() -> bool: - ''' - Determines if host Windows version is one with bugs that make it unsuitable for use - (defaults to checking the host OS release if one is not specified) - ''' - dockerVersion = parse_version(DockerUtils.version()['Version']) - build = WindowsUtils.getWindowsBuild() - return build in WindowsUtils._blacklistedHosts and dockerVersion < parse_version('19.03.6') - - @staticmethod - def isWindowsServer() -> bool: - ''' - Determines if the Windows host system is Windows Server - ''' - # TODO: Replace this with something more reliable - return 'Windows Server' in WindowsUtils._getVersionRegKey('ProductName') - - @staticmethod - def getDllSrcImage(basetag: str) -> str: - ''' - Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image - ''' - tag = { - 'ltsc2019': '1809', - }.get(basetag, basetag) - - return f'mcr.microsoft.com/windows:{tag}' - - @staticmethod - def getKnownBaseTags() -> [str]: - ''' - Returns the list of known tags for the Windows Server Core base image, in ascending chronological release order - ''' - return WindowsUtils._knownTags - - @staticmethod - def isNewerBaseTag(older: str, newer: str) -> Optional[bool]: - ''' - Determines if the base tag `newer` is chronologically newer than the base tag `older` - ''' - try: - return WindowsUtils._knownTags.index(newer) > WindowsUtils._knownTags.index(older) - except ValueError: - return None + # The oldest Windows build we support + _minimumRequiredBuild = 17763 + + # This lookup table is based on the list of valid tags from + # and list of build-to-release mapping from https://docs.microsoft.com/en-us/windows/release-health/release-information + _knownTagsByBuildNumber = { + 17763: "ltsc2019", + 18362: "1903", + 18363: "1909", + 19041: "2004", + 19042: "20H2", + 19043: "21H1", + } + + _knownTags = list(_knownTagsByBuildNumber.values()) + + # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs + # (See: ) + _blacklistedHosts = [18362, 18363] + + @staticmethod + def _getVersionRegKey(subkey: str) -> str: + """ + Retrieves the specified Windows version key from the registry + + @raises FileNotFoundError if registry key doesn't exist + """ + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" + ) + value = winreg.QueryValueEx(key, subkey) + winreg.CloseKey(key) + return value[0] + + @staticmethod + def requiredSizeLimit() -> float: + """ + Returns the minimum required image size limit (in GB) for Windows containers + """ + return 400.0 + + @staticmethod + def minimumRequiredBuild() -> int: + """ + Returns the minimum required version of Windows 10 / Windows Server + """ + return WindowsUtils._minimumRequiredBuild + + @staticmethod + def systemString() -> str: + """ + Generates a verbose human-readable version string for the Windows host system + """ + return "{} (Build {}.{})".format( + WindowsUtils._getVersionRegKey("ProductName"), + WindowsUtils.getWindowsBuild(), + WindowsUtils._getVersionRegKey("UBR"), + ) + + @staticmethod + def getHostBaseTag() -> Optional[str]: + """ + Retrieves the tag for the Windows Server Core base image matching the host Windows system + """ + + hostBuild = WindowsUtils.getWindowsBuild() + + return WindowsUtils._knownTagsByBuildNumber.get(hostBuild) + + @staticmethod + def getWindowsBuild() -> int: + """ + Returns build number for the Windows host system + """ + return sys.getwindowsversion().build + + @staticmethod + def isBlacklistedWindowsHost() -> bool: + """ + Determines if host Windows version is one with bugs that make it unsuitable for use + (defaults to checking the host OS release if one is not specified) + """ + dockerVersion = parse_version(DockerUtils.version()["Version"]) + build = WindowsUtils.getWindowsBuild() + return ( + build in WindowsUtils._blacklistedHosts + and dockerVersion < parse_version("19.03.6") + ) + + @staticmethod + def isWindowsServer() -> bool: + """ + Determines if the Windows host system is Windows Server + """ + # TODO: Replace this with something more reliable + return "Windows Server" in WindowsUtils._getVersionRegKey("ProductName") + + @staticmethod + def getDllSrcImage(basetag: str) -> str: + """ + Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image + """ + tag = { + "ltsc2019": "1809", + }.get(basetag, basetag) + + return f"mcr.microsoft.com/windows:{tag}" + + @staticmethod + def getKnownBaseTags() -> [str]: + """ + Returns the list of known tags for the Windows Server Core base image, in ascending chronological release order + """ + return WindowsUtils._knownTags + + @staticmethod + def isNewerBaseTag(older: str, newer: str) -> Optional[bool]: + """ + Determines if the base tag `newer` is chronologically newer than the base tag `older` + """ + try: + return WindowsUtils._knownTags.index(newer) > WindowsUtils._knownTags.index( + older + ) + except ValueError: + return None diff --git a/ue4docker/main.py b/ue4docker/main.py index 4a4bc3d7..47534dd7 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -1,4 +1,10 @@ -from .infrastructure import DarwinUtils, DockerUtils, Logger, PrettyPrinting, WindowsUtils +from .infrastructure import ( + DarwinUtils, + DockerUtils, + Logger, + PrettyPrinting, + WindowsUtils, +) from .build import build from .clean import clean from .diagnostics_cmd import diagnostics @@ -9,99 +15,113 @@ from .version_cmd import version import logging, os, platform, sys + def _exitWithError(err): - Logger().error(err) - sys.exit(1) + Logger().error(err) + sys.exit(1) + def main(): - - # Configure verbose logging if the user requested it - # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) - if '-v' in sys.argv or '--verbose' in sys.argv: - - # Enable verbose logging - logging.getLogger().setLevel(logging.DEBUG) - - # Filter out the verbose flag to avoid breaking commands that don't support it - if not (len(sys.argv) > 1 and sys.argv[1] in ['build']): - sys.argv = list([arg for arg in sys.argv if arg not in ['-v', '--verbose']]) - - # Verify that Docker is installed - installed, error = DockerUtils.installed() - if installed == False: - _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}'.format(error)) - - # Under Windows, verify that the host is a supported version - if platform.system() == 'Windows': - host_build = WindowsUtils.getWindowsBuild() - min_build = WindowsUtils.minimumRequiredBuild() - if host_build < min_build: - _exitWithError('Error: the detected build of Windows ({}) is not supported. {} or newer is required.'.format(host_build, min_build)) - - # Under macOS, verify that the host is a supported version - if platform.system() == 'Darwin' and DarwinUtils.isSupportedMacOsVersion() == False: - _exitWithError('Error: the detected version of macOS ({}) is not supported. macOS {} or newer is required.'.format(DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion())) - - # Our supported commands - COMMANDS = { - 'build': { - 'function': build, - 'description': 'Builds container images for a specific version of UE4' - }, - 'clean': { - 'function': clean, - 'description': 'Cleans built container images' - }, - 'diagnostics': { - 'function': diagnostics, - 'description': 'Runs diagnostics to detect issues with the host system configuration' - }, - 'export': { - 'function': export, - 'description': 'Exports components from built container images to the host system' - }, - 'info': { - 'function': info, - 'description': 'Displays information about the host system and Docker daemon' - }, - 'setup': { - 'function': setup, - 'description': 'Automatically configures the host system where possible' - }, - 'test': { - 'function': test, - 'description': 'Runs tests to verify the correctness of built container images' - }, - 'version': { - 'function': version, - 'description': 'Prints the ue4-docker version number' - } - } - - # Truncate argv[0] to just the command name without the full path - sys.argv[0] = os.path.basename(sys.argv[0]) - - # Determine if a command has been specified - if len(sys.argv) > 1: - - # Verify that the specified command is valid - command = sys.argv[1] - if command not in COMMANDS: - print('Error: unrecognised command "{}".'.format(command), file=sys.stderr) - sys.exit(1) - - # Invoke the command - sys.argv = [sys.argv[0]] + sys.argv[2:] - COMMANDS[command]['function']() - - else: - - # Print usage syntax - print('Usage: {} COMMAND [OPTIONS]\n'.format(sys.argv[0])) - print('Windows and Linux containers for Unreal Engine 4\n') - print('Commands:') - PrettyPrinting.printColumns([ - (command, COMMANDS[command]['description']) - for command in COMMANDS - ]) - print('\nRun `{} COMMAND --help` for more information on a command.'.format(sys.argv[0])) + + # Configure verbose logging if the user requested it + # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) + if "-v" in sys.argv or "--verbose" in sys.argv: + + # Enable verbose logging + logging.getLogger().setLevel(logging.DEBUG) + + # Filter out the verbose flag to avoid breaking commands that don't support it + if not (len(sys.argv) > 1 and sys.argv[1] in ["build"]): + sys.argv = list([arg for arg in sys.argv if arg not in ["-v", "--verbose"]]) + + # Verify that Docker is installed + installed, error = DockerUtils.installed() + if installed == False: + _exitWithError( + "Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}".format( + error + ) + ) + + # Under Windows, verify that the host is a supported version + if platform.system() == "Windows": + host_build = WindowsUtils.getWindowsBuild() + min_build = WindowsUtils.minimumRequiredBuild() + if host_build < min_build: + _exitWithError( + "Error: the detected build of Windows ({}) is not supported. {} or newer is required.".format( + host_build, min_build + ) + ) + + # Under macOS, verify that the host is a supported version + if platform.system() == "Darwin" and DarwinUtils.isSupportedMacOsVersion() == False: + _exitWithError( + "Error: the detected version of macOS ({}) is not supported. macOS {} or newer is required.".format( + DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion() + ) + ) + + # Our supported commands + COMMANDS = { + "build": { + "function": build, + "description": "Builds container images for a specific version of UE4", + }, + "clean": {"function": clean, "description": "Cleans built container images"}, + "diagnostics": { + "function": diagnostics, + "description": "Runs diagnostics to detect issues with the host system configuration", + }, + "export": { + "function": export, + "description": "Exports components from built container images to the host system", + }, + "info": { + "function": info, + "description": "Displays information about the host system and Docker daemon", + }, + "setup": { + "function": setup, + "description": "Automatically configures the host system where possible", + }, + "test": { + "function": test, + "description": "Runs tests to verify the correctness of built container images", + }, + "version": { + "function": version, + "description": "Prints the ue4-docker version number", + }, + } + + # Truncate argv[0] to just the command name without the full path + sys.argv[0] = os.path.basename(sys.argv[0]) + + # Determine if a command has been specified + if len(sys.argv) > 1: + + # Verify that the specified command is valid + command = sys.argv[1] + if command not in COMMANDS: + print('Error: unrecognised command "{}".'.format(command), file=sys.stderr) + sys.exit(1) + + # Invoke the command + sys.argv = [sys.argv[0]] + sys.argv[2:] + COMMANDS[command]["function"]() + + else: + + # Print usage syntax + print("Usage: {} COMMAND [OPTIONS]\n".format(sys.argv[0])) + print("Windows and Linux containers for Unreal Engine 4\n") + print("Commands:") + PrettyPrinting.printColumns( + [(command, COMMANDS[command]["description"]) for command in COMMANDS] + ) + print( + "\nRun `{} COMMAND --help` for more information on a command.".format( + sys.argv[0] + ) + ) diff --git a/ue4docker/setup_cmd.py b/ue4docker/setup_cmd.py index 4a8e0c35..13a730a2 100644 --- a/ue4docker/setup_cmd.py +++ b/ue4docker/setup_cmd.py @@ -3,107 +3,156 @@ # Runs a command without displaying its output and returns the exit code def _runSilent(command): - result = SubprocessUtils.capture(command, check=False) - return result.returncode + result = SubprocessUtils.capture(command, check=False) + return result.returncode + # Performs setup for Linux hosts def _setupLinux(): - - # Pull the latest version of the Alpine container image - alpineImage = 'alpine:latest' - SubprocessUtils.capture(['docker', 'pull', alpineImage]) - - # Start the credential endpoint with blank credentials - endpoint = CredentialEndpoint('', '') - endpoint.start() - - try: - - # Run an Alpine container to see if we can access the host port for the credential endpoint - SubprocessUtils.capture([ - 'docker', 'run', '--rm', alpineImage, - 'wget', '--timeout=1', '--post-data=dummy', 'http://{}:9876'.format(NetworkUtils.hostIP()) - ], check=True) - - # If we reach this point then the host port is accessible - print('No firewall configuration required.') - - except: - - # The host port is blocked, so we need to perform firewall configuration - print('Creating firewall rule for credential endpoint...') - - # Create the firewall rule - subprocess.run(['iptables', '-I', 'INPUT', '-p', 'tcp', '--dport', '9876', '-j', 'ACCEPT'], check=True) - - # Ensure the firewall rule persists after reboot - # (Requires the `iptables-persistent` service to be installed and running) - os.makedirs('/etc/iptables', exist_ok=True) - subprocess.run('iptables-save > /etc/iptables/rules.v4', shell=True, check=True) - - # Inform users of the `iptables-persistent` requirement - print('Firewall rule created. Note that the `iptables-persistent` service will need to') - print('be installed for the rule to persist after the host system reboots.') - - finally: - - # Stop the credential endpoint - endpoint.stop() + + # Pull the latest version of the Alpine container image + alpineImage = "alpine:latest" + SubprocessUtils.capture(["docker", "pull", alpineImage]) + + # Start the credential endpoint with blank credentials + endpoint = CredentialEndpoint("", "") + endpoint.start() + + try: + + # Run an Alpine container to see if we can access the host port for the credential endpoint + SubprocessUtils.capture( + [ + "docker", + "run", + "--rm", + alpineImage, + "wget", + "--timeout=1", + "--post-data=dummy", + "http://{}:9876".format(NetworkUtils.hostIP()), + ], + check=True, + ) + + # If we reach this point then the host port is accessible + print("No firewall configuration required.") + + except: + + # The host port is blocked, so we need to perform firewall configuration + print("Creating firewall rule for credential endpoint...") + + # Create the firewall rule + subprocess.run( + ["iptables", "-I", "INPUT", "-p", "tcp", "--dport", "9876", "-j", "ACCEPT"], + check=True, + ) + + # Ensure the firewall rule persists after reboot + # (Requires the `iptables-persistent` service to be installed and running) + os.makedirs("/etc/iptables", exist_ok=True) + subprocess.run("iptables-save > /etc/iptables/rules.v4", shell=True, check=True) + + # Inform users of the `iptables-persistent` requirement + print( + "Firewall rule created. Note that the `iptables-persistent` service will need to" + ) + print("be installed for the rule to persist after the host system reboots.") + + finally: + + # Stop the credential endpoint + endpoint.stop() + # Performs setup for Windows Server hosts def _setupWindowsServer(): - - # Check if we need to configure the maximum image size - requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit: - - # Attempt to stop the Docker daemon - print('Stopping the Docker daemon...') - subprocess.run(['sc.exe', 'stop', 'docker'], check=True) - - # Attempt to set the maximum image size - print('Setting maximum image size to {}GB...'.format(requiredLimit)) - config = DockerUtils.getConfig() - sizeOpt = 'size={}GB'.format(requiredLimit) - if 'storage-opts' in config: - config['storage-opts'] = list([o for o in config['storage-opts'] if o.lower().startswith('size=') == False]) - config['storage-opts'].append(sizeOpt) - else: - config['storage-opts'] = [sizeOpt] - DockerUtils.setConfig(config) - - # Attempt to start the Docker daemon - print('Starting the Docker daemon...') - subprocess.run(['sc.exe', 'start', 'docker'], check=True) - - else: - print('Maximum image size is already correctly configured.') - - # Determine if we need to configure Windows firewall - ruleName = 'Open TCP port 9876 for ue4-docker credential endpoint' - ruleExists = _runSilent(['netsh', 'advfirewall', 'firewall', 'show', 'rule', 'name={}'.format(ruleName)]) == 0 - if ruleExists == False: - - # Add a rule to ensure Windows firewall allows access to the credential helper from our containers - print('Creating firewall rule for credential endpoint...') - subprocess.run([ - 'netsh', 'advfirewall', - 'firewall', 'add', 'rule', - 'name={}'.format(ruleName), 'dir=in', 'action=allow', 'protocol=TCP', 'localport=9876' - ], check=True) - - else: - print('Firewall rule for credential endpoint is already configured.') + + # Check if we need to configure the maximum image size + requiredLimit = WindowsUtils.requiredSizeLimit() + if DockerUtils.maxsize() < requiredLimit: + + # Attempt to stop the Docker daemon + print("Stopping the Docker daemon...") + subprocess.run(["sc.exe", "stop", "docker"], check=True) + + # Attempt to set the maximum image size + print("Setting maximum image size to {}GB...".format(requiredLimit)) + config = DockerUtils.getConfig() + sizeOpt = "size={}GB".format(requiredLimit) + if "storage-opts" in config: + config["storage-opts"] = list( + [ + o + for o in config["storage-opts"] + if o.lower().startswith("size=") == False + ] + ) + config["storage-opts"].append(sizeOpt) + else: + config["storage-opts"] = [sizeOpt] + DockerUtils.setConfig(config) + + # Attempt to start the Docker daemon + print("Starting the Docker daemon...") + subprocess.run(["sc.exe", "start", "docker"], check=True) + + else: + print("Maximum image size is already correctly configured.") + + # Determine if we need to configure Windows firewall + ruleName = "Open TCP port 9876 for ue4-docker credential endpoint" + ruleExists = ( + _runSilent( + [ + "netsh", + "advfirewall", + "firewall", + "show", + "rule", + "name={}".format(ruleName), + ] + ) + == 0 + ) + if ruleExists == False: + + # Add a rule to ensure Windows firewall allows access to the credential helper from our containers + print("Creating firewall rule for credential endpoint...") + subprocess.run( + [ + "netsh", + "advfirewall", + "firewall", + "add", + "rule", + "name={}".format(ruleName), + "dir=in", + "action=allow", + "protocol=TCP", + "localport=9876", + ], + check=True, + ) + + else: + print("Firewall rule for credential endpoint is already configured.") + def setup(): - - # We don't currently support auto-config for VM-based containers - if platform.system() == 'Darwin' or (platform.system() == 'Windows' and WindowsUtils.isWindowsServer() == False): - print('Manual configuration is required under Windows 10 and macOS. Automatic configuration is not available.') - return - - # Perform setup based on the host system type - if platform.system() == 'Linux': - _setupLinux() - else: - _setupWindowsServer() + + # We don't currently support auto-config for VM-based containers + if platform.system() == "Darwin" or ( + platform.system() == "Windows" and WindowsUtils.isWindowsServer() == False + ): + print( + "Manual configuration is required under Windows 10 and macOS. Automatic configuration is not available." + ) + return + + # Perform setup based on the host system type + if platform.system() == "Linux": + _setupLinux() + else: + _setupWindowsServer() diff --git a/ue4docker/test.py b/ue4docker/test.py index 8da5ab88..100bf0ff 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -2,63 +2,84 @@ from container_utils import ContainerUtils, ImageUtils import docker, os, platform, sys + def test(): - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} test] '.format(sys.argv[0])) - - # Create our Docker API client - client = docker.from_env() - - # Check that an image tag has been specified - if len(sys.argv) > 1 and sys.argv[1].strip('-') not in ['h', 'help']: - - # Verify that the specified container image exists - tag = sys.argv[1] - image = GlobalConfiguration.resolveTag('ue4-full:{}'.format(tag) if ':' not in tag else tag) - if DockerUtils.exists(image) == False: - logger.error('Error: the specified container image "{}" does not exist.'.format(image)) - sys.exit(1) - - # Use process isolation mode when testing Windows containers, since running Hyper-V containers don't currently support manipulating the filesystem - platform = ImageUtils.image_platform(client, image) - isolation = 'process' if platform == 'windows' else None - - # Start a container to run our tests in, automatically stopping and removing the container when we finish - logger.action('Starting a container using the "{}" image...'.format(image), False) - container = ContainerUtils.start_for_exec(client, image, isolation=isolation) - with ContainerUtils.automatically_stop(container): - - # Create the workspace directory in the container - workspaceDir = ContainerUtils.workspace_dir(container) - ContainerUtils.exec(container, ContainerUtils.shell_prefix(container) + ['mkdir ' + workspaceDir]) - - # Copy our test scripts into the container - testDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'tests') - ContainerUtils.copy_from_host(container, testDir, workspaceDir) - - # Create a harness to invoke individual tests - containerPath = ContainerUtils.path(container) - pythonCommand = 'python' if ContainerUtils.container_platform(container) == 'windows' else 'python3' - def runTest(script): - logger.action('Running test "{}"...'.format(script), False) - try: - ContainerUtils.exec(container, [pythonCommand, containerPath.join(workspaceDir, script)], workdir=workspaceDir) - logger.action('Passed test "{}"'.format(script), False) - except RuntimeError as e: - logger.error('Error: test "{}" failed!'.format(script)) - raise e from None - - # Run each of our tests in turn - runTest('build-and-package.py') - runTest('consume-external-deps.py') - - # If we've reached this point then all of the tests passed - logger.action('All tests passed.', False) - - else: - - # Print usage syntax - print('Usage: {} test TAG'.format(sys.argv[0])) - print('Runs tests to verify the correctness of built container images\n') - print('TAG should specify the tag of the ue4-full image to test.') + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} test] ".format(sys.argv[0])) + + # Create our Docker API client + client = docker.from_env() + + # Check that an image tag has been specified + if len(sys.argv) > 1 and sys.argv[1].strip("-") not in ["h", "help"]: + + # Verify that the specified container image exists + tag = sys.argv[1] + image = GlobalConfiguration.resolveTag( + "ue4-full:{}".format(tag) if ":" not in tag else tag + ) + if DockerUtils.exists(image) == False: + logger.error( + 'Error: the specified container image "{}" does not exist.'.format( + image + ) + ) + sys.exit(1) + + # Use process isolation mode when testing Windows containers, since running Hyper-V containers don't currently support manipulating the filesystem + platform = ImageUtils.image_platform(client, image) + isolation = "process" if platform == "windows" else None + + # Start a container to run our tests in, automatically stopping and removing the container when we finish + logger.action( + 'Starting a container using the "{}" image...'.format(image), False + ) + container = ContainerUtils.start_for_exec(client, image, isolation=isolation) + with ContainerUtils.automatically_stop(container): + + # Create the workspace directory in the container + workspaceDir = ContainerUtils.workspace_dir(container) + ContainerUtils.exec( + container, + ContainerUtils.shell_prefix(container) + ["mkdir " + workspaceDir], + ) + + # Copy our test scripts into the container + testDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tests") + ContainerUtils.copy_from_host(container, testDir, workspaceDir) + + # Create a harness to invoke individual tests + containerPath = ContainerUtils.path(container) + pythonCommand = ( + "python" + if ContainerUtils.container_platform(container) == "windows" + else "python3" + ) + + def runTest(script): + logger.action('Running test "{}"...'.format(script), False) + try: + ContainerUtils.exec( + container, + [pythonCommand, containerPath.join(workspaceDir, script)], + workdir=workspaceDir, + ) + logger.action('Passed test "{}"'.format(script), False) + except RuntimeError as e: + logger.error('Error: test "{}" failed!'.format(script)) + raise e from None + + # Run each of our tests in turn + runTest("build-and-package.py") + runTest("consume-external-deps.py") + + # If we've reached this point then all of the tests passed + logger.action("All tests passed.", False) + + else: + + # Print usage syntax + print("Usage: {} test TAG".format(sys.argv[0])) + print("Runs tests to verify the correctness of built container images\n") + print("TAG should specify the tag of the ue4-full image to test.") diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index 257237e8..3871ee15 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -4,23 +4,23 @@ # Runs a command, raising an error if it returns a nonzero exit code def run(command, **kwargs): - print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) - return subprocess.run(command, check=True, **kwargs) + print("[RUN COMMAND] {} {}".format(command, kwargs), flush=True) + return subprocess.run(command, check=True, **kwargs) # Retrieve the short version string for the Engine manager = ue4cli.UnrealManagerFactory.create() -version = manager.getEngineVersion('short') +version = manager.getEngineVersion("short") # Create an auto-deleting temporary directory to work in with tempfile.TemporaryDirectory() as tempDir: - - # Clone a simple C++ project and verify that we can build and package it - repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) - projectDir = os.path.join(tempDir, 'BasicCxx') - run(['git', 'clone', '--depth=1', repo, projectDir]) - run(['ue4', 'package', 'Shipping'], cwd=projectDir) - - # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory - if platform.system() == 'Windows': - run(['del', '/f', '/s', '/q', os.path.join(projectDir, '.git')], shell=True) + + # Clone a simple C++ project and verify that we can build and package it + repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) + projectDir = os.path.join(tempDir, "BasicCxx") + run(["git", "clone", "--depth=1", repo, projectDir]) + run(["ue4", "package", "Shipping"], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == "Windows": + run(["del", "/f", "/s", "/q", os.path.join(projectDir, ".git")], shell=True) diff --git a/ue4docker/tests/consume-external-deps.py b/ue4docker/tests/consume-external-deps.py index 6ff50b6b..2d093956 100644 --- a/ue4docker/tests/consume-external-deps.py +++ b/ue4docker/tests/consume-external-deps.py @@ -4,57 +4,62 @@ # Reads data from a file def read(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') + with open(filename, "rb") as f: + return f.read().decode("utf-8") + # Runs a command, raising an error if it returns a nonzero exit code def run(command, **kwargs): - print('[RUN COMMAND] {} {}'.format(command, kwargs), flush=True) - return subprocess.run(command, check=True, **kwargs) + print("[RUN COMMAND] {} {}".format(command, kwargs), flush=True) + return subprocess.run(command, check=True, **kwargs) + # Writes data to a file def write(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) # Retrieve the short version string for the Engine manager = ue4cli.UnrealManagerFactory.create() -version = manager.getEngineVersion('short') +version = manager.getEngineVersion("short") # Create an auto-deleting temporary directory to work in with tempfile.TemporaryDirectory() as tempDir: - - # Clone a simple C++ project - repo = 'https://gitlab.com/ue4-test-projects/{}/BasicCxx.git'.format(version) - projectDir = os.path.join(tempDir, 'BasicCxx') - run(['git', 'clone', '--depth=1', repo, projectDir]) - - # Generate a code module to wrap our external dependencies - sourceDir = os.path.join(projectDir, 'Source') - run(['ue4', 'conan', 'boilerplate', 'WrapperModule'], cwd=sourceDir) - - # Add the wrapper module as a dependency of the project's main source code module - rulesFile = os.path.join(sourceDir, 'BasicCxx', 'BasicCxx.Build.cs') - rules = read(rulesFile) - rules = rules.replace('PublicDependencyModuleNames.AddRange(new string[] {', 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ') - write(rulesFile, rules) - - # Add some dependencies to the module's conanfile.py - moduleDir = os.path.join(sourceDir, 'WrapperModule') - conanfile = os.path.join(moduleDir, 'conanfile.py') - deps = read(conanfile) - deps = deps.replace('pass', 'self._requireUnreal("zlib/ue4@adamrehn/{}")') - write(conanfile, deps) - - # Verify that we can build the project with dynamically located dependencies - run(['ue4', 'build'], cwd=projectDir) - run(['ue4', 'clean'], cwd=projectDir) - - # Verify that we can build the project with precomputed dependency data - run(['ue4', 'conan', 'precompute', 'host'], cwd=moduleDir) - run(['ue4', 'build'], cwd=projectDir) - - # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory - if platform.system() == 'Windows': - run(['del', '/f', '/s', '/q', os.path.join(projectDir, '.git')], shell=True) + + # Clone a simple C++ project + repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) + projectDir = os.path.join(tempDir, "BasicCxx") + run(["git", "clone", "--depth=1", repo, projectDir]) + + # Generate a code module to wrap our external dependencies + sourceDir = os.path.join(projectDir, "Source") + run(["ue4", "conan", "boilerplate", "WrapperModule"], cwd=sourceDir) + + # Add the wrapper module as a dependency of the project's main source code module + rulesFile = os.path.join(sourceDir, "BasicCxx", "BasicCxx.Build.cs") + rules = read(rulesFile) + rules = rules.replace( + "PublicDependencyModuleNames.AddRange(new string[] {", + 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ', + ) + write(rulesFile, rules) + + # Add some dependencies to the module's conanfile.py + moduleDir = os.path.join(sourceDir, "WrapperModule") + conanfile = os.path.join(moduleDir, "conanfile.py") + deps = read(conanfile) + deps = deps.replace("pass", 'self._requireUnreal("zlib/ue4@adamrehn/{}")') + write(conanfile, deps) + + # Verify that we can build the project with dynamically located dependencies + run(["ue4", "build"], cwd=projectDir) + run(["ue4", "clean"], cwd=projectDir) + + # Verify that we can build the project with precomputed dependency data + run(["ue4", "conan", "precompute", "host"], cwd=moduleDir) + run(["ue4", "build"], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == "Windows": + run(["del", "/f", "/s", "/q", os.path.join(projectDir, ".git")], shell=True) diff --git a/ue4docker/version.py b/ue4docker/version.py index 7464c991..8a5bfd7f 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = '0.0.88' +__version__ = "0.0.88" diff --git a/ue4docker/version_cmd.py b/ue4docker/version_cmd.py index 01cfc13b..e74c6800 100644 --- a/ue4docker/version_cmd.py +++ b/ue4docker/version_cmd.py @@ -1,4 +1,5 @@ from .version import __version__ + def version(): - print(__version__) + print(__version__) From 3b0a3c06064ecbfcd114efbd7854eec7ec513602 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 22:25:39 +0300 Subject: [PATCH 187/430] Bump version to 0.0.89 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 8a5bfd7f..a20d2104 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.88" +__version__ = "0.0.89" From 1e1cfd05935c39764dbdb66948bdf15bb777ccc2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 22:29:21 +0300 Subject: [PATCH 188/430] Fix setup.py to properly parse version number from version.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a081ab12..789ed47b 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Read the version number from version.py with open(abspath(join(dirname(__file__), "ue4docker", "version.py"))) as versionFile: __version__ = ( - versionFile.read().strip().replace("__version__ = ", "").replace("'", "") + versionFile.read().strip().replace("__version__ = ", "").replace("\"", "") ) setup( From 16d3035783e07b4d3826e04fd7aa206e0a957b62 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 22:31:50 +0300 Subject: [PATCH 189/430] Apply Black formatting --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 789ed47b..de848fa9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ # Read the version number from version.py with open(abspath(join(dirname(__file__), "ue4docker", "version.py"))) as versionFile: __version__ = ( - versionFile.read().strip().replace("__version__ = ", "").replace("\"", "") + versionFile.read().strip().replace("__version__ = ", "").replace('"', "") ) setup( From 9bd34d9074c6d77bdf6437b7c0fef76b60c9b2c2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 31 Aug 2021 22:43:15 +0300 Subject: [PATCH 190/430] Run choco-cleaner in ue4-build-prerequisites image to reduce its size by ~300MB --- .../windows/install-prerequisites.bat | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 13277be3..d92d02aa 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,7 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error @rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y curl vcredist-all windows-sdk-10-version-1809-windbg || goto :error +choco install -y choco-cleaner curl vcredist-all windows-sdk-10-version-1809-windbg || goto :error choco install -y python --version=3.7.5 || goto :error @rem Reload our environment variables from the registry so the `git` command works @@ -60,6 +60,9 @@ python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error rmdir /S /Q \\?\%TEMP% mkdir %TEMP% +@rem This shaves off ~300MB as of 2021-08-31 +choco-cleaner || goto :error + @rem Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file @rem Just remove it, so a proper one will be generated on next NuGet run @rem See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034 From e6b92a928cbb5984c381f61791b7e3f4d1ccaf55 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 1 Sep 2021 10:11:32 +0300 Subject: [PATCH 191/430] Force LF line endings in .sh files This commit fixes error in `ue4-docker build --linux` when using Git clone of ue4-docker on Windows. resolves #194 --- .gitattributes | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..50ca329f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh eol=lf From 62d0e0005f357992d26b4bd74a0988c4ae86b3e9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 6 Sep 2021 19:52:57 +0300 Subject: [PATCH 192/430] Install Windows SDK >= 18362 when using VS2019 to fix compilation of UE-4.27.0 (#195) resolves #192 --- .../windows/install-prerequisites.bat | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index d92d02aa..d862879b 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -33,6 +33,29 @@ powershell -Command "Copy-Item -Path \"*\x64\vulkan-1.dll\" -Destination C:\Wind set VISUAL_STUDIO_BUILD_NUMBER=%~1 +@rem Use the latest available Windows SDK. The motivation behind this is: +@rem 1. Newer SDKs allow developers to use newer APIs. Developers can guard that API usage with runtime version checks if they want to continue to support older Windows releases. +@rem 2. Unreal Engine slowly moves to newer Windows SDK. 4.27.0 no longer compiles with SDKs older than 18362 and even if it will be fixed in 4.27.x, +@rem this is just a question of a time when older SDKs support will be dropped completely +@rem 3. UE5 doesn't support VS2017 at all, so in the future that argument for continuing to use Windows SDK 17763 from VS2017 era will be weaker and weaker. +@rem +@rem We can't use newer SDK for VS2017 that is used to compile older engines because 18362 SDK support was only added in UE-4.23. +@rem +@rem See https://github.com/adamrehn/ue4-docker/issues/192 +@rem See https://forums.unrealengine.com/t/ndis_miniport_major_version-is-not-defined-error/135058 +@rem See https://github.com/EpicGames/UnrealEngine/blame/4.23.0-release/Engine/Source/Programs/UnrealBuildTool/Platform/Windows/UEBuildWindows.cs#L1822-L1823 +@rem See https://github.com/EpicGames/UnrealEngine/commit/ecc4872c3269e75a24adc40734cc8bcc9bbed1ca +@rem See https://udn.unrealengine.com/s/question/0D54z000079HcjJCAS/d3d12h427-error-c4668-winapipartitiongames-is-not-defined-as-a-preprocessor-macro-replacing-with-0-for-ifelif +@rem +@rem Keywords for Google: +@rem error C4668: 'NDIS_MINIPORT_MAJOR_VERSION' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif +@rem d3d12.h(427): error C4668: 'WINAPI_PARTITION_GAMES' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' +if "%VISUAL_STUDIO_BUILD_NUMBER%" == "15" ( + set WINDOWS_SDK_VERSION=17763 +) else ( + set WINDOWS_SDK_VERSION=20348 +) + @rem Install the Visual Studio Build Tools workloads and components we need @rem NOTE: We use the Visual Studio 2019 installer even for Visual Studio 2017 here because the old installer now breaks @rem NOTE: VS2019 Build Tools doesn't have 4.6.2 .NET SDK and what actually gets installed is 4.8 @@ -49,7 +72,7 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output --add Microsoft.VisualStudio.Workload.MSBuildTools ^ --add Microsoft.VisualStudio.Component.NuGet ^ --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ - --add Microsoft.VisualStudio.Component.Windows10SDK.17763 ^ + --add Microsoft.VisualStudio.Component.Windows10SDK.%WINDOWS_SDK_VERSION% ^ --add Microsoft.Net.Component.4.5.TargetingPack ^ --add Microsoft.Net.ComponentGroup.4.6.2.DeveloperTools ^ --add Microsoft.NetCore.Component.SDK From 6f83074d1eb619ad5396c2099fb4463e98719c1b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 8 Sep 2021 02:01:02 -0700 Subject: [PATCH 193/430] Add support for Windows Server 2022 LTSC (#199) resolves #190 --- ue4docker/infrastructure/WindowsUtils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 36021ea0..84dd3fa7 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -21,6 +21,7 @@ class WindowsUtils(object): 19041: "2004", 19042: "20H2", 19043: "21H1", + 20348: "ltsc2022", } _knownTags = list(_knownTagsByBuildNumber.values()) @@ -111,11 +112,16 @@ def getDllSrcImage(basetag: str) -> str: """ Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image """ + # TODO: we also need to use Windows Server image when user specifies custom tags, like '10.0.20348.169' + image = { + "ltsc2022": "mcr.microsoft.com/windows/server", + }.get(basetag, "mcr.microsoft.com/windows") + tag = { "ltsc2019": "1809", }.get(basetag, basetag) - return f"mcr.microsoft.com/windows:{tag}" + return f"{image}:{tag}" @staticmethod def getKnownBaseTags() -> [str]: From 603a8c46b22bbe2a978b786dec963cfc390ee224 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 8 Sep 2021 14:17:07 -0700 Subject: [PATCH 194/430] Fix crash during OpenCVLensDistortion plugin startup (#198) `opencv_world331.dll` depends on `avifil32.dll`/`msacm32.dll` that are missing in ue4-docker images. This causes OpenCVLensDistortion fail to load and crash, leading to DDC build failure on UE-4.27 unless ue4-docker build is invoked with `--exclude ddc`. resolves #196 --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 5c1470bc..6bead1cc 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -10,6 +10,7 @@ SHELL ["cmd", "/S", "/C"] # Gather the system DLLs that we need from the full Windows base image COPY --from=dlls ` C:\Windows\System32\avicap32.dll ` + C:\Windows\System32\avifil32.dll ` C:\Windows\System32\avrt.dll ` C:\Windows\System32\d3d10warp.dll ` C:\Windows\System32\D3DSCache.dll ` @@ -20,6 +21,7 @@ COPY --from=dlls ` C:\Windows\System32\mfplat.dll ` C:\Windows\System32\mfplay.dll ` C:\Windows\System32\mfreadwrite.dll ` + C:\Windows\System32\msacm32.dll ` C:\Windows\System32\msdmo.dll ` C:\Windows\System32\msvfw32.dll ` C:\Windows\System32\opengl32.dll ` From 5760732174b92b9b093ebe14e722aad96b9ff73d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 9 Sep 2021 00:18:08 +0300 Subject: [PATCH 195/430] Bump version to 0.0.90 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index a20d2104..69d27b41 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.89" +__version__ = "0.0.90" From c78a3ede76c55f19ca1bf48c66fb69b8a6a86cf5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 9 Sep 2021 19:57:47 +0300 Subject: [PATCH 196/430] Fix ue4-full build failure on Linux caused by boost-1.67 missing in Conan repo This commit drops [MediaIPC] and [UE4Capture] from ue4-full image. The motivation behind this change is: 1. UE4Capture only works for 4.19 + 4.20. We just clone into /home/ue4/UE4Capture and not connect to the engine in any way 2. MediaIPC is only needed for UE4Capture, but see previous item. 3. Both of these libs do not look like something useful for generic Unreal Engine usage. Users can always install whatever software they need on top of images built by ue4-docker resolves #201 [MediaIPC]: https://github.com/adamrehn/MediaIPC [UE4Capture]: https://github.com/adamrehn/UE4Capture --- ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index aa4da63c..99566bf9 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -30,9 +30,6 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION -# Clone the UE4Capture repository -RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture - # Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake @@ -51,9 +48,6 @@ COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan RUN ue4 setroot /home/ue4/UnrealEngine RUN ue4 conan generate --profile-only -# Build the Conan packages for the UE4Capture dependencies -RUN ue4 conan build MediaIPC-ue4 - # Enable PulseAudio support USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils From 505a995d6fc2cd196edeeaee5265eefb0eed0f8d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 10 Sep 2021 13:40:23 +1000 Subject: [PATCH 197/430] Fix #169: add support for exporting Installed Builds from ue4-minimal --- ue4docker/exports/export_installed.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py index e078d05d..762f399c 100644 --- a/ue4docker/exports/export_installed.py +++ b/ue4docker/exports/export_installed.py @@ -1,10 +1,5 @@ -from ..infrastructure import DockerUtils, PackageUtils, SubprocessUtils -import os, platform, shutil, subprocess, sys - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile( - "semver", os.path.join(PackageUtils.getPackageLocation("semver"), "semver.py") -) +from ..infrastructure import DockerUtils, SubprocessUtils +import json, os, platform, shutil, subprocess, sys def exportInstalledBuild(image, destination, extraArgs): @@ -23,11 +18,12 @@ def exportInstalledBuild(image, destination, extraArgs): # Verify that the Installed Build in the specified image is at least 4.21.0 versionResult = SubprocessUtils.capture( - ["docker", "run", "--rm", "-ti", image, "ue4", "version"] + ["docker", "run", "--rm", "-ti", image, "cat", "/home/ue4/UnrealEngine/Engine/Build/Build.version"], + universal_newlines=True ) try: - version = semver.parse(SubprocessUtils.extractLines(versionResult.stdout)[-1]) - if version["minor"] < 21: + version = json.loads(versionResult.stdout) + if version["MajorVersion"] == 4 and version["MinorVersion"] < 21: raise Exception() except: print( From 62101e4647906a358d17a65aaace7a04195b1ec6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 10 Sep 2021 14:12:14 +1000 Subject: [PATCH 198/430] Disable credential endpoint when using BuildKit secrets --- ue4docker/build.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 31faead5..5450d41b 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -331,8 +331,10 @@ def build(): resourceMonitor.start() # Start the HTTP credential endpoint as a child process and wait for it to start - endpoint = CredentialEndpoint(username, password) - endpoint.start() + endpoint = None + if config.opts.get("credential_mode", "endpoint") == "endpoint": + endpoint = CredentialEndpoint(username, password) + endpoint.start() try: @@ -374,7 +376,7 @@ def build(): # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets secrets = {} - if config.opts.get("use_build_secrets", False) == True: + if config.opts.get("credential_mode", "endpoint") == "secrets": secrets = {"username": username, "password": password} # Build the UE4 source image @@ -539,12 +541,14 @@ def build(): resourceMonitor.stop() # Stop the HTTP server - endpoint.stop() + if endpoint is not None: + endpoint.stop() except (Exception, KeyboardInterrupt) as e: # One of the images failed to build logger.error("Error: {}".format(e)) resourceMonitor.stop() - endpoint.stop() + if endpoint is not None: + endpoint.stop() sys.exit(1) From 9fdb6d98141fa7b831d07665eb7b13022909384d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 10 Sep 2021 14:13:08 +1000 Subject: [PATCH 199/430] Apply formatting with Black for commit 505a995 --- ue4docker/exports/export_installed.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py index 762f399c..2c94e7d3 100644 --- a/ue4docker/exports/export_installed.py +++ b/ue4docker/exports/export_installed.py @@ -18,8 +18,16 @@ def exportInstalledBuild(image, destination, extraArgs): # Verify that the Installed Build in the specified image is at least 4.21.0 versionResult = SubprocessUtils.capture( - ["docker", "run", "--rm", "-ti", image, "cat", "/home/ue4/UnrealEngine/Engine/Build/Build.version"], - universal_newlines=True + [ + "docker", + "run", + "--rm", + "-ti", + image, + "cat", + "/home/ue4/UnrealEngine/Engine/Build/Build.version", + ], + universal_newlines=True, ) try: version = json.loads(versionResult.stdout) From 27cee0f7b7ea02b497a868718d536563d122a987 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 27 Sep 2021 14:32:23 +0300 Subject: [PATCH 200/430] Bump version to 0.0.91 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 69d27b41..e4692667 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.90" +__version__ = "0.0.91" From 331807a00f6a67c368bfb181883718447bdf781e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 29 Sep 2021 17:51:44 +1000 Subject: [PATCH 201/430] Add option to enable glibc DSO patch bundled by @slonopotamus --- .../ue4-build-prerequisites/linux/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index a0f894ad..4cbeed14 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -20,6 +20,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python3-dev \ python3-pip \ shared-mime-info \ + software-properties-common \ sudo \ tzdata \ unzip \ @@ -27,6 +28,17 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zip && \ rm -rf /var/lib/apt/lists/* +{% if enable_dso_patch %} +# Install the glibc DSO patch to improve Editor startup times +RUN add-apt-repository -y ppa:slonopotamus/glibc-dso && \ + apt-get update && \ + apt upgrade -y libc6 && \ + rm -rf /var/lib/apt/lists/* + +# Enable the glibc DSO patch +ENV GLIBC_TUNABLES=glibc.rtld.dynamic_sort=2 +{% endif %} + # Disable the default "lecture" message the first time a user runs a command using sudo RUN echo 'Defaults lecture="never"' >> /etc/sudoers From ee7d29e25cd5a5b0e81820a839c07038334e1d96 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 5 Oct 2021 19:44:01 +1000 Subject: [PATCH 202/430] Move X11 support to ue4-minimal image to support cooking WebBrowserWidget projects --- .../dockerfiles/ue4-full/linux/Dockerfile | 27 ----------------- .../dockerfiles/ue4-minimal/linux/Dockerfile | 29 +++++++++++++++++++ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 99566bf9..be372c2c 100644 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -52,31 +52,4 @@ RUN ue4 conan generate --profile-only USER root RUN apt-get install -y --no-install-recommends pulseaudio-utils COPY pulseaudio-client.conf /etc/pulse/client.conf - -# Enable X11 support (including the libraries required by CEF) and xvfb so we can create a dummy display if needed -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display -RUN apt-get install -y --no-install-recommends \ - libasound2 \ - libatk1.0-0 \ - libcairo2 \ - libfontconfig1 \ - libfreetype6 \ - libglu1 \ - libnss3 \ - libnspr4 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libsm6 \ - libxcomposite1 \ - libxcursor1 \ - libxi6 \ - libxrandr2 \ - libxrender1 \ - libxss1 \ - libxv1 \ - x11-xkb-utils \ - xauth \ - xfonts-base \ - xkb-data \ - xvfb USER ue4 diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 3d7c85a6..449ab981 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -109,4 +109,33 @@ RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ \"api_version\" : \"${VULKAN_API_VERSION}\"\ }\ }" > /etc/vulkan/icd.d/nvidia_icd.json + +# Install the X11 runtime libraries required by CEF so we can cook Unreal Engine projects that use the WebBrowserWidget plugin +# (This also allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display +RUN apt-get update && apt-get install -y --no-install-recommends \ + libasound2 \ + libatk1.0-0 \ + libcairo2 \ + libfontconfig1 \ + libfreetype6 \ + libglu1 \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libsm6 \ + libxcomposite1 \ + libxcursor1 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + libxv1 \ + x11-xkb-utils \ + xauth \ + xfonts-base \ + xkb-data && \ + rm -rf /var/lib/apt/lists/* USER ue4 From 99bba1ee99ea73d140e27f4d233122bb20a2063f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 7 Oct 2021 14:45:53 +1000 Subject: [PATCH 203/430] Add option to override changelist value in Build.version --- ue4docker/build.py | 7 ++++++- .../dockerfiles/ue4-minimal/linux/Dockerfile | 3 ++- .../ue4-minimal/linux/set-changelist.py | 15 +++++++++++---- .../dockerfiles/ue4-minimal/windows/Dockerfile | 3 ++- .../ue4-minimal/windows/set-changelist.py | 15 +++++++++++---- ue4docker/infrastructure/BuildConfiguration.py | 6 ++++++ 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 5450d41b..e6939f7b 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -419,12 +419,17 @@ def build(): ) # Build the minimal UE4 CI image, unless requested otherwise by the user + minimalArgs = ( + ["--build-arg", "CHANGELIST={}".format(config.changelist)] + if config.changelist is not None + else [] + ) buildUe4Minimal = config.noMinimal == False if buildUe4Minimal == True: builder.build( "ue4-minimal", mainTags, - commonArgs + config.platformArgs + ue4BuildArgs, + commonArgs + config.platformArgs + ue4BuildArgs + minimalArgs, ) builtImages.append("ue4-minimal") else: diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 449ab981..1e548569 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -11,8 +11,9 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder RUN rm -rf /home/ue4/UnrealEngine/.git # Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST COPY set-changelist.py /tmp/set-changelist.py -RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version +RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version $CHANGELIST {% if (not disable_all_patches) and (not disable_opengl_patch) %} # Enable the OpenGL RHI for Engine versions where it is present but deprecated diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py index fc5aa38a..ef0f52e1 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import json, os, sys +import json, sys def readFile(filename): @@ -12,9 +12,16 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file +# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided versionFile = sys.argv[1] +changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 else None details = json.loads(readFile(versionFile)) -details["Changelist"] = details["CompatibleChangelist"] +details["Changelist"] = ( + changelistOverride + if changelistOverride is not None + else details["CompatibleChangelist"] +) details["IsPromotedBuild"] = 1 -writeFile(versionFile, json.dumps(details, indent=4)) +patchedJson = json.dumps(details, indent=4) +writeFile(versionFile, patchedJson) +print("PATCHED BUILD.VERSION:\n{}".format(patchedJson), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 2aeca816..0ea15765 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -12,8 +12,9 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder RUN rmdir /s /q C:\UnrealEngine\.git # Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST COPY set-changelist.py C:\set-changelist.py -RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version +RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% {% if (not disable_all_patches) and (not disable_buildgraph_patches) %} # Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index fc5aa38a..ef0f52e1 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import json, os, sys +import json, sys def readFile(filename): @@ -12,9 +12,16 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file +# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided versionFile = sys.argv[1] +changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 else None details = json.loads(readFile(versionFile)) -details["Changelist"] = details["CompatibleChangelist"] +details["Changelist"] = ( + changelistOverride + if changelistOverride is not None + else details["CompatibleChangelist"] +) details["IsPromotedBuild"] = 1 -writeFile(versionFile, json.dumps(details, indent=4)) +patchedJson = json.dumps(details, indent=4) +writeFile(versionFile, patchedJson) +print("PATCHED BUILD.VERSION:\n{}".format(patchedJson), file=sys.stderr) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 1590512a..1e268e68 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -228,6 +228,11 @@ def addArguments(parser): action="store_true", help="Enable verbose output during builds (useful for debugging)", ) + parser.add_argument( + "-changelist", + default=None, + help="Set a specific changelist number in the Unreal Engine's Build.version file", + ) def __init__(self, parser, argv): """ @@ -303,6 +308,7 @@ def __init__(self, parser, argv): self.verbose = self.args.verbose self.layoutDir = self.args.layout self.combine = self.args.combine + self.changelist = self.args.changelist # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them self.ue4cliVersion = self._processPackageVersion("ue4cli", self.args.ue4cli) From 030e456472a595619ba6ba195d06cf56d369e4aa Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 7 Oct 2021 15:52:52 +1000 Subject: [PATCH 204/430] Automatically use correct changelist number for supported .0 releases --- ue4docker/build.py | 8 +++++++ .../infrastructure/BuildConfiguration.py | 24 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index e6939f7b..4a1f3a1a 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -279,6 +279,14 @@ def build(): # Report which Engine components are being excluded (if any) logger.info("GENERAL SETTINGS", False) + logger.info( + "Changelist override: {}".format( + config.changelist + if config.changelist is not None + else "(None specified)" + ), + False, + ) if len(config.excludedComponents) > 0: logger.info("Excluding the following Engine components:", False) for component in config.describeExcludedComponents(): diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 1e268e68..644dd004 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -29,6 +29,18 @@ # The default memory limit (in GB) under Windows DEFAULT_MEMORY_LIMIT = 10.0 +# The Perforce changelist numbers for each supported .0 release of the Unreal Engine +UNREAL_ENGINE_RELEASE_CHANGELISTS = { + "4.20.0": 4212847, + "4.21.0": 4541578, + "4.22.0": 5660361, + "4.23.0": 8386587, + "4.24.0": 10570792, + "4.25.0": 13144385, + "4.26.0": 14830424, + "4.27.0": 17155196, +} + class VisualStudio(object): VS2017 = "2017" @@ -230,6 +242,7 @@ def addArguments(parser): ) parser.add_argument( "-changelist", + type=int, default=None, help="Set a specific changelist number in the Unreal Engine's Build.version file", ) @@ -244,6 +257,7 @@ def __init__(self, parser, argv): # Parse the supplied command-line arguments self.args = parser.parse_args(argv) + self.changelist = self.args.changelist # Determine if we are building a custom version of UE4 rather than an official release self.args.release = self.args.release.lower() @@ -288,6 +302,15 @@ def __init__(self, parser, argv): self.branch = "{}-release".format(self.release) self.custom = False + # If the user specified a .0 release of the Unreal Engine and did not specify a changelist override then + # use the official changelist number for that release to ensure consistency with Epic Games Launcher builds + # (This is necessary because .0 releases do not include a `CompatibleChangelist` value in Build.version) + if ( + self.changelist is None + and self.release in UNREAL_ENGINE_RELEASE_CHANGELISTS + ): + self.changelist = UNREAL_ENGINE_RELEASE_CHANGELISTS[self.release] + # Store our common configuration settings self.containerPlatform = ( "windows" @@ -308,7 +331,6 @@ def __init__(self, parser, argv): self.verbose = self.args.verbose self.layoutDir = self.args.layout self.combine = self.args.combine - self.changelist = self.args.changelist # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them self.ue4cliVersion = self._processPackageVersion("ue4cli", self.args.ue4cli) From 5bb353fa470f624144c45abc0cf94c7ed6e383b2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 7 Oct 2021 15:59:11 +1000 Subject: [PATCH 205/430] Bump version to 0.0.92 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index e4692667..418ccd19 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.91" +__version__ = "0.0.92" From d1221ad9d7e24fbdfb0138157ca6afefb45260d6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 26 Oct 2021 14:36:41 +0300 Subject: [PATCH 206/430] Add missing ResampleDMO.dll on Windows Server 2022 (#211) ResampleDMO.dll is a dependency of dsound.dll resolves #210 --- ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 6bead1cc..245f06d1 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -25,6 +25,7 @@ COPY --from=dlls ` C:\Windows\System32\msdmo.dll ` C:\Windows\System32\msvfw32.dll ` C:\Windows\System32\opengl32.dll ` + C:\Windows\System32\ResampleDMO.dll ` C:\Windows\System32\ResourcePolicyClient.dll ` C:\Windows\System32\ From 0079ca6d9b6a0a3815ee09246c36340521054bf5 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 27 Oct 2021 11:54:00 +1000 Subject: [PATCH 207/430] Work around Windows env expansion behaviour for undefined vars --- ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index ef0f52e1..93381883 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -14,7 +14,7 @@ def writeFile(filename, data): # Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided versionFile = sys.argv[1] -changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 else None +changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2] != '%CHANGELIST%' else None details = json.loads(readFile(versionFile)) details["Changelist"] = ( changelistOverride From 00fe69849d6c1e098bc276f9797e9b7542e7d55b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 27 Oct 2021 11:56:16 +1000 Subject: [PATCH 208/430] Run Black formatter for previous commit --- ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index 93381883..72734f1f 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -14,7 +14,9 @@ def writeFile(filename, data): # Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided versionFile = sys.argv[1] -changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2] != '%CHANGELIST%' else None +changelistOverride = ( + int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2] != "%CHANGELIST%" else None +) details = json.loads(readFile(versionFile)) details["Changelist"] = ( changelistOverride From 1205c2c7690114443042e7350b81c59baeea6d12 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Nov 2021 18:36:37 +1000 Subject: [PATCH 209/430] Add libnss3 to build prereqs under Linux since UE5 needs it for Quixel Bridge plugin --- ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 4cbeed14..3179b26d 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ git \ git-lfs \ + libnss3 \ python3 \ python3-dev \ python3-pip \ From c9e72938596ee2da406312bd7a878669b4850df0 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 1 Nov 2021 18:38:52 +1000 Subject: [PATCH 210/430] Accept semver version strings for UE5 --- ue4docker/infrastructure/BuildConfiguration.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 644dd004..33b2d954 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -285,14 +285,17 @@ def __init__(self, parser, argv): # Validate the specified version string try: ue4Version = semver.parse(self.args.release) - if ue4Version["major"] != 4 or ue4Version["prerelease"] != None: + if ( + ue4Version["major"] not in [4, 5] + or ue4Version["prerelease"] != None + ): raise Exception() self.release = semver.format_version( ue4Version["major"], ue4Version["minor"], ue4Version["patch"] ) except: raise RuntimeError( - 'invalid UE4 release number "{}", full semver format required (e.g. "4.20.0")'.format( + 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.20.0")'.format( self.args.release ) ) From f56b6173a018a505bc8995673224e77bdf816095 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 2 Nov 2021 17:39:53 +1000 Subject: [PATCH 211/430] UE5 compat: move all CEF dependencies to build prereqs image under Linux --- .../ue4-build-prerequisites/linux/Dockerfile | 33 ++++++++++++++++++- .../dockerfiles/ue4-minimal/linux/Dockerfile | 29 ---------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 3179b26d..4e970f5a 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -9,6 +9,10 @@ LABEL com.adamrehn.ue4-docker.sentinel="1" # Disable interactive prompts during package installation ENV DEBIAN_FRONTEND=noninteractive +# Add the "display" driver capability for NVIDIA GPUs +# (This also allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display + # Install our build prerequisites RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ @@ -16,7 +20,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ git \ git-lfs \ - libnss3 \ python3 \ python3-dev \ python3-pip \ @@ -29,6 +32,34 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zip && \ rm -rf /var/lib/apt/lists/* +# Install the X11 runtime libraries required by CEF so we can cook Unreal Engine projects that use the WebBrowserWidget plugin +# (Starting in Unreal Engine 5.0, we need these installed before creating an Installed Build to prevent cooking failures related to loading the Quixel Bridge plugin) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libasound2 \ + libatk1.0-0 \ + libcairo2 \ + libfontconfig1 \ + libfreetype6 \ + libglu1 \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libsm6 \ + libxcomposite1 \ + libxcursor1 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + libxv1 \ + x11-xkb-utils \ + xauth \ + xfonts-base \ + xkb-data && \ + rm -rf /var/lib/apt/lists/* + {% if enable_dso_patch %} # Install the glibc DSO patch to improve Editor startup times RUN add-apt-repository -y ppa:slonopotamus/glibc-dso && \ diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 1e548569..b08e5f07 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -110,33 +110,4 @@ RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ \"api_version\" : \"${VULKAN_API_VERSION}\"\ }\ }" > /etc/vulkan/icd.d/nvidia_icd.json - -# Install the X11 runtime libraries required by CEF so we can cook Unreal Engine projects that use the WebBrowserWidget plugin -# (This also allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display -RUN apt-get update && apt-get install -y --no-install-recommends \ - libasound2 \ - libatk1.0-0 \ - libcairo2 \ - libfontconfig1 \ - libfreetype6 \ - libglu1 \ - libnss3 \ - libnspr4 \ - libpango-1.0-0 \ - libpangocairo-1.0-0 \ - libsm6 \ - libxcomposite1 \ - libxcursor1 \ - libxi6 \ - libxrandr2 \ - libxrender1 \ - libxss1 \ - libxtst6 \ - libxv1 \ - x11-xkb-utils \ - xauth \ - xfonts-base \ - xkb-data && \ - rm -rf /var/lib/apt/lists/* USER ue4 From d1919796ba25298e0851ad479bf28e98b05d11d9 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 2 Nov 2021 17:58:03 +1000 Subject: [PATCH 212/430] Invalidate BuildKit cache for git clone when CL override changes Note that this is only necessary when using BuildKit with build secrets, since using the credential endpoint will always invalidate the cache between runs due to changing token values, and copying the source from the build context will automatically invalidate the cache if the files change. --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 8402d83a..c210df42 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -33,9 +33,12 @@ ENV GIT_ASKPASS=/tmp/git-credential-helper-secrets.sh RUN chmod +x /tmp/git-credential-helper-secrets.sh # Clone the UE4 git repository using the build secret credentials +# (Note that we include the changelist override value here to ensure any cached source code is invalidated if +# the override is modified between runs, which is useful when testing preview versions of the Unreal Engine) +ARG CHANGELIST RUN --mount=type=secret,id=username,uid=1000,required \ --mount=type=secret,id=password,uid=1000,required \ - git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine + CHANGELIST="$CHANGELIST" git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine {% else %} From 6c0782139bb30c9d2de1eabaf10c5d4b25e2d482 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 4 Nov 2021 17:22:22 +1000 Subject: [PATCH 213/430] Leverage BuildKit cache mounts for binary dependencies when available --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index c210df42..e46d2930 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -70,12 +70,24 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT {% endif %} # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` -# (Note that the `-no-cache` flag disables caching of dependency data in `.git/ue4-gitdeps`, saving disk space) +{% if credential_mode == "secrets" %} + +# When running with BuildKit, we use a cache mount to cache the dependency data in `.git/ue4-gitdeps` across multiple build invocations +WORKDIR /home/ue4/UnrealEngine +RUN --mount=type=cache,target=/home/ue4/UnrealEngine/.git/ue4-gitdeps,uid=1000,gid=1000 sudo apt-get update && \ + ./Setup.sh && \ + sudo rm -rf /var/lib/apt/lists/* + +{% else %} + +# When running without BuildKit, we use the `-no-cache` flag to disable caching of dependency data in `.git/ue4-gitdeps`, saving disk space WORKDIR /home/ue4/UnrealEngine RUN sudo apt-get update && \ ./Setup.sh -no-cache && \ sudo rm -rf /var/lib/apt/lists/* +{% endif %} + {% if (not disable_all_patches) and (not disable_linker_fixup) %} # The linker bundled with UE4.20.0 onwards chokes on system libraries built with newer compilers, # so redirect the bundled clang to use the system linker instead From 082681ee603ce954a9e2027a07196c189c101c30 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 5 Nov 2021 15:56:13 +1000 Subject: [PATCH 214/430] Add missing dependency for UE5 Quixel Bridge plugin under Linux --- ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 4e970f5a..cc92cbc8 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \ libasound2 \ libatk1.0-0 \ + libatk-bridge2.0-0 \ libcairo2 \ libfontconfig1 \ libfreetype6 \ From ac15080670f7b1ee15a870e867fbcdaf521041a0 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 8 Nov 2021 17:29:34 +1000 Subject: [PATCH 215/430] Bump version to 0.0.93 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 418ccd19..a31af385 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.92" +__version__ = "0.0.93" From cceebd24829c560f93af49951a37532972c3afae Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 28 Nov 2021 20:23:29 +0300 Subject: [PATCH 216/430] Change `docker export installed` to use stopped container (#212) This is a preparation step for #126. According to it, `docker cp` cannot copy files out of running Hyper-V Windows containers. --- ue4docker/exports/export_installed.py | 8 ++++---- ue4docker/infrastructure/DockerUtils.py | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py index 2c94e7d3..5c01b2c3 100644 --- a/ue4docker/exports/export_installed.py +++ b/ue4docker/exports/export_installed.py @@ -40,16 +40,16 @@ def exportInstalledBuild(image, destination, extraArgs): ) sys.exit(1) - # Start a container from which we will copy files - container = DockerUtils.start(image, "bash") + # Create a container from which we will copy files + container = DockerUtils.create(image) # Attempt to perform the export print("Exporting to {}...".format(destination)) containerPath = "{}:/home/ue4/UnrealEngine".format(container.name) exportResult = subprocess.call(["docker", "cp", containerPath, destination]) - # Stop the container, irrespective of whether or not the export succeeded - container.stop() + # Remove the container, irrespective of whether or not the export succeeded + container.remove() # If the export succeeded, regenerate the linker symlinks on the host system if exportResult == 0: diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 536db54c..acb56044 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -1,4 +1,6 @@ import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re +from docker.models.containers import Container + from .FilesystemUtils import FilesystemUtils @@ -85,6 +87,14 @@ def start(image, command, **kwargs): client = docker.from_env() return client.containers.run(image, command, detach=True, **kwargs) + @staticmethod + def create(image: str, **kwargs) -> Container: + """ + Creates a stopped container for specified image name and returns the container handle + """ + client = docker.from_env() + return client.containers.create(image, **kwargs) + @staticmethod def configFilePath(): """ From 1e8bc3a9b105192a3120b60501d43d068d31e72a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 2 Dec 2021 20:43:30 +0300 Subject: [PATCH 217/430] Add `ue4-docker export installed` support on Windows (#213) Closes #126 --- ue4docker/exports/export_installed.py | 99 +++++++++++++++------------ 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py index 5c01b2c3..a5484081 100644 --- a/ue4docker/exports/export_installed.py +++ b/ue4docker/exports/export_installed.py @@ -1,60 +1,73 @@ +import tempfile + +from docker.models.containers import Container + from ..infrastructure import DockerUtils, SubprocessUtils import json, os, platform, shutil, subprocess, sys def exportInstalledBuild(image, destination, extraArgs): - - # Verify that we are running under Linux - if platform.system() != "Linux": - print( - "Error: Installed Builds can only be exported under Linux.", file=sys.stderr - ) - sys.exit(1) - # Verify that the destination directory does not already exist if os.path.exists(destination) == True: print("Error: the destination directory already exists.", file=sys.stderr) sys.exit(1) - # Verify that the Installed Build in the specified image is at least 4.21.0 - versionResult = SubprocessUtils.capture( - [ - "docker", - "run", - "--rm", - "-ti", - image, - "cat", - "/home/ue4/UnrealEngine/Engine/Build/Build.version", - ], - universal_newlines=True, - ) - try: - version = json.loads(versionResult.stdout) - if version["MajorVersion"] == 4 and version["MinorVersion"] < 21: - raise Exception() - except: - print( - "Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.", - file=sys.stderr, - ) - sys.exit(1) - # Create a container from which we will copy files container = DockerUtils.create(image) + exit_code = 1 + try: + exit_code = doExportInstalledBuild(container, destination, extraArgs) + except Exception as e: + print("Error: failed to export Installed Build.", file=sys.stderr) + raise e + finally: + # Remove the container, irrespective of whether or not the export succeeded + container.remove() + + sys.exit(exit_code) + + +def doExportInstalledBuild(container: Container, destination: str, extraArgs) -> int: + if platform.system() == "Windows": + engineRoot = "C:/UnrealEngine" + else: + engineRoot = "/home/ue4/UnrealEngine" + + with tempfile.TemporaryDirectory() as tmpdir: + versionFilePath = os.path.join(tmpdir, "Build.version") + # Verify that the Installed Build in the specified image is at least 4.21.0 + subprocess.run( + [ + "docker", + "cp", + f"{container.name}:{engineRoot}/Engine/Build/Build.version", + versionFilePath, + ], + check=True, + ) + try: + with open(versionFilePath, "r") as versionFile: + version = json.load(versionFile) + if version["MajorVersion"] == 4 and version["MinorVersion"] < 21: + raise Exception() + except: + print( + "Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.", + file=sys.stderr, + ) + return 1 + # Attempt to perform the export print("Exporting to {}...".format(destination)) - containerPath = "{}:/home/ue4/UnrealEngine".format(container.name) - exportResult = subprocess.call(["docker", "cp", containerPath, destination]) - - # Remove the container, irrespective of whether or not the export succeeded - container.remove() + subprocess.run( + ["docker", "cp", f"{container.name}:{engineRoot}", destination], check=True + ) # If the export succeeded, regenerate the linker symlinks on the host system - if exportResult == 0: + if platform.system() != "Windows": print("Performing linker symlink fixup...") - subprocess.call( + subprocess.run( [ sys.executable, os.path.join( @@ -69,10 +82,6 @@ def exportInstalledBuild(image, destination, extraArgs): "Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64", ), shutil.which("ld"), - ] + ], + check=True, ) - - # Report any failures - if exportResult != 0: - print("Error: failed to export Installed Build.", file=sys.stderr) - sys.exit(1) From 46664f81abf5edae4e6c765b7802d7c384829396 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 13 Dec 2021 21:32:33 +0300 Subject: [PATCH 218/430] Allow passing custom options to git clone via --opts=clone_opts= --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index e46d2930..73bab9b9 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -38,7 +38,7 @@ RUN chmod +x /tmp/git-credential-helper-secrets.sh ARG CHANGELIST RUN --mount=type=secret,id=username,uid=1000,required \ --mount=type=secret,id=password,uid=1000,required \ - CHANGELIST="$CHANGELIST" git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine + CHANGELIST="$CHANGELIST" git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO {{ clone_opts }} /home/ue4/UnrealEngine {% else %} @@ -56,7 +56,7 @@ ENV GIT_ASKPASS=/tmp/git-credential-helper-endpoint.sh RUN chmod +x /tmp/git-credential-helper-endpoint.sh # Clone the UE4 git repository using the endpoint-supplied credentials -RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine +RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO {{ clone_opts }} /home/ue4/UnrealEngine {% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 337117b2..c4de71fa 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -40,7 +40,7 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat # Clone the UE4 git repository using the host-supplied credentials WORKDIR C:\ -RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine +RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% {{ clone_opts }} C:\UnrealEngine {% endif %} From 7f575f98a4430fd888bf52de044a7df0e8385a13 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 15 Dec 2021 12:22:38 +1000 Subject: [PATCH 219/430] Add functionality to automatically parse CL number from git commit --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 6 +-- .../ue4-minimal/linux/set-changelist.py | 49 ++++++++++++++++-- .../ue4-minimal/windows/Dockerfile | 6 +-- .../ue4-minimal/windows/set-changelist.py | 51 +++++++++++++++++-- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index b08e5f07..eb7b2bab 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -7,14 +7,14 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder {% endif %} -# Remove the .git directory to disable UBT `git status` calls and speed up the build process -RUN rm -rf /home/ue4/UnrealEngine/.git - # Set the changelist number in Build.version to ensure our Build ID is generated correctly ARG CHANGELIST COPY set-changelist.py /tmp/set-changelist.py RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version $CHANGELIST +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN rm -rf /home/ue4/UnrealEngine/.git + {% if (not disable_all_patches) and (not disable_opengl_patch) %} # Enable the OpenGL RHI for Engine versions where it is present but deprecated COPY enable-opengl.py /tmp/enable-opengl.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py index ef0f52e1..c902526b 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -import json, sys +from os.path import dirname +from subprocess import run, PIPE +import json, re, sys def readFile(filename): @@ -12,9 +14,50 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided +# Determine whether a changelist override value was specified +changelistOverride = None +if len(sys.argv) > 2: + + # If the override was "auto" then attempt to retrieve the CL number from the git commit message + if sys.argv[2] == "auto": + + # Retrieve the commit message from git + engineRoot = dirname(dirname(dirname(sys.argv[1]))) + commitMessage = run( + ["git", "log", "-n", "1", "--format=%s%n%b"], + cwd=engineRoot, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + ).stdout.strip() + + # If the commit is a tagged engine release then it won't have a CL number, and using "auto" is user error + if re.fullmatch("[0-9\\.]+ release", commitMessage) is not None: + print( + "Error: you are attempting to automatically retrieve the CL number for a tagged Unreal Engine release.\n" + "For hotfix releases of the Unreal Engine, a CL override is not required and should not be specified.\n" + "For supported .0 releases of the Unreal Engine, ue4-docker ships with known CL numbers, so an override should not be necessary.", + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to extract the CL number from the commit message + match = re.search("\\[CL ([0-9]+) by .+ in .+ branch\\]", commitMessage) + if match is not None: + changelistOverride = int(match.group(1)) + else: + print( + "Error: failed to find a CL number in the git commit message! This was the commit message:\n\n" + + commitMessage, + file=sys.stderr, + ) + sys.exit(1) + + else: + changelistOverride = int(sys.argv[2]) + +# Update the `Changelist` field to reflect the override if it was supplied, or else the `CompatibleChangelist` field in our version file versionFile = sys.argv[1] -changelistOverride = int(sys.argv[2]) if len(sys.argv) > 2 else None details = json.loads(readFile(versionFile)) details["Changelist"] = ( changelistOverride diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 0ea15765..41c22bfa 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -8,14 +8,14 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder {% endif %} -# Remove the .git directory to disable UBT `git status` calls and speed up the build process -RUN rmdir /s /q C:\UnrealEngine\.git - # Set the changelist number in Build.version to ensure our Build ID is generated correctly ARG CHANGELIST COPY set-changelist.py C:\set-changelist.py RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN rmdir /s /q C:\UnrealEngine\.git + {% if (not disable_all_patches) and (not disable_buildgraph_patches) %} # Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 COPY patch-filters-xml.py C:\patch-filters-xml.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index 72734f1f..7c772393 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -import json, sys +from os.path import dirname +from subprocess import run, PIPE +import json, re, sys def readFile(filename): @@ -12,11 +14,50 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file, unless a specific value was provided +# Determine whether a changelist override value was specified +changelistOverride = None +if len(sys.argv) > 2 and sys.argv[2] != "%CHANGELIST%": + + # If the override was "auto" then attempt to retrieve the CL number from the git commit message + if sys.argv[2] == "auto": + + # Retrieve the commit message from git + engineRoot = dirname(dirname(dirname(sys.argv[1]))) + commitMessage = run( + ["git", "log", "-n", "1", "--format=%s%n%b"], + cwd=engineRoot, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + ).stdout.strip() + + # If the commit is a tagged engine release then it won't have a CL number, and using "auto" is user error + if re.fullmatch("[0-9\\.]+ release", commitMessage) is not None: + print( + "Error: you are attempting to automatically retrieve the CL number for a tagged Unreal Engine release.\n" + "For hotfix releases of the Unreal Engine, a CL override is not required and should not be specified.\n" + "For supported .0 releases of the Unreal Engine, ue4-docker ships with known CL numbers, so an override should not be necessary.", + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to extract the CL number from the commit message + match = re.search("\\[CL ([0-9]+) by .+ in .+ branch\\]", commitMessage) + if match is not None: + changelistOverride = int(match.group(1)) + else: + print( + "Error: failed to find a CL number in the git commit message! This was the commit message:\n\n" + + commitMessage, + file=sys.stderr, + ) + sys.exit(1) + + else: + changelistOverride = int(sys.argv[2]) + +# Update the `Changelist` field to reflect the override if it was supplied, or else the `CompatibleChangelist` field in our version file versionFile = sys.argv[1] -changelistOverride = ( - int(sys.argv[2]) if len(sys.argv) > 2 and sys.argv[2] != "%CHANGELIST%" else None -) details = json.loads(readFile(versionFile)) details["Changelist"] = ( changelistOverride From 69de21ad15706184a89f00951e23f5059bfbebde Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 15 Dec 2021 12:28:37 +1000 Subject: [PATCH 220/430] Add the ability to clone a specific commit of the Unreal Engine --- .../dockerfiles/ue4-source/linux/Dockerfile | 17 ++++++++++++++--- .../dockerfiles/ue4-source/windows/Dockerfile | 9 +++++++-- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 73bab9b9..6ce382f6 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -22,7 +22,7 @@ COPY ${SOURCE_LOCATION} /home/ue4/UnrealEngine # The git repository that we will clone ARG GIT_REPO="" -# The git branch/tag that we will checkout +# The git branch/tag/commit that we will checkout ARG GIT_BRANCH="" {% if credential_mode == "secrets" %} @@ -38,7 +38,13 @@ RUN chmod +x /tmp/git-credential-helper-secrets.sh ARG CHANGELIST RUN --mount=type=secret,id=username,uid=1000,required \ --mount=type=secret,id=password,uid=1000,required \ - CHANGELIST="$CHANGELIST" git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO {{ clone_opts }} /home/ue4/UnrealEngine + CHANGELIST="$CHANGELIST" \ + mkdir /home/ue4/UnrealEngine && \ + cd /home/ue4/UnrealEngine && \ + git init && \ + git remote add origin "$GIT_REPO" && \ + git fetch --progress --depth 1 {{ clone_opts }} origin "$GIT_BRANCH" && \ + git checkout FETCH_HEAD {% else %} @@ -56,7 +62,12 @@ ENV GIT_ASKPASS=/tmp/git-credential-helper-endpoint.sh RUN chmod +x /tmp/git-credential-helper-endpoint.sh # Clone the UE4 git repository using the endpoint-supplied credentials -RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO {{ clone_opts }} /home/ue4/UnrealEngine +RUN mkdir /home/ue4/UnrealEngine && \ + cd /home/ue4/UnrealEngine && \ + git init && \ + git remote add origin "$GIT_REPO" && \ + git fetch --progress --depth 1 {{ clone_opts }} origin "$GIT_BRANCH" && \ + git checkout FETCH_HEAD {% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index c4de71fa..43377ce1 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -23,7 +23,7 @@ COPY ${SOURCE_LOCATION} C:\UnrealEngine # The git repository that we will clone ARG GIT_REPO="" -# The git branch/tag that we will checkout +# The git branch/tag/commit that we will checkout ARG GIT_BRANCH="" # Retrieve the address for the host that will supply git credentials @@ -40,7 +40,12 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat # Clone the UE4 git repository using the host-supplied credentials WORKDIR C:\ -RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% {{ clone_opts }} C:\UnrealEngine +RUN mkdir C:\UnrealEngine && \ + cd C:\UnrealEngine && \ + git init && \ + git remote add origin %GIT_REPO% && \ + git fetch --progress --depth 1 {{ clone_opts }} origin %GIT_BRANCH% && \ + git checkout FETCH_HEAD {% endif %} From 4db4b3b3adb010ec733fb322eea17b82b020153d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 15 Dec 2021 15:47:33 +1000 Subject: [PATCH 221/430] Add ability to specify git config options --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 14 ++++++++++++-- .../dockerfiles/ue4-source/windows/Dockerfile | 7 ++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 6ce382f6..83389431 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -42,8 +42,13 @@ RUN --mount=type=secret,id=username,uid=1000,required \ mkdir /home/ue4/UnrealEngine && \ cd /home/ue4/UnrealEngine && \ git init && \ + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && \ + {% endfor %} + {% endif %} git remote add origin "$GIT_REPO" && \ - git fetch --progress --depth 1 {{ clone_opts }} origin "$GIT_BRANCH" && \ + git fetch --progress --depth 1 {{ fetch_opts }} origin "$GIT_BRANCH" && \ git checkout FETCH_HEAD {% else %} @@ -65,8 +70,13 @@ RUN chmod +x /tmp/git-credential-helper-endpoint.sh RUN mkdir /home/ue4/UnrealEngine && \ cd /home/ue4/UnrealEngine && \ git init && \ + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && \ + {% endfor %} + {% endif %} git remote add origin "$GIT_REPO" && \ - git fetch --progress --depth 1 {{ clone_opts }} origin "$GIT_BRANCH" && \ + git fetch --progress --depth 1 {{ fetch_opts }} origin "$GIT_BRANCH" && \ git checkout FETCH_HEAD {% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 43377ce1..25c6b101 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -43,8 +43,13 @@ WORKDIR C:\ RUN mkdir C:\UnrealEngine && \ cd C:\UnrealEngine && \ git init && \ + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && \ + {% endfor %} + {% endif %} git remote add origin %GIT_REPO% && \ - git fetch --progress --depth 1 {{ clone_opts }} origin %GIT_BRANCH% && \ + git fetch --progress --depth 1 {{ fetch_opts }} origin %GIT_BRANCH% && \ git checkout FETCH_HEAD {% endif %} From 9ba09bca96f6d1e9d3d3ea2b215f94a5142a8afb Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 15 Dec 2021 15:55:39 +1000 Subject: [PATCH 222/430] Remove option to pass extra flags to `git fetch` --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 4 ++-- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 83389431..c95badf3 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -48,7 +48,7 @@ RUN --mount=type=secret,id=username,uid=1000,required \ {% endfor %} {% endif %} git remote add origin "$GIT_REPO" && \ - git fetch --progress --depth 1 {{ fetch_opts }} origin "$GIT_BRANCH" && \ + git fetch --progress --depth 1 origin "$GIT_BRANCH" && \ git checkout FETCH_HEAD {% else %} @@ -76,7 +76,7 @@ RUN mkdir /home/ue4/UnrealEngine && \ {% endfor %} {% endif %} git remote add origin "$GIT_REPO" && \ - git fetch --progress --depth 1 {{ fetch_opts }} origin "$GIT_BRANCH" && \ + git fetch --progress --depth 1 origin "$GIT_BRANCH" && \ git checkout FETCH_HEAD {% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 25c6b101..62ee9dc5 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -49,7 +49,7 @@ RUN mkdir C:\UnrealEngine && \ {% endfor %} {% endif %} git remote add origin %GIT_REPO% && \ - git fetch --progress --depth 1 {{ fetch_opts }} origin %GIT_BRANCH% && \ + git fetch --progress --depth 1 origin %GIT_BRANCH% && \ git checkout FETCH_HEAD {% endif %} From 28905e7a8e59209159f3beed47047f5a44a332db Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 15 Dec 2021 16:00:50 +1000 Subject: [PATCH 223/430] Bump version to 0.0.94 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index a31af385..cffae176 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.93" +__version__ = "0.0.94" From 812e0c0b6ff7cfe5711335e89e13fb1fa582fd44 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 17 Dec 2021 21:19:17 +0300 Subject: [PATCH 224/430] Fix multiline directives in ue4-source for Windows Broken in 69de21ad15706184a89f00951e23f5059bfbebde --- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 62ee9dc5..9a8f2ce4 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -40,16 +40,16 @@ ENV GIT_ASKPASS=C:\git-credential-helper.bat # Clone the UE4 git repository using the host-supplied credentials WORKDIR C:\ -RUN mkdir C:\UnrealEngine && \ - cd C:\UnrealEngine && \ - git init && \ +RUN mkdir C:\UnrealEngine && ` + cd C:\UnrealEngine && ` + git init && ` {% if git_config %} {% for key, value in git_config.items() %} - git config {{ key }} {{ value }} && \ + git config {{ key }} {{ value }} && ` {% endfor %} {% endif %} - git remote add origin %GIT_REPO% && \ - git fetch --progress --depth 1 origin %GIT_BRANCH% && \ + git remote add origin %GIT_REPO% && ` + git fetch --progress --depth 1 origin %GIT_BRANCH% && ` git checkout FETCH_HEAD {% endif %} From 63af15e334c13fbe8c831828c4616369754b11f8 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 5 Jan 2022 16:54:39 +0300 Subject: [PATCH 225/430] Bump ver5sion to 0.0.95 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index cffae176..e5ca841d 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.94" +__version__ = "0.0.95" From 4ad926620fb7ee86dbaa2b80c22a78ecdf5e5287 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 7 Jan 2022 14:57:52 +0300 Subject: [PATCH 226/430] resolves #217 add CPU model to `ue4-docker info` --- ue4docker/info.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ue4docker/info.py b/ue4docker/info.py index ceb57459..18a1763e 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -47,6 +47,7 @@ def info(): memVirtual = psutil.swap_memory().total cpuPhysical = psutil.cpu_count(False) cpuLogical = psutil.cpu_count() + cpuModel = platform.processor() # Attempt to query PyPI to determine the latest version of ue4-docker # (We ignore any errors here to ensure the `ue4-docker info` command still works without network access) @@ -81,8 +82,8 @@ def info(): ), ), ( - "Number of processors", - "{} physical, {} logical".format(cpuPhysical, cpuLogical), + "CPU", + "{} physical, {} logical ({})".format(cpuPhysical, cpuLogical, cpuModel), ), ] From 03d935d6fbda29f5f43446027a147b0695d387ea Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 22 Jan 2022 01:45:27 +0300 Subject: [PATCH 227/430] Fix one more occurrence of curl --progress This is a follow-up to 3907f45f87f3e3458e16770f9bc45f76fc5490d4 --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index d862879b..1f141671 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -21,7 +21,7 @@ expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\Windows\System32\ && ^ expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\Windows\System32\ || goto :error @rem Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) -curl --progress -L "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" --output %TEMP%\dxc.zip && ^ +curl --progress-bar -L "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" --output %TEMP%\dxc.zip && ^ powershell -Command "Expand-Archive -Path \"$env:TEMP\dxc.zip\" -DestinationPath $env:TEMP" && ^ xcopy /y %TEMP%\bin\x64\dxcompiler.dll C:\Windows\System32\ && ^ xcopy /y %TEMP%\bin\x64\dxil.dll C:\Windows\System32\ || goto :error From 9feba63f825d31ac4978d0798b9a48ad539ec840 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 22 Jan 2022 01:46:19 +0300 Subject: [PATCH 228/430] Bump version to 0.0.96 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index e5ca841d..5dce8eb0 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.95" +__version__ = "0.0.96" From 31a6a31524d31d357d305fcd0ccb147e5d462097 Mon Sep 17 00:00:00 2001 From: Russ Webber Date: Tue, 15 Feb 2022 13:40:23 +1100 Subject: [PATCH 229/430] add ubuntu 20.04 cudagl image Part of the ticket Create UE4 Docker Image on Ubuntu Focal [sc-3279] --- ue4docker/infrastructure/BuildConfiguration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 33b2d954..b4183c9c 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -20,11 +20,12 @@ "10.0": "nvidia/cudagl:10.0-devel-ubuntu18.04", "10.1": "nvidia/cudagl:10.1-devel-ubuntu18.04", "10.2": "nvidia/cudagl:10.2-devel-ubuntu18.04", + "11.4": "nvidia/cudagl:11.4.2-devel-ubuntu20.04", }, } # The default CUDA version to use when `--cuda` is specified without a value -DEFAULT_CUDA_VERSION = "9.2" +DEFAULT_CUDA_VERSION = "11.4" # The default memory limit (in GB) under Windows DEFAULT_MEMORY_LIMIT = 10.0 From 8b010d6bef935a80e356d4ab18986d18bb8d5a80 Mon Sep 17 00:00:00 2001 From: Russ Webber Date: Wed, 16 Feb 2022 10:09:56 +1100 Subject: [PATCH 230/430] feat: use the `basetag` argument to select ubuntu base. --- .../infrastructure/BuildConfiguration.py | 22 +++++++++++++------ 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index b4183c9c..509b4528 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -14,16 +14,19 @@ # The base images for Linux containers LINUX_BASE_IMAGES = { - "opengl": "nvidia/opengl:1.0-glvnd-devel-ubuntu18.04", + "opengl": "nvidia/opengl:1.0-glvnd-devel-{ubuntu}", "cudagl": { - "9.2": "nvidia/cudagl:9.2-devel-ubuntu18.04", - "10.0": "nvidia/cudagl:10.0-devel-ubuntu18.04", - "10.1": "nvidia/cudagl:10.1-devel-ubuntu18.04", - "10.2": "nvidia/cudagl:10.2-devel-ubuntu18.04", - "11.4": "nvidia/cudagl:11.4.2-devel-ubuntu20.04", + "9.2": "nvidia/cudagl:9.2-devel-{ubuntu}", + "10.0": "nvidia/cudagl:10.0-devel-{ubuntu}", + "10.1": "nvidia/cudagl:10.1-devel-{ubuntu}", + "10.2": "nvidia/cudagl:10.2-devel-{ubuntu}", + "11.4": "nvidia/cudagl:11.4.2-devel-{ubuntu}", }, } +# The default ubuntu base to use +DEFAULT_LINUX_VERSION = "ubuntu18.04" + # The default CUDA version to use when `--cuda` is specified without a value DEFAULT_CUDA_VERSION = "11.4" @@ -189,7 +192,8 @@ def addArguments(parser): parser.add_argument( "-basetag", default=None, - help="Windows Server Core base image tag to use for Windows containers (default is the host OS version)", + help="Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu18.04). " \ + "For Windows this is the Windows Server Core base image tag (default is the host OS version)", ) parser.add_argument( "-suffix", default="", help="Add a suffix to the tags of the built images" @@ -515,6 +519,8 @@ def _generateLinuxConfig(self): if self.suffix.startswith("opengl") or self.suffix.startswith("cudagl"): raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') + self.args.basetag = self.args.basetag if self.args.basetag is not None else DEFAULT_LINUX_VERSION + # Determine if we are building CUDA-enabled container images self.cuda = None if self.args.cuda is not None: @@ -535,6 +541,8 @@ def _generateLinuxConfig(self): self.baseImage = LINUX_BASE_IMAGES["opengl"] self.prereqsTag = "opengl" + self.baseImage = self.baseImage.format(ubuntu=self.args.basetag) + def _processPackageVersion(self, package, version): # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified From 94dbefd34c2c055df7144954cfbaf7e33c10bdef Mon Sep 17 00:00:00 2001 From: Harry Horsley Date: Fri, 25 Feb 2022 01:16:06 +1100 Subject: [PATCH 231/430] Add missing dependency for WebBrowserWidget plugin under Linux (#223) --- ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index cc92cbc8..322dc5ce 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -49,6 +49,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libsm6 \ libxcomposite1 \ libxcursor1 \ + libxdamage1 \ libxi6 \ libxrandr2 \ libxrender1 \ From 06ced1c745d4600a0f8fb9be055092e98ca09a05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 21:14:18 +0000 Subject: [PATCH 232/430] Bump actions/setup-python from 2 to 3 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 3. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb3bde99..512a9fbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: 3.7 - name: Install dependencies From 753266c7eab01788cc74050f89befade37bae2d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Mar 2022 07:36:37 +0300 Subject: [PATCH 233/430] Bump actions/checkout from 2 to 3 (#226) Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 512a9fbb..c1e75eaf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,13 +4,13 @@ jobs: lint: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: psf/black@stable build: runs-on: ubuntu-20.04 steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: From d59e9c8a25a636a8abf8d48f12a0b08dda8d62d4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 4 Mar 2022 18:45:42 +0300 Subject: [PATCH 234/430] Use DOCKER_BUILDKIT=1 env variable instead of `docker buildx` `docker buildx` is problematic because it isn't shipped with Docker client by default and needs to be installed separately --- ue4docker/infrastructure/DockerUtils.py | 2 +- ue4docker/infrastructure/ImageBuilder.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index acb56044..1f1ce461 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -64,7 +64,7 @@ def buildx(tags, context, args, secrets): """ tagArgs = [["-t", tag] for tag in tags] return ( - ["docker", "buildx", "build"] + ["docker", "build"] + list(itertools.chain.from_iterable(tagArgs)) + [context] + ["--progress=plain"] diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 19284961..1253cf50 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,3 +1,5 @@ +from typing import Optional + from .DockerUtils import DockerUtils from .FilesystemUtils import FilesystemUtils from .GlobalConfiguration import GlobalConfiguration @@ -69,6 +71,7 @@ def build(self, name, tags, args, secrets=None): # Determine whether we are building using `docker buildx` with build secrets imageTags = self._formatTags(name, tags) command = DockerUtils.build(imageTags, self.context(name), args) + env = None if self.platform == "linux" and secrets is not None and len(secrets) > 0: # Create temporary files to store the contents of each of our secrets @@ -82,9 +85,10 @@ def build(self, name, tags, args, secrets=None): command = DockerUtils.buildx( imageTags, self.context(name), args, secretFlags ) + env = {"DOCKER_BUILDKIT": "1"} # Build the image if it doesn't already exist - self._processImage(imageTags[0], name, command, "build", "built") + self._processImage(imageTags[0], name, command, "build", "built", env=env) def context(self, name): """ @@ -119,7 +123,15 @@ def _willProcess(self, image): """ return self.rebuild == True or DockerUtils.exists(image) == False - def _processImage(self, image, name, command, actionPresentTense, actionPastTense): + def _processImage( + self, + image: str, + name: Optional[str], + command: [str], + actionPresentTense: str, + actionPastTense: str, + env: Optional[dict[str, str]] = None, + ): """ Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) """ @@ -202,7 +214,7 @@ def _processImage(self, image, name, command, actionPresentTense, actionPastTens # Attempt to process the image using the supplied command startTime = time.time() - exitCode = subprocess.call(command) + exitCode = subprocess.call(command, env=env) endTime = time.time() # Determine if processing succeeded From c0406a5b5333f39564b9538ccc20d75a064fa851 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 22 Mar 2022 18:33:34 +0300 Subject: [PATCH 235/430] Use BuildKit for all images (#229) BuildKit offers much shorter build times --- ue4docker/infrastructure/ImageBuilder.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 1253cf50..a7534e1b 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -71,7 +71,7 @@ def build(self, name, tags, args, secrets=None): # Determine whether we are building using `docker buildx` with build secrets imageTags = self._formatTags(name, tags) command = DockerUtils.build(imageTags, self.context(name), args) - env = None + if self.platform == "linux" and secrets is not None and len(secrets) > 0: # Create temporary files to store the contents of each of our secrets @@ -85,10 +85,16 @@ def build(self, name, tags, args, secrets=None): command = DockerUtils.buildx( imageTags, self.context(name), args, secretFlags ) - env = {"DOCKER_BUILDKIT": "1"} # Build the image if it doesn't already exist - self._processImage(imageTags[0], name, command, "build", "built", env=env) + self._processImage( + imageTags[0], + name, + command, + "build", + "built", + env={"DOCKER_BUILDKIT": "1"}, + ) def context(self, name): """ From acba48725e3c091d6d9b87c93ce5a6b19e77d54b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 25 Mar 2022 12:55:03 +0300 Subject: [PATCH 236/430] Passthrough environment variables to Docker subprocesses This fixes `exec: "com.docker.cli": executable file not found in $PATH% error in certain environments Broken by d59e9c8a25a636a8abf8d48f12a0b08dda8d62d4 --- ue4docker/infrastructure/ImageBuilder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index a7534e1b..9de55963 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -93,7 +93,7 @@ def build(self, name, tags, args, secrets=None): command, "build", "built", - env={"DOCKER_BUILDKIT": "1"}, + env=dict(os.environ, DOCKER_BUILDKIT="1"), ) def context(self, name): From 9c1bd1e5a06014130b318e153ef284435d53a44e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 25 Mar 2022 18:58:49 +0300 Subject: [PATCH 237/430] Fix "invalid windows mount type: 'bind'" error on Windows It turns out, Docker _tries_ to use BuildKit on Windows but fails miserably. --- ue4docker/infrastructure/ImageBuilder.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 9de55963..f3f131ed 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -86,6 +86,10 @@ def build(self, name, tags, args, secrets=None): imageTags, self.context(name), args, secretFlags ) + env = os.environ.copy() + if self.platform == "linux": + env["DOCKER_BUILDKIT"] = "1" + # Build the image if it doesn't already exist self._processImage( imageTags[0], @@ -93,7 +97,7 @@ def build(self, name, tags, args, secrets=None): command, "build", "built", - env=dict(os.environ, DOCKER_BUILDKIT="1"), + env=env, ) def context(self, name): From 47c81a4480fbd06b5cda425ff214bcb628c0083b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 26 Mar 2022 15:18:31 +0300 Subject: [PATCH 238/430] resolves #231 require setuptools >= 60.4.0 to fix installation path of `ue4-docker.exe` shim on Windows See pypa/setuptools#3001 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index de848fa9..935508f3 100644 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ "psutil", "requests", "semver>=2.7.9,<3.0.0", - "setuptools>=38.6.0", + "setuptools>=60.4.0", "termcolor", "twine>=1.11.0", "wheel>=0.31.0", From 4e05201725af5ca585903a23db627875d5f060c6 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 4 Apr 2022 15:35:07 +1000 Subject: [PATCH 239/430] Fix #232: explicitly use Hyper-V isolation mode when not using process isolation mode under Windows --- ue4docker/infrastructure/BuildConfiguration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index ef11cd69..bbd51396 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -474,7 +474,7 @@ def _generateWindowsConfig(self): self.isolation = self.args.isolation else: - # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode + # If we are able to use process isolation mode then use it, otherwise use Hyper-V isolation mode differentKernels = self.basetag != self.hostBasetag dockerSupportsProcess = parse_version( DockerUtils.version()["Version"] @@ -482,7 +482,7 @@ def _generateWindowsConfig(self): if not differentKernels and dockerSupportsProcess: self.isolation = "process" else: - self.isolation = DockerUtils.info()["Isolation"] + self.isolation = "hyperv" # Set the isolation mode Docker flag self.platformArgs.append("--isolation=" + self.isolation) From 1e8c2b167faa3d3c00285203cb81ce50497a2f0b Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Thu, 21 Oct 2021 03:09:20 -0500 Subject: [PATCH 240/430] Implement --target to replace the old --no-* options. --- ue4docker/build.py | 164 +++++------ .../infrastructure/BuildConfiguration.py | 257 +++++++++++++----- ue4docker/main.py | 2 +- 3 files changed, 274 insertions(+), 149 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 4a1f3a1a..383bcd73 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -105,11 +105,12 @@ def build(): config.combine, ) - # Resolve our main set of tags for the generated images - mainTags = [ - "{}{}-{}".format(config.release, config.suffix, config.prereqsTag), - config.release + config.suffix, - ] + # Resolve our main set of tags for the generated images; this is used only for Source and downstream + if config.buildTargetSource: + mainTags = [ + "{}{}-{}".format(config.release, config.suffix, config.prereqsTag), + config.release + config.suffix, + ] # Print the command-line invocation that triggered this build, masking any supplied passwords args = [ @@ -128,10 +129,11 @@ def build(): ) if config.custom == True: logger.info("Custom name: " + config.release, False) - else: + elif config.release is not None: logger.info("Release: " + config.release, False) - logger.info("Repository: " + config.repository, False) - logger.info("Branch/tag: " + config.branch + "\n", False) + if config.repository is not None: + logger.info("Repository: " + config.repository, False) + logger.info("Branch/tag: " + config.branch + "\n", False) # Determine if we are using a custom version for ue4cli or conan-ue4cli if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: @@ -241,7 +243,7 @@ def build(): # Ensure the Docker daemon is configured correctly requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit: + if DockerUtils.maxsize() < requiredLimit and config.buildTargetSource: logger.error("SETUP REQUIRED:") logger.error( "The max image size for Windows containers must be set to at least {}GB.".format( @@ -314,7 +316,10 @@ def build(): username = "" password = "" - elif builder.willBuild("ue4-source", mainTags) == False: + elif ( + not config.buildTargetSource + or builder.willBuild("ue4-source", mainTags) == False + ): # Don't bother prompting the user for any credentials if we're not building the ue4-source image logger.info( @@ -338,11 +343,8 @@ def build(): if config.args.monitor == True: resourceMonitor.start() - # Start the HTTP credential endpoint as a child process and wait for it to start + # Prep for endpoint cleanup, if necessary endpoint = None - if config.opts.get("credential_mode", "endpoint") == "endpoint": - endpoint = CredentialEndpoint(username, password) - endpoint.start() try: @@ -363,58 +365,71 @@ def build(): "NAMESPACE={}".format(GlobalConfiguration.getTagNamespace()), ] - # Compute the build options for the UE4 build prerequisites image - # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) - prereqsArgs = ["--build-arg", "BASEIMAGE=" + config.baseImage] - if config.containerPlatform == "windows": - prereqsArgs = prereqsArgs + [ - "--build-arg", - "DLLSRCIMAGE=" + config.dllSrcImage, - "--build-arg", - "VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber, - ] - # Build the UE4 build prerequisites image - builder.build( - "ue4-build-prerequisites", - [config.prereqsTag], - commonArgs + config.platformArgs + prereqsArgs, - ) - builtImages.append("ue4-build-prerequisites") + if config.buildTargetPrerequisites: + # Compute the build options for the UE4 build prerequisites image + # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) + prereqsArgs = ["--build-arg", "BASEIMAGE=" + config.baseImage] + if config.containerPlatform == "windows": + prereqsArgs = prereqsArgs + [ + "--build-arg", + "DLLSRCIMAGE=" + config.dllSrcImage, + "--build-arg", + "VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber, + ] - # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets - secrets = {} - if config.opts.get("credential_mode", "endpoint") == "secrets": - secrets = {"username": username, "password": password} + builder.build( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + ) + builtImages.append("ue4-build-prerequisites") + + prereqConsumerArgs = [ + "--build-arg", + "PREREQS_TAG={}".format(config.prereqsTag), + ] + else: + logger.info("Skipping ue4-build-prerequisities image build.") # Build the UE4 source image - prereqConsumerArgs = [ - "--build-arg", - "PREREQS_TAG={}".format(config.prereqsTag), - ] - credentialArgs = [] if len(secrets) > 0 else endpoint.args() - ue4SourceArgs = prereqConsumerArgs + [ - "--build-arg", - "GIT_REPO={}".format(config.repository), - "--build-arg", - "GIT_BRANCH={}".format(config.branch), - "--build-arg", - "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), - ] - builder.build( - "ue4-source", - mainTags, - commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, - secrets, - ) - builtImages.append("ue4-source") + if config.buildTargetSource: + # Start the HTTP credential endpoint as a child process and wait for it to start + if config.opts.get("credential_mode", "endpoint") == "endpoint": + endpoint = CredentialEndpoint(username, password) + endpoint.start() + + # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets + secrets = {} + if config.opts.get("credential_mode", "endpoint") == "secrets": + secrets = {"username": username, "password": password} + credentialArgs = [] if len(secrets) > 0 else endpoint.args() + + ue4SourceArgs = prereqConsumerArgs + [ + "--build-arg", + "GIT_REPO={}".format(config.repository), + "--build-arg", + "GIT_BRANCH={}".format(config.branch), + "--build-arg", + "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), + ] + builder.build( + "ue4-source", + mainTags, + commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, + secrets, + ) + builtImages.append("ue4-source") + else: + logger.info("Skipping ue4-source image build.") # Build the UE4 Engine source build image, unless requested otherwise by the user - ue4BuildArgs = prereqConsumerArgs + [ - "--build-arg", - "TAG={}".format(mainTags[1]), - ] - if config.noEngine == False: + if config.buildTargetEngine: + ue4BuildArgs = prereqConsumerArgs + [ + "--build-arg", + "TAG={}".format(mainTags[1]), + ] + builder.build( "ue4-engine", mainTags, @@ -422,18 +437,16 @@ def build(): ) builtImages.append("ue4-engine") else: - logger.info( - "User specified `--no-engine`, skipping ue4-engine image build." - ) + logger.info("Skipping ue4-engine image build.") # Build the minimal UE4 CI image, unless requested otherwise by the user - minimalArgs = ( - ["--build-arg", "CHANGELIST={}".format(config.changelist)] - if config.changelist is not None - else [] - ) - buildUe4Minimal = config.noMinimal == False - if buildUe4Minimal == True: + if config.buildTargetMinimal == True: + minimalArgs = ( + ["--build-arg", "CHANGELIST={}".format(config.changelist)] + if config.changelist is not None + else [] + ) + builder.build( "ue4-minimal", mainTags, @@ -441,13 +454,10 @@ def build(): ) builtImages.append("ue4-minimal") else: - logger.info( - "User specified `--no-minimal`, skipping ue4-minimal image build." - ) + logger.info("Skipping ue4-minimal image build.") # Build the full UE4 CI image, unless requested otherwise by the user - buildUe4Full = buildUe4Minimal == True and config.noFull == False - if buildUe4Full == True: + if config.buildTargetFull: # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them infrastructureFlags = [] @@ -477,9 +487,7 @@ def build(): ) builtImages.append("ue4-full") else: - logger.info( - "Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build." - ) + logger.info("Skipping ue4-full image build.") # If we are generating Dockerfiles then include information about the options used to generate them if config.layoutDir is not None: diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index bbd51396..ef95a5c3 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -99,6 +99,12 @@ def addArguments(parser): """ parser.add_argument( "release", + nargs="?", # aka "required = False", but that doesn't work in positionals + help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)', + ) + parser.add_argument( + "--ue-version", + default=None, help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch', ) parser.add_argument( @@ -117,19 +123,28 @@ def addArguments(parser): help="Print `docker build` commands instead of running them", ) parser.add_argument( - "--no-engine", action="store_true", help="Don't build the ue4-engine image" + "--no-engine", + action="store_true", + help="Don't build the ue4-engine image (deprecated, use --target instead)", ) parser.add_argument( "--no-minimal", action="store_true", - help="Don't build the ue4-minimal image", + help="Don't build the ue4-minimal image (deprecated, use --target instead)", ) parser.add_argument( - "--no-full", action="store_true", help="Don't build the ue4-full image" + "--no-full", + action="store_true", + help="Don't build the ue4-full image (deprecated, use --target instead)", ) parser.add_argument( "--no-cache", action="store_true", help="Disable Docker build cache" ) + parser.add_argument( + "--target", + action="append", + help="Add a target to the build list. Valid targets are `build-prerequisites`, `source`, `engine`, `minimal`, `full`, and `all`. May be specified multiple times or comma-separated. Defaults to `all`.", + ) parser.add_argument( "--random-memory", action="store_true", @@ -264,60 +279,163 @@ def __init__(self, parser, argv): self.args = parser.parse_args(argv) self.changelist = self.args.changelist - # Determine if we are building a custom version of UE4 rather than an official release - self.args.release = self.args.release.lower() - if self.args.release == "custom" or self.args.release.startswith("custom:"): + # Figure out what targets we have; this is needed to find out if we need --ue-version. + using_target_specifier_old = ( + self.args.no_engine or self.args.no_minimal or self.args.no_full + ) + using_target_specifier_new = self.args.target is not None - # Both a custom repository and a custom branch/tag must be specified - if self.args.repo is None or self.args.branch is None: - raise RuntimeError( - "both a repository and branch/tag must be specified when building a custom version of the Engine" - ) + # If we specified nothing, it's the same as specifying `all` + if not using_target_specifier_old and not using_target_specifier_new: + self.args.target = ["all"] + elif using_target_specifier_old and not using_target_specifier_new: + # Convert these to the new style + + # no-minimal implies no-full + if self.args.no_minimal: + self.args.no_full = True + + # Change into target descriptors + self.args.target = [] + + if not self.args.no_full: + self.args.target += ["full"] + + if not self.args.no_minimal: + self.args.target += ["minimal"] + + if not self.args.no_engine: + self.args.target += ["engine"] + + # disabling these was never supported + self.args.target += ["source"] + self.args.target += ["build-prerequisites"] - # Use the specified repository and branch/tag - customName = ( - self.args.release.split(":", 2)[1].strip() - if ":" in self.args.release - else "" + elif using_target_specifier_new and not using_target_specifier_old: + # these can be token-delimited, so let's just split them apart and then remerge them into one list + split = [item.split(",") for item in self.args.target] + self.args.target = [item for sublist in split for item in sublist] + + elif using_target_specifier_old and using_target_specifier_new: + # uhoh + raise RuntimeError( + "specified both `--target` and the old `--no-*` options; please use only `--target`!" ) - self.release = customName if len(customName) > 0 else "custom" - self.repository = self.args.repo - self.branch = self.args.branch - self.custom = True - else: + # Now that we have our options in `self.args.target`, evaluate our dependencies + # In a theoretical ideal world this should be code-driven; if you find yourself adding a lot more code to this, consider a redesign! + # Adding things to the target list while parsing is weird and feels dangerously superlinear, + # but given how few things exist, it is unlikely to be a perf problem before it gets redesigned out anyway. - # Validate the specified version string - try: - ue4Version = semver.parse(self.args.release) - if ( - ue4Version["major"] not in [4, 5] - or ue4Version["prerelease"] != None - ): - raise Exception() - self.release = semver.format_version( - ue4Version["major"], ue4Version["minor"], ue4Version["patch"] - ) - except: - raise RuntimeError( - 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.20.0")'.format( - self.args.release + # build-prereq -> source -> engine + # build-prereq -> source -> minimal -> full + + self.buildTargetPrerequisites = False + self.buildTargetSource = False + self.buildTargetEngine = False + self.buildTargetMinimal = False + self.buildTargetFull = False + + if "full" in self.args.target or "all" in self.args.target: + self.buildTargetFull = True + self.args.target += ["minimal"] + + if "minimal" in self.args.target or "all" in self.args.target: + self.buildTargetMinimal = True + self.args.target += ["source"] + + if "engine" in self.args.target or "all" in self.args.target: + self.buildTargetEngine = True + self.args.target += ["source"] + + if "source" in self.args.target or "all" in self.args.target: + self.buildTargetSource = True + self.args.target += ["build-prerequisites"] + + if "build-prerequisites" in self.args.target or "all" in self.args.target: + self.buildTargetPrerequisites = True + + if not self.buildTargetPrerequisites: + raise RuntimeError( + "we're not building anything; this shouldn't even be possible, but is definitely not useful" + ) + + # See if the user specified both the old positional version option and the new ue-version option + if self.args.release is not None and self.args.ue_version is not None: + raise RuntimeError( + "specified both `--ue-version` and the old positional version option; please use only `--ue-version`!" + ) + + # For the sake of a simpler pull request, we use self.args.release as the canonical place for this data. + # If support for the old positional version option is removed, this should be fixed. + if self.args.ue_version is not None: + self.args.release = self.args.ue_version + + # We care about the version number only if we're building source + if self.buildTargetSource: + if self.args.release is None: + raise RuntimeError("missing `--ue-version` when building source") + + # Determine if we are building a custom version of UE4 rather than an official release + self.args.release = self.args.release.lower() + if self.args.release == "custom" or self.args.release.startswith("custom:"): + + # Both a custom repository and a custom branch/tag must be specified + if self.args.repo is None or self.args.branch is None: + raise RuntimeError( + "both a repository and branch/tag must be specified when building a custom version of the Engine" ) + + # Use the specified repository and branch/tag + customName = ( + self.args.release.split(":", 2)[1].strip() + if ":" in self.args.release + else "" ) + self.release = customName if len(customName) > 0 else "custom" + self.repository = self.args.repo + self.branch = self.args.branch + self.custom = True - # Use the default repository and the release tag for the specified version - self.repository = DEFAULT_GIT_REPO - self.branch = "{}-release".format(self.release) - self.custom = False + else: - # If the user specified a .0 release of the Unreal Engine and did not specify a changelist override then - # use the official changelist number for that release to ensure consistency with Epic Games Launcher builds - # (This is necessary because .0 releases do not include a `CompatibleChangelist` value in Build.version) - if ( - self.changelist is None - and self.release in UNREAL_ENGINE_RELEASE_CHANGELISTS - ): - self.changelist = UNREAL_ENGINE_RELEASE_CHANGELISTS[self.release] + # Validate the specified version string + try: + ue4Version = semver.parse(self.args.release) + if ( + ue4Version["major"] not in [4, 5] + or ue4Version["prerelease"] != None + ): + raise Exception() + self.release = semver.format_version( + ue4Version["major"], ue4Version["minor"], ue4Version["patch"] + ) + except: + raise RuntimeError( + 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.20.0")'.format( + self.args.release + ) + ) + + # Use the default repository and the release tag for the specified version + self.repository = DEFAULT_GIT_REPO + self.branch = "{}-release".format(self.release) + self.custom = False + + # If the user specified a .0 release of the Unreal Engine and did not specify a changelist override then + # use the official changelist number for that release to ensure consistency with Epic Games Launcher builds + # (This is necessary because .0 releases do not include a `CompatibleChangelist` value in Build.version) + if ( + self.changelist is None + and self.release in UNREAL_ENGINE_RELEASE_CHANGELISTS + ): + self.changelist = UNREAL_ENGINE_RELEASE_CHANGELISTS[self.release] + else: + # defaults needed by other parts of the codebase + self.custom = False + self.release = None + self.repository = None + self.branch = None # Store our common configuration settings self.containerPlatform = ( @@ -327,9 +445,6 @@ def __init__(self, parser, argv): ) self.dryRun = self.args.dry_run self.rebuild = self.args.rebuild - self.noEngine = self.args.no_engine - self.noMinimal = self.args.no_minimal - self.noFull = self.args.no_full self.suffix = self.args.suffix self.platformArgs = ["--no-cache"] if self.args.no_cache == True else [] self.excludedComponents = set(self.args.exclude) @@ -373,27 +488,29 @@ def __init__(self, parser, argv): "the `-layout` flag must be used when specifying the `--combine` flag" ) - # Verify that the value for `source_mode` is valid if specified - validSourceModes = ["git", "copy"] - if self.opts.get("source_mode", "git") not in validSourceModes: - raise RuntimeError( - "invalid value specified for the `source_mode` option, valid values are {}".format( - validSourceModes + # We care about source_mode and credential_mode only if we're building source + if self.buildTargetSource: + # Verify that the value for `source_mode` is valid if specified + validSourceModes = ["git", "copy"] + if self.opts.get("source_mode", "git") not in validSourceModes: + raise RuntimeError( + "invalid value specified for the `source_mode` option, valid values are {}".format( + validSourceModes + ) ) - ) - # Verify that the value for `credential_mode` is valid if specified - validCredentialModes = ( - ["endpoint", "secrets"] - if self.containerPlatform == "linux" - else ["endpoint"] - ) - if self.opts.get("credential_mode", "endpoint") not in validCredentialModes: - raise RuntimeError( - "invalid value specified for the `credential_mode` option, valid values are {} when building {} containers".format( - validCredentialModes, self.containerPlatform.title() - ) + # Verify that the value for `credential_mode` is valid if specified + validCredentialModes = ( + ["endpoint", "secrets"] + if self.containerPlatform == "linux" + else ["endpoint"] ) + if self.opts.get("credential_mode", "endpoint") not in validCredentialModes: + raise RuntimeError( + "invalid value specified for the `credential_mode` option, valid values are {} when building {} containers".format( + validCredentialModes, self.containerPlatform.title() + ) + ) # Generate Jinja context values for keeping or excluding components self.opts["excluded_components"] = { @@ -427,7 +544,7 @@ def describeExcludedComponents(self): def _generateWindowsConfig(self): self.visualStudio = self.args.visual_studio - if not self.custom: + if self.release is not None and not self.custom: # Check whether specified Unreal Engine release is compatible with specified Visual Studio vsMinSupportedUnreal = VisualStudio.MinSupportedUnreal.get( self.visualStudio, None diff --git a/ue4docker/main.py b/ue4docker/main.py index 47534dd7..8432b883 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -66,7 +66,7 @@ def main(): COMMANDS = { "build": { "function": build, - "description": "Builds container images for a specific version of UE4", + "description": "Builds container images for UE4", }, "clean": {"function": clean, "description": "Cleans built container images"}, "diagnostics": { From 8bce4dbac5345d989c3b25736c3c247608ea5677 Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Tue, 5 Apr 2022 05:47:25 -0500 Subject: [PATCH 241/430] Fix: ue4BuildArgs not initialized in all necessary paths. --- ue4docker/build.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 383bcd73..2038b8d1 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -423,8 +423,7 @@ def build(): else: logger.info("Skipping ue4-source image build.") - # Build the UE4 Engine source build image, unless requested otherwise by the user - if config.buildTargetEngine: + if config.buildTargetEngine or config.buildTargetMinimal: ue4BuildArgs = prereqConsumerArgs + [ "--build-arg", "TAG={}".format(mainTags[1]), From c02fd1fdd6b2e95f3338e9059cea5ef170b0f22a Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Tue, 5 Apr 2022 05:47:33 -0500 Subject: [PATCH 242/430] Fix: Unnecessary comparison. --- ue4docker/build.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ue4docker/build.py b/ue4docker/build.py index 2038b8d1..9206de1e 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -429,6 +429,8 @@ def build(): "TAG={}".format(mainTags[1]), ] + # Build the UE4 Engine source build image, unless requested otherwise by the user + if config.buildTargetEngine: builder.build( "ue4-engine", mainTags, From 9871fb081506e7f59876391f7de2e7877a8b35eb Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 5 Apr 2022 20:54:17 +0300 Subject: [PATCH 243/430] Improve ue4-build-prerequisites on Linux (#222) This is a follow-up to adamrehn/ue4-docker#221 This commit: 1. Removes hardcoded list of CUDA images. Now user can specify whatever version they want, and we do not need to maintain this list 2. Adds base OS name to ue4-build-prerequisites tag for clarity 3. Prints base image tag in `ue4 build` header output --- ue4docker/build.py | 11 +----- .../infrastructure/BuildConfiguration.py | 36 ++++++------------- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 9206de1e..15ff3758 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -264,18 +264,9 @@ def build(): sys.exit(1) elif config.containerPlatform == "linux": - - # Determine if we are building CUDA-enabled container images - capabilities = ( - "CUDA {} + OpenGL".format(config.cuda) - if config.cuda is not None - else "OpenGL" - ) logger.info("LINUX CONTAINER SETTINGS", False) logger.info( - "Building GPU-enabled images compatible with NVIDIA Docker ({} support).\n".format( - capabilities - ), + "Base OS image: {}\n".format(config.baseImage), False, ) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index ef95a5c3..3d22eaf7 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -15,13 +15,7 @@ # The base images for Linux containers LINUX_BASE_IMAGES = { "opengl": "nvidia/opengl:1.0-glvnd-devel-{ubuntu}", - "cudagl": { - "9.2": "nvidia/cudagl:9.2-devel-{ubuntu}", - "10.0": "nvidia/cudagl:10.0-devel-{ubuntu}", - "10.1": "nvidia/cudagl:10.1-devel-{ubuntu}", - "10.2": "nvidia/cudagl:10.2-devel-{ubuntu}", - "11.4": "nvidia/cudagl:11.4.2-devel-{ubuntu}", - }, + "cudagl": "nvidia/cudagl:{cuda}-devel-{ubuntu}", } # The default ubuntu base to use @@ -206,7 +200,7 @@ def addArguments(parser): ) parser.add_argument( "-basetag", - default=None, + default=None if platform.system() == "Windows" else DEFAULT_LINUX_VERSION, help="Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu18.04). " "For Windows this is the Windows Server Core base image tag (default is the host OS version)", ) @@ -636,33 +630,25 @@ def _generateLinuxConfig(self): if self.suffix.startswith("opengl") or self.suffix.startswith("cudagl"): raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') - self.args.basetag = ( - self.args.basetag - if self.args.basetag is not None - else DEFAULT_LINUX_VERSION - ) - # Determine if we are building CUDA-enabled container images self.cuda = None if self.args.cuda is not None: # Verify that the specified CUDA version is valid self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION - if self.cuda not in LINUX_BASE_IMAGES["cudagl"]: - raise RuntimeError( - 'unsupported CUDA version "{}", supported versions are: {}'.format( - self.cuda, ", ".join([v for v in LINUX_BASE_IMAGES["cudagl"]]) - ) - ) - # Use the appropriate base image for the specified CUDA version - self.baseImage = LINUX_BASE_IMAGES["cudagl"][self.cuda] - self.prereqsTag = "cudagl{}".format(self.cuda) + self.baseImage = LINUX_BASE_IMAGES["cudagl"] + self.prereqsTag = "cudagl{cuda}-{ubuntu}" else: self.baseImage = LINUX_BASE_IMAGES["opengl"] - self.prereqsTag = "opengl" + self.prereqsTag = "opengl-{ubuntu}" - self.baseImage = self.baseImage.format(ubuntu=self.args.basetag) + self.baseImage = self.baseImage.format( + cuda=self.args.cuda, ubuntu=self.args.basetag + ) + self.prereqsTag = self.prereqsTag.format( + cuda=self.args.cuda, ubuntu=self.args.basetag + ) def _processPackageVersion(self, package, version): From b32e0457fadd0910dc3107e233943d8662f863e3 Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Tue, 5 Apr 2022 13:09:44 -0500 Subject: [PATCH 244/430] Move config.buildTarget* into a dict (#235) --- ue4docker/build.py | 18 +++++------ .../infrastructure/BuildConfiguration.py | 30 +++++++++++-------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 15ff3758..660b793e 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -106,7 +106,7 @@ def build(): ) # Resolve our main set of tags for the generated images; this is used only for Source and downstream - if config.buildTargetSource: + if config.buildTargets["source"]: mainTags = [ "{}{}-{}".format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix, @@ -243,7 +243,7 @@ def build(): # Ensure the Docker daemon is configured correctly requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit and config.buildTargetSource: + if DockerUtils.maxsize() < requiredLimit and config.buildTargets["source"]: logger.error("SETUP REQUIRED:") logger.error( "The max image size for Windows containers must be set to at least {}GB.".format( @@ -308,7 +308,7 @@ def build(): password = "" elif ( - not config.buildTargetSource + not config.buildTargets["source"] or builder.willBuild("ue4-source", mainTags) == False ): @@ -357,7 +357,7 @@ def build(): ] # Build the UE4 build prerequisites image - if config.buildTargetPrerequisites: + if config.buildTargets["build-prerequisites"]: # Compute the build options for the UE4 build prerequisites image # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) prereqsArgs = ["--build-arg", "BASEIMAGE=" + config.baseImage] @@ -384,7 +384,7 @@ def build(): logger.info("Skipping ue4-build-prerequisities image build.") # Build the UE4 source image - if config.buildTargetSource: + if config.buildTargets["source"]: # Start the HTTP credential endpoint as a child process and wait for it to start if config.opts.get("credential_mode", "endpoint") == "endpoint": endpoint = CredentialEndpoint(username, password) @@ -414,14 +414,14 @@ def build(): else: logger.info("Skipping ue4-source image build.") - if config.buildTargetEngine or config.buildTargetMinimal: + if config.buildTargets["engine"] or config.buildTargets["minimal"]: ue4BuildArgs = prereqConsumerArgs + [ "--build-arg", "TAG={}".format(mainTags[1]), ] # Build the UE4 Engine source build image, unless requested otherwise by the user - if config.buildTargetEngine: + if config.buildTargets["engine"]: builder.build( "ue4-engine", mainTags, @@ -432,7 +432,7 @@ def build(): logger.info("Skipping ue4-engine image build.") # Build the minimal UE4 CI image, unless requested otherwise by the user - if config.buildTargetMinimal == True: + if config.buildTargets["minimal"]: minimalArgs = ( ["--build-arg", "CHANGELIST={}".format(config.changelist)] if config.changelist is not None @@ -449,7 +449,7 @@ def build(): logger.info("Skipping ue4-minimal image build.") # Build the full UE4 CI image, unless requested otherwise by the user - if config.buildTargetFull: + if config.buildTargets["full"]: # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them infrastructureFlags = [] diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 3d22eaf7..30d26df9 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -324,32 +324,36 @@ def __init__(self, parser, argv): # build-prereq -> source -> engine # build-prereq -> source -> minimal -> full - self.buildTargetPrerequisites = False - self.buildTargetSource = False - self.buildTargetEngine = False - self.buildTargetMinimal = False - self.buildTargetFull = False + # We initialize these with all the options, with the intent that you should be accessing them directly and not checking for existence + # This is to avoid typos giving false-negatives; KeyError is reliable and tells you what you did wrong + self.buildTargets = { + "build-prerequisites": False, + "source": False, + "engine": False, + "minimal": False, + "full": False, + } if "full" in self.args.target or "all" in self.args.target: - self.buildTargetFull = True + self.buildTargets["full"] = True self.args.target += ["minimal"] if "minimal" in self.args.target or "all" in self.args.target: - self.buildTargetMinimal = True + self.buildTargets["minimal"] = True self.args.target += ["source"] if "engine" in self.args.target or "all" in self.args.target: - self.buildTargetEngine = True + self.buildTargets["engine"] = True self.args.target += ["source"] if "source" in self.args.target or "all" in self.args.target: - self.buildTargetSource = True + self.buildTargets["source"] = True self.args.target += ["build-prerequisites"] if "build-prerequisites" in self.args.target or "all" in self.args.target: - self.buildTargetPrerequisites = True + self.buildTargets["build-prerequisites"] = True - if not self.buildTargetPrerequisites: + if not self.buildTargets["build-prerequisites"]: raise RuntimeError( "we're not building anything; this shouldn't even be possible, but is definitely not useful" ) @@ -366,7 +370,7 @@ def __init__(self, parser, argv): self.args.release = self.args.ue_version # We care about the version number only if we're building source - if self.buildTargetSource: + if self.buildTargets["source"]: if self.args.release is None: raise RuntimeError("missing `--ue-version` when building source") @@ -483,7 +487,7 @@ def __init__(self, parser, argv): ) # We care about source_mode and credential_mode only if we're building source - if self.buildTargetSource: + if self.buildTargets["source"]: # Verify that the value for `source_mode` is valid if specified validSourceModes = ["git", "copy"] if self.opts.get("source_mode", "git") not in validSourceModes: From 2d97e2239d104ae426c1b258ee644a58400c1ede Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Wed, 6 Apr 2022 03:54:45 -0500 Subject: [PATCH 245/430] Add build target status output on startup (#237) --- ue4docker/build.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ue4docker/build.py b/ue4docker/build.py index 660b793e..7759e51c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -272,6 +272,20 @@ def build(): # Report which Engine components are being excluded (if any) logger.info("GENERAL SETTINGS", False) + logger.info( + "Build targets: {}".format( + " ".join( + sorted( + [ + target + for target, enabled in config.buildTargets.items() + if enabled + ] + ) + ) + ), + False, + ) logger.info( "Changelist override: {}".format( config.changelist From 84a6f6bd292be913ff163852279495961fc4f272 Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Wed, 6 Apr 2022 05:35:42 -0500 Subject: [PATCH 246/430] Do dependency analysis with a set (#238) --- .../infrastructure/BuildConfiguration.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 30d26df9..5a6a1c0c 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -318,8 +318,7 @@ def __init__(self, parser, argv): # Now that we have our options in `self.args.target`, evaluate our dependencies # In a theoretical ideal world this should be code-driven; if you find yourself adding a lot more code to this, consider a redesign! - # Adding things to the target list while parsing is weird and feels dangerously superlinear, - # but given how few things exist, it is unlikely to be a perf problem before it gets redesigned out anyway. + active_targets = set(self.args.target) # build-prereq -> source -> engine # build-prereq -> source -> minimal -> full @@ -334,23 +333,23 @@ def __init__(self, parser, argv): "full": False, } - if "full" in self.args.target or "all" in self.args.target: + if "full" in active_targets or "all" in active_targets: self.buildTargets["full"] = True - self.args.target += ["minimal"] + active_targets.add("minimal") - if "minimal" in self.args.target or "all" in self.args.target: + if "minimal" in active_targets or "all" in active_targets: self.buildTargets["minimal"] = True - self.args.target += ["source"] + active_targets.add("source") - if "engine" in self.args.target or "all" in self.args.target: + if "engine" in active_targets or "all" in active_targets: self.buildTargets["engine"] = True - self.args.target += ["source"] + active_targets.add("source") - if "source" in self.args.target or "all" in self.args.target: + if "source" in active_targets or "all" in active_targets: self.buildTargets["source"] = True - self.args.target += ["build-prerequisites"] + active_targets.add("build-prerequisites") - if "build-prerequisites" in self.args.target or "all" in self.args.target: + if "build-prerequisites" in active_targets or "all" in active_targets: self.buildTargets["build-prerequisites"] = True if not self.buildTargets["build-prerequisites"]: From 5e9fb636c9bc9e293b6fb65847d79f86dda3ad5c Mon Sep 17 00:00:00 2001 From: Ben Rog-Wilhelm Date: Wed, 6 Apr 2022 06:16:33 -0500 Subject: [PATCH 247/430] Add error on invalid target name (#239) --- ue4docker/infrastructure/BuildConfiguration.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 5a6a1c0c..356f8dd0 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -333,6 +333,13 @@ def __init__(self, parser, argv): "full": False, } + for target in active_targets: + if target != "all" and target not in self.buildTargets: + valid_options = sorted(self.buildTargets.keys()) + raise RuntimeError( + f"unknown build target '{target}', valid options are: all {' '.join(valid_options)}" + ) + if "full" in active_targets or "all" in active_targets: self.buildTargets["full"] = True active_targets.add("minimal") From edcedca7149272b0620b1cde148edd8a584eca70 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 13 Apr 2022 22:59:34 +1000 Subject: [PATCH 248/430] UE5 compat: add DLLs required by OpenCV under Windows --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 245f06d1..09d0a7bd 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -17,7 +17,9 @@ COPY --from=dlls ` C:\Windows\System32\dsound.dll ` C:\Windows\System32\dxva2.dll ` C:\Windows\System32\glu32.dll ` + C:\Windows\System32\ksuser.dll ` C:\Windows\System32\mf.dll ` + C:\Windows\System32\mfcore.dll ` C:\Windows\System32\mfplat.dll ` C:\Windows\System32\mfplay.dll ` C:\Windows\System32\mfreadwrite.dll ` From 6db04e5dcdabaf5d803b7f61199573fdd05f53fc Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Wed, 13 Apr 2022 23:14:41 +1000 Subject: [PATCH 249/430] Default to Windows Server 2022 base tag on Windows 11 hosts --- ue4docker/infrastructure/WindowsUtils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 84dd3fa7..22e387b6 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -13,7 +13,9 @@ class WindowsUtils(object): _minimumRequiredBuild = 17763 # This lookup table is based on the list of valid tags from - # and list of build-to-release mapping from https://docs.microsoft.com/en-us/windows/release-health/release-information + # and list of build-to-release mappings from the following pages: + # - https://docs.microsoft.com/en-us/windows/release-health/release-information + # - https://docs.microsoft.com/en-us/windows/release-health/windows11-release-information _knownTagsByBuildNumber = { 17763: "ltsc2019", 18362: "1903", @@ -22,6 +24,7 @@ class WindowsUtils(object): 19042: "20H2", 19043: "21H1", 20348: "ltsc2022", + 22000: "ltsc2022", } _knownTags = list(_knownTagsByBuildNumber.values()) From d58dcc885383016b3770cc7c17079a2077e48a41 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 13 Apr 2022 19:18:31 +0300 Subject: [PATCH 250/430] Drop dependency on Flask (#240) This commit rewrites credentials endpoint using http.server Python module Resolves #233 --- setup.py | 1 - .../infrastructure/CredentialEndpoint.py | 63 +++++++++++-------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index 935508f3..7c4dd932 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,6 @@ "colorama", "container-utils", "docker>=3.0.0", - "flask", "humanfriendly", "Jinja2>=2.11.3", "packaging>=19.1", diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/ue4docker/infrastructure/CredentialEndpoint.py index 87156324..6c9dcf66 100644 --- a/ue4docker/infrastructure/CredentialEndpoint.py +++ b/ue4docker/infrastructure/CredentialEndpoint.py @@ -1,10 +1,37 @@ -import logging, multiprocessing, os, platform, secrets, time, urllib.parse +import multiprocessing, secrets, time, urllib.parse from .NetworkUtils import NetworkUtils -from flask import Flask, request +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import urlparse, parse_qs +from functools import partial + + +class CredentialRequestHandler(BaseHTTPRequestHandler): + def __init__(self, username: str, password: str, token: str, *args, **kwargs): + self.username = username + self.password = password + self.token = token + super().__init__(*args, **kwargs) + + def log_request(self, code: str = "-", size: str = "-") -> None: + # We do not want to log each and every incoming request + pass + + def do_POST(self): + query_components = parse_qs(urlparse(self.path).query) + + self.send_response(200) + self.end_headers() + + if "token" in query_components and query_components["token"][0] == self.token: + content_length = int(self.headers["Content-Length"]) + prompt = self.rfile.read(content_length).decode("utf-8") + + response = self.password if "Password for" in prompt else self.username + self.wfile.write(response.encode("utf-8")) class CredentialEndpoint(object): - def __init__(self, username, password): + def __init__(self, username: str, password: str): """ Creates an endpoint manager for the supplied credentials """ @@ -17,7 +44,7 @@ def __init__(self, username, password): # Generate a security token to require when requesting credentials self.token = secrets.token_hex(16) - def args(self): + def args(self) -> [str]: """ Returns the Docker build arguments for creating containers that require Git credentials """ @@ -33,7 +60,7 @@ def args(self): "HOST_TOKEN_ARG=" + urllib.parse.quote_plus(self.token), ] - def start(self): + def start(self) -> None: """ Starts the HTTP endpoint as a child process """ @@ -49,10 +76,10 @@ def start(self): time.sleep(2) # Verify that the endpoint started correctly - if self.endpoint.is_alive() == False: + if not self.endpoint.is_alive(): raise RuntimeError("failed to start the credential endpoint") - def stop(self): + def stop(self) -> None: """ Stops the HTTP endpoint child process """ @@ -60,25 +87,11 @@ def stop(self): self.endpoint.join() @staticmethod - def _endpoint(username, password, token): + def _endpoint(username: str, password: str, token: str) -> None: """ Implements a HTTP endpoint to provide Git credentials to Docker containers """ - server = Flask(__name__) - - # Disable the first-run banner message - os.environ["WERKZEUG_RUN_MAIN"] = "true" - - # Disable Flask log output (from ) - log = logging.getLogger("werkzeug") - log.setLevel(logging.ERROR) - - @server.route("/", methods=["POST"]) - def credentials(): - if "token" in request.args and request.args["token"] == token: - prompt = request.data.decode("utf-8") - return password if "Password for" in prompt else username - else: - return "Invalid security token" + handler = partial(CredentialRequestHandler, username, password, token) - server.run(host="0.0.0.0", port=9876) + server = HTTPServer(("0.0.0.0", 9876), RequestHandlerClass=handler) + server.serve_forever() From ea48f8b2384a15637962227bd782fd717df86cbf Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 13 Apr 2022 20:23:32 +0300 Subject: [PATCH 251/430] Bump version to 0.0.97 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 5dce8eb0..625f000d 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.96" +__version__ = "0.0.97" From a6f15bf4daf037dd0323fee87aa25151ca8a6299 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 14 Apr 2022 16:41:58 +0300 Subject: [PATCH 252/430] Multiple improvements to BuildGraph options 1. Disable LinuxArm64 platform by default on UE-5.0 We already disabled LinuxAArch64, but it was renamed to LinuxArm64, so our code stopped doing its job 2. Allow passing custom flags to BuildGraph so users can control what platforms and targets are built (for example, they might want to disable building of Win64Server) --- .../dockerfiles/ue4-minimal/linux/patch-build-graph.py | 7 ++++++- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 7 +++++-- .../dockerfiles/ue4-minimal/windows/patch-build-graph.py | 3 ++- ue4docker/infrastructure/BuildConfiguration.py | 7 ++----- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py index f281b800..3bf2ddf1 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py @@ -26,11 +26,16 @@ def writeFile(filename, data): 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ', ) -# Disable building for AArch64 by default (enabled by default since 4.24.0) +# Disable AArch64 by default (enabled since 4.24.0) code = code.replace( 'Property Name="DefaultWithLinuxAArch64" Value="true"', 'Property Name="DefaultWithLinuxAArch64" Value="false"', ) +# AArch64 was renamed to Arm64 in UE-5.0, so also disable it +code = code.replace( + 'Property Name="DefaultWithLinuxArm64" Value="true"', + 'Property Name="DefaultWithLinuxArm64" Value="false"', +) # Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail # (See for details of the bug and its fix) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 41c22bfa..eb65de5e 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -27,9 +27,12 @@ RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineB {% endif %} # Create an Installed Build of the Engine -ARG BUILDGRAPH_ARGS WORKDIR C:\UnrealEngine -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && ` +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` + -target="Make Installed Build Win64" ` + -script=Engine/Build/InstalledEngineBuild.xml ` + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` + {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py index 06de3a2a..ae2de692 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py @@ -16,7 +16,8 @@ def writeFile(filename, data): buildXml = sys.argv[1] code = readFile(buildXml) -# Prevent HoloLens support from being enabled by default under Windows in 4.23.0 onwards +# Disable HoloLens by default on 4.23. +# On later versions, it is guarded by DefaultWithPlatform and disabled because of HostPlatformOnly=true code = code.replace( 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"', diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 356f8dd0..02b401d2 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -466,7 +466,7 @@ def __init__(self, parser, argv): ) # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) - self.opts = {} + self.opts = {"buildgraph_args": "-set:HostPlatformOnly=true"} for o in self.args.opt: if "=" in o: key, value = o.split("=", 1) @@ -568,10 +568,7 @@ def _generateWindowsConfig(self): # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. # Otherwise, UAT errors out with "Unknown argument: VS2019". if self.visualStudio != VisualStudio.VS2017: - self.opts["buildgraph_args"] = ( - self.opts.get("buildgraph_args", "") - + f" -set:VS{self.visualStudio}=true" - ) + self.opts["buildgraph_args"] += f" -set:VS{self.visualStudio}=true" # Determine base tag for the Windows release of the host system self.hostBasetag = WindowsUtils.getHostBaseTag() From 64ea750cabed9a8873af53accde1948417d6d932 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 14 Apr 2022 23:47:48 +1000 Subject: [PATCH 253/430] Fix #241: avoid copying DLL files that already exist under Windows (#242) --- .../ue4-build-prerequisites/windows/Dockerfile | 18 ++++++++++++++++-- .../windows/remove-duplicate-dlls.ps1 | 11 +++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 09d0a7bd..a0e37518 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -4,9 +4,14 @@ ARG DLLSRCIMAGE FROM ${DLLSRCIMAGE} as dlls -FROM ${BASEIMAGE} as prerequisites +FROM ${BASEIMAGE} as deduplication SHELL ["cmd", "/S", "/C"] +{% if not disable_labels %} +# Add a sentinel label so we can easily identify all derived images, including intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} + # Gather the system DLLs that we need from the full Windows base image COPY --from=dlls ` C:\Windows\System32\avicap32.dll ` @@ -29,7 +34,16 @@ COPY --from=dlls ` C:\Windows\System32\opengl32.dll ` C:\Windows\System32\ResampleDMO.dll ` C:\Windows\System32\ResourcePolicyClient.dll ` - C:\Windows\System32\ + C:\GatheredDLLs\ + +# Remove any DLL files that already exist in the target base image, to avoid permission errors when attempting to overwrite existing files with a COPY directive +COPY remove-duplicate-dlls.ps1 C:\remove-duplicate-dlls.ps1 +RUN powershell -ExecutionPolicy Bypass -File C:\remove-duplicate-dlls.ps1 + +# Copy the DLL files into a clean image +FROM ${BASEIMAGE} as prerequisites +SHELL ["cmd", "/S", "/C"] +COPY --from=deduplication C:\GatheredDlls\ C:\Windows\System32\ {% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 new file mode 100644 index 00000000..47a1ad00 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 @@ -0,0 +1,11 @@ +$dlls = (Get-ChildItem "C:\GatheredDLLs\*.dll") +foreach ($dll in $dlls) +{ + $filename = $dll.Name + $existing = (Get-ChildItem "C:\Windows\System32\${filename}" -ErrorAction SilentlyContinue) + if ($existing) + { + [Console]::Error.WriteLine("${filename} already exists in System32 in the target base image, excluding it from the list of DLL files to copy.") + Remove-Item $dll + } +} From edac031165a228770a0c1623f3c59907badaa0e9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 14 Apr 2022 19:13:02 +0300 Subject: [PATCH 254/430] Fix duplicate `-set:HostPlatformOnly=true` on Linux This is a follow-up to a6f15bf4daf037dd0323fee87aa25151ca8a6299 --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index eb7b2bab..1e3696c7 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -37,7 +37,10 @@ RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \ +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ + -target="Make Installed Build Linux" \ + -script=Engine/Build/InstalledEngineBuild.xml \ + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers From 5547ec916b05faa28f9c41da13f8b61571b1ce87 Mon Sep 17 00:00:00 2001 From: Justin Lewis Salmon Date: Sun, 17 Apr 2022 07:35:56 +1000 Subject: [PATCH 255/430] fix: ensure copied source has correct ownership When using `-layout` and `--opt source_mode=copy` followed by `docker build` from inside a docker container (docker-in-docker) where the executing user is `root`, the copied source ends up being owned by `root` and the build fails during the `Setup.sh` phase due to permissions issues. This patch fixes the issue by ensuring the copied source is owned by the `ue4` user. --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index c95badf3..702a7044 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -15,7 +15,7 @@ ARG VERBOSE_OUTPUT=0 # Copy the Unreal Engine source code from the host system ARG SOURCE_LOCATION -COPY ${SOURCE_LOCATION} /home/ue4/UnrealEngine +COPY --chown=ue4:ue4 ${SOURCE_LOCATION} /home/ue4/UnrealEngine {% else %} From 0be149b2b27d85b9349a80d34c4e2e511c4b04dd Mon Sep 17 00:00:00 2001 From: Justin Lewis Salmon Date: Mon, 18 Apr 2022 11:54:37 +1000 Subject: [PATCH 256/430] fix: don't try to delete .git if it's not present When using `--opt source_mode=copy` with a custom `.dockerignore` that excludes the `.git` directory from the docker build context, the `rmdir .git` command fails with `The system cannot find the file specified.` since docker does not have that directory in context. This patch avoids this scenario with a simple existence check. --- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index eb65de5e..aaf5fa27 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -14,7 +14,7 @@ COPY set-changelist.py C:\set-changelist.py RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% # Remove the .git directory to disable UBT `git status` calls and speed up the build process -RUN rmdir /s /q C:\UnrealEngine\.git +RUN if exist C:\UnrealEngine\.git rmdir /s /q C:\UnrealEngine\.git {% if (not disable_all_patches) and (not disable_buildgraph_patches) %} # Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 From 4a25cf68903967f52b924d2a708cd3797d3cb92a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 25 Apr 2022 17:03:17 +0300 Subject: [PATCH 257/430] Add warning for --no-* target specifiers (#245) Closes #236 --- ue4docker/build.py | 2 +- ue4docker/infrastructure/BuildConfiguration.py | 6 +++++- ue4docker/infrastructure/Logger.py | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 7759e51c..c98b37a7 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -48,7 +48,7 @@ def build(): # Parse the supplied command-line arguments try: - config = BuildConfiguration(parser, sys.argv[1:]) + config = BuildConfiguration(parser, sys.argv[1:], logger) except RuntimeError as e: logger.error("Error: {}".format(e)) sys.exit(1) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 02b401d2..60bc0d10 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -261,7 +261,7 @@ def addArguments(parser): help="Set a specific changelist number in the Unreal Engine's Build.version file", ) - def __init__(self, parser, argv): + def __init__(self, parser, argv, logger): """ Creates a new build configuration based on the supplied arguments object """ @@ -284,6 +284,10 @@ def __init__(self, parser, argv): self.args.target = ["all"] elif using_target_specifier_old and not using_target_specifier_new: # Convert these to the new style + logger.warning( + "Using deprecated `--no-*` target specifiers; recommend changing to `--target`", + False, + ) # no-minimal implies no-full if self.args.no_minimal: diff --git a/ue4docker/infrastructure/Logger.py b/ue4docker/infrastructure/Logger.py index 512892e7..758aafc4 100644 --- a/ue4docker/infrastructure/Logger.py +++ b/ue4docker/infrastructure/Logger.py @@ -26,6 +26,12 @@ def info(self, output, newline=True): """ Prints information that does not pertain to an action or an error """ + self._print("green", output, newline) + + def warning(self, output, newline=True): + """ + Prints a warning (something that is not an error as of today, but might break in future releases) + """ self._print("yellow", output, newline) def _print(self, colour, output, newline): From 48be3f93c59c52a8c48ac2521660b37ac4cd1aa9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 27 Apr 2022 15:05:37 +0300 Subject: [PATCH 258/430] Fix Python 3.8 compatibility Resolves #246 --- ue4docker/infrastructure/ImageBuilder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index f3f131ed..609f6e93 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Dict, Optional from .DockerUtils import DockerUtils from .FilesystemUtils import FilesystemUtils @@ -140,7 +140,7 @@ def _processImage( command: [str], actionPresentTense: str, actionPastTense: str, - env: Optional[dict[str, str]] = None, + env: Optional[Dict[str, str]] = None, ): """ Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) From 5de5e07259d1c5cfc90fff02dc13028a5480e8df Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 27 Apr 2022 15:06:59 +0300 Subject: [PATCH 259/430] Bump version to 0.0.98 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 625f000d..12ca2072 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.97" +__version__ = "0.0.98" From 27ed2244759778d84b36eea3854e07466cf0d3c0 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 27 Apr 2022 18:52:36 +0300 Subject: [PATCH 260/430] Build ue4-build-prerequisites as a poor-man's testing on CI (#247) --- .github/workflows/ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1e75eaf..cc7740e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,3 +25,19 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + test: + strategy: + matrix: + os: [ windows-2019, ubuntu-20.04 ] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: 3.7 + - name: Install + run: pip install . --user + - name: Test + run: ue4-docker build --target=build-prerequisites From cd42602859dda4274b9ea26dffd34d9869c66ae7 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 12 May 2022 12:06:20 +0300 Subject: [PATCH 261/430] Extend CI test matrix with older supported environments (#252) --- .github/workflows/ci.yml | 12 +++++++----- setup.py | 8 +++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc7740e6..6032ebd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,19 +2,19 @@ name: CI on: [push, pull_request] jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: psf/black@stable build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v3 - name: Setup Python uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: "3.10" - name: Install dependencies run: pip install setuptools wheel - name: Build package @@ -28,7 +28,9 @@ jobs: test: strategy: matrix: - os: [ windows-2019, ubuntu-20.04 ] + # Test oldest and newest supported environments + os: [ ubuntu-18.04, ubuntu-latest, windows-2019, windows-latest ] + python: ["3.6", "3.10"] runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -36,7 +38,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 with: - python-version: 3.7 + python-version: ${{ matrix.python }} - name: Install run: pip install . --user - name: Test diff --git a/setup.py b/setup.py index 7c4dd932..da5fab4c 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,9 @@ "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Build Tools", "Environment :: Console", ], @@ -47,7 +50,10 @@ "psutil", "requests", "semver>=2.7.9,<3.0.0", - "setuptools>=60.4.0", + # We want newer setuptools to avoid shim path bug on newer Pythons. + # See #231 and https://github.com/pypa/setuptools/issues/3001 + # Unfortunately, that version of setuptools doesn't have Python 3.6 support + "setuptools>=60.4.0;python_version>='3.7.0'", "termcolor", "twine>=1.11.0", "wheel>=0.31.0", From 8ba1d5f8a873f76583a32f61e5399827193f0e11 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 14 Apr 2022 12:31:58 +0300 Subject: [PATCH 262/430] Stop pinning Python to 3.7 This was done back in 2019 as a temporary workaround for #53 As of today, we neither need this workaround nor use pywin32 at all --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 1f141671..2de7862d 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,8 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error @rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y choco-cleaner curl vcredist-all windows-sdk-10-version-1809-windbg || goto :error -choco install -y python --version=3.7.5 || goto :error +choco install -y choco-cleaner curl python vcredist-all windows-sdk-10-version-1809-windbg || goto :error @rem Reload our environment variables from the registry so the `git` command works call refreshenv From 55be232feb66ec2309dc21a42cba86b239584be2 Mon Sep 17 00:00:00 2001 From: jonpas Date: Sun, 15 May 2022 21:36:04 +0200 Subject: [PATCH 263/430] Add support for Visual Studio 2022 (#253) * Add support for Visual Studio 2022 * Default to VS2019 instead * Default back to VS2017 * Use Visual Studio 2022 installer * Build CI on Visual Studio 2017, 2019 and 2022 (matrix) * see #253 adjust .NET installation so it is compatible with VS2022 * Set minimal UE version to 4.27 for VS2022 Note that neither 4.26 nor 4.25 was actually tested. Also, as of today, latest VS2022 crashes even with UE 5/4.27: https://developercommunity.visualstudio.com/t/vs2022-1710-internal-compiler-error-when-building-1/1677086 Co-authored-by: Marat Radchenko --- .github/workflows/ci.yml | 22 +++++++++++++++---- .../windows/install-prerequisites.bat | 8 +++---- .../infrastructure/BuildConfiguration.py | 6 ++++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6032ebd7..0113818a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,23 @@ jobs: test: strategy: matrix: - # Test oldest and newest supported environments - os: [ ubuntu-18.04, ubuntu-latest, windows-2019, windows-latest ] - python: ["3.6", "3.10"] + include: + - { os: ubuntu-18.04, python: "3.6" } + - { os: ubuntu-18.04, python: "3.10" } + - { os: ubuntu-latest, python: "3.6" } + - { os: ubuntu-latest, python: "3.10" } + - { os: windows-2019, python: "3.6", visual-studio: 2017 } + - { os: windows-2019, python: "3.10", visual-studio: 2017 } + - { os: windows-2019, python: "3.6", visual-studio: 2019 } + - { os: windows-2019, python: "3.10", visual-studio: 2019 } + - { os: windows-2019, python: "3.6", visual-studio: 2022 } + - { os: windows-2019, python: "3.10", visual-studio: 2022 } + - { os: windows-latest, python: "3.6", visual-studio: 2017 } + - { os: windows-latest, python: "3.10", visual-studio: 2017 } + - { os: windows-latest, python: "3.6", visual-studio: 2019 } + - { os: windows-latest, python: "3.10", visual-studio: 2019 } + - { os: windows-latest, python: "3.6", visual-studio: 2022 } + - { os: windows-latest, python: "3.10", visual-studio: 2022 } runs-on: ${{ matrix.os }} steps: - name: Checkout @@ -42,4 +56,4 @@ jobs: - name: Install run: pip install . --user - name: Test - run: ue4-docker build --target=build-prerequisites + run: ue4-docker build --target=build-prerequisites --visual-studio ${{ matrix.visual-studio || '2017' }} diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 2de7862d..0f2401e1 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -56,10 +56,9 @@ if "%VISUAL_STUDIO_BUILD_NUMBER%" == "15" ( ) @rem Install the Visual Studio Build Tools workloads and components we need -@rem NOTE: We use the Visual Studio 2019 installer even for Visual Studio 2017 here because the old installer now breaks -@rem NOTE: VS2019 Build Tools doesn't have 4.6.2 .NET SDK and what actually gets installed is 4.8 +@rem NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks @rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019. And it is actually *needed* only for UE5 -curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error +curl --progress-bar -L "https://aka.ms/vs/17/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ --channelUri "https://aka.ms/vs/%VISUAL_STUDIO_BUILD_NUMBER%/release/channel" ^ @@ -73,7 +72,8 @@ curl --progress-bar -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ --add Microsoft.VisualStudio.Component.Windows10SDK.%WINDOWS_SDK_VERSION% ^ --add Microsoft.Net.Component.4.5.TargetingPack ^ - --add Microsoft.Net.ComponentGroup.4.6.2.DeveloperTools ^ + --add Microsoft.Net.Component.4.6.2.TargetingPack ^ + --add Microsoft.Net.ComponentGroup.DevelopmentPrerequisites ^ --add Microsoft.NetCore.Component.SDK python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 60bc0d10..0f1a0dd8 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -43,10 +43,12 @@ class VisualStudio(object): VS2017 = "2017" VS2019 = "2019" + VS2022 = "2022" BuildNumbers = { VS2017: "15", VS2019: "16", + VS2022: "17", } MinSupportedUnreal = { @@ -55,7 +57,9 @@ class VisualStudio(object): # # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 - VS2019: semver.VersionInfo(4, 25) + VS2019: semver.VersionInfo(4, 25), + + VS2022: semver.VersionInfo(4, 27), } From e9d7d29c9ba1713ef2aab81b768c3177080eca7e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 15 May 2022 22:41:44 +0300 Subject: [PATCH 264/430] reformat code --- ue4docker/infrastructure/BuildConfiguration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 0f1a0dd8..c3548264 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -58,7 +58,6 @@ class VisualStudio(object): # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 VS2019: semver.VersionInfo(4, 25), - VS2022: semver.VersionInfo(4, 27), } From 74b028a148ea4aa487e968a064f075bb85cca7ea Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 16 May 2022 12:34:12 +0300 Subject: [PATCH 265/430] Use builtin curl on Windows Resolves #218 --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index 0f2401e1..c1488811 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -1,7 +1,7 @@ @rem Install the chocolatey packages we need choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error @rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y choco-cleaner curl python vcredist-all windows-sdk-10-version-1809-windbg || goto :error +choco install -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg || goto :error @rem Reload our environment variables from the registry so the `git` command works call refreshenv From bed211c5507d86364f0efd141c43a1c825fefa03 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 17 May 2022 19:07:58 +0300 Subject: [PATCH 266/430] Document why .NET 4.5 is needed --- .../ue4-build-prerequisites/windows/install-prerequisites.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index c1488811..d44e26e6 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -57,7 +57,8 @@ if "%VISUAL_STUDIO_BUILD_NUMBER%" == "15" ( @rem Install the Visual Studio Build Tools workloads and components we need @rem NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks -@rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019. And it is actually *needed* only for UE5 +@rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5 +@rem NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 curl --progress-bar -L "https://aka.ms/vs/17/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ From e31ce2ee1aed0fe2017b70d0deeffaf150ae1165 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 18 May 2022 15:34:28 +0300 Subject: [PATCH 267/430] Add CL number for UE-5.0 --- ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index c3548264..5bf8c713 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -37,6 +37,7 @@ "4.25.0": 13144385, "4.26.0": 14830424, "4.27.0": 17155196, + "5.0.0": 19505902, } From fff76b6cd082fded133f67a4fa79f9b0d4808198 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 18 May 2022 16:23:08 +0300 Subject: [PATCH 268/430] Warn users about 20GiB COPY Docker bug (#255) See #99 --- ue4docker/build.py | 2 +- ue4docker/diagnostics/diagnostic_20gig.py | 3 +-- .../infrastructure/BuildConfiguration.py | 27 +++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index c98b37a7..e42f4fc1 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -237,7 +237,7 @@ def build(): ) sys.exit(1) elif newer_check is None: - logger.info( + logger.warning( "Warning: unable to determine whether host system is new enough to use specified base tag" ) diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/ue4docker/diagnostics/diagnostic_20gig.py index b7372565..42539a01 100644 --- a/ue4docker/diagnostics/diagnostic_20gig.py +++ b/ue4docker/diagnostics/diagnostic_20gig.py @@ -40,9 +40,8 @@ def getDescription(self): return "\n".join( [ "This diagnostic determines if the Docker daemon suffers from 20GiB COPY bug", - "reported at https://github.com/moby/moby/issues/37352 (affects Windows containers only)", "", - "#37352 was fixed in https://github.com/moby/moby/pull/41636 but that fix was not released yet", + "See https://github.com/adamrehn/ue4-docker/issues/99#issuecomment-1079702817 for details and workarounds", "", self._parser.format_help(), ] diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 5bf8c713..4c88d2a0 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -531,6 +531,33 @@ def __init__(self, parser, argv, logger): "templates": ExcludedComponent.Templates in self.excludedComponents, } + # Warn user that they are in danger of Docker 20GB COPY bug + # Unfortunately, we don't have a cheap way to check whether user environment is affected + # See https://github.com/adamrehn/ue4-docker/issues/99 + if self.containerPlatform == "windows": + warn20GiB = False + if ExcludedComponent.Debug not in self.excludedComponents: + logger.warning("Warning: You didn't pass --exclude debug", False) + warn20GiB = True + if ( + self.release + and not self.custom + and semver.VersionInfo.parse(self.release) >= semver.VersionInfo(5, 0) + ): + logger.warning("Warning: You're building Unreal Engine 5", False) + warn20GiB = True + + if warn20GiB: + logger.warning("Warning: You might hit Docker 20GiB COPY bug", False) + logger.warning( + "Warning: Make sure that `ue4-docker diagnostics 20gig` passes", + False, + ) + logger.warning( + "Warning: See https://github.com/adamrehn/ue4-docker/issues/99#issuecomment-1079702817 for details and workarounds", + False, + ) + # If we're building Windows containers, generate our Windows-specific configuration settings if self.containerPlatform == "windows": self._generateWindowsConfig() From 7ecc98ac0ae351d36ff8aea8b1542f789c038920 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 26 May 2022 12:07:21 +1000 Subject: [PATCH 269/430] Add option to disable building UBT before running BuildGraph under Linux --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 1e3696c7..376de880 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -30,10 +30,13 @@ COPY patch-build-graph.py /tmp/patch-build-graph.py RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version {% endif %} +{% if (not disable_all_patches) and (not disable_ubt_prebuild) %} +# For versions of the Unreal Engine prior to 5.0 when the C# build tools migrated to .NET Core: # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the -# target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, +# target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild +{% endif %} # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine From 4a1ca70073c1b2a3f8492375aa845e48570e74de Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 30 May 2022 16:10:31 +0300 Subject: [PATCH 270/430] Always pass `-set:HostPlatformOnly=true` to build graph Motivation: it just sets defaults and user is still able to enable platforms they need. On the other hand, before this commit, `-set:HostPlatformOnly=true` would be lost if user specified *any* custom buildgraph_args This is a follow-up to a6f15bf4daf037dd0323fee87aa25151ca8a6299 --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 4 +++- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 1 + ue4docker/infrastructure/BuildConfiguration.py | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 376de880..9af573ff 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -43,7 +43,9 @@ WORKDIR /home/ue4/UnrealEngine RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ -target="Make Installed Build Linux" \ -script=Engine/Build/InstalledEngineBuild.xml \ - -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} {{ buildgraph_args }} && \ + -set:HostPlatformOnly=true \ + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} \ + {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index aaf5fa27..23bb53e8 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -31,6 +31,7 @@ WORKDIR C:\UnrealEngine RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` -target="Make Installed Build Win64" ` -script=Engine/Build/InstalledEngineBuild.xml ` + -set:HostPlatformOnly=true ` -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 4c88d2a0..05a307a2 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -474,7 +474,7 @@ def __init__(self, parser, argv, logger): ) # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) - self.opts = {"buildgraph_args": "-set:HostPlatformOnly=true"} + self.opts = {} for o in self.args.opt: if "=" in o: key, value = o.split("=", 1) From a9b9ace065da68a8d4a89edda179dd9d591808a6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 30 May 2022 16:51:29 +0300 Subject: [PATCH 271/430] Fix building on Windows with VS != 2017 Broken by 4a1ca70073c1b2a3f8492375aa845e48570e74de --- ue4docker/infrastructure/BuildConfiguration.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 05a307a2..6ccf2cbb 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -603,7 +603,10 @@ def _generateWindowsConfig(self): # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. # Otherwise, UAT errors out with "Unknown argument: VS2019". if self.visualStudio != VisualStudio.VS2017: - self.opts["buildgraph_args"] += f" -set:VS{self.visualStudio}=true" + self.opts["buildgraph_args"] = ( + self.opts.get("buildgraph_args", "") + + f" -set:VS{self.visualStudio}=true" + ) # Determine base tag for the Windows release of the host system self.hostBasetag = WindowsUtils.getHostBaseTag() From f6b30c7c1cc55c747241eb3ffb64fc802139f5a6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 2 Jun 2022 00:03:28 +0300 Subject: [PATCH 272/430] Adjust UBT pre-build warmup so it is compatible with UE >= 5.0.2 (#261) This is a follow-up to 7ecc98ac0ae351d36ff8aea8b1542f789c038920 This change accounts for `Engine/Build/BatchFiles/Linux/Build.sh` changes in https://github.com/EpicGames/UnrealEngine/commit/6bfd4d435dd6ca51c4d2b5819d5ed2f7e1dd9a12 --- ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 9af573ff..2be28863 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -30,13 +30,10 @@ COPY patch-build-graph.py /tmp/patch-build-graph.py RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version {% endif %} -{% if (not disable_all_patches) and (not disable_ubt_prebuild) %} -# For versions of the Unreal Engine prior to 5.0 when the C# build tools migrated to .NET Core: # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild -{% endif %} +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild -buildubt # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine From d973bd8d039523fc0bba6fbba457f5c172a9b8f4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 8 Jun 2022 10:14:49 +0300 Subject: [PATCH 273/430] Add support for custom build-prerequisites Dockerfile (#260) This commit adds `--prerequisites-dockerfile=[path]` flag that allows injecting custom image in place of ue4-build-prerequisites. Note that vanilla ue4-build-prerequisites is still build and available with ue4-base-build-prerequisites name, so user-provided Dockerfile can use it as a base. Primary goal of this feature is to allow installation of platform SDKs before engine is checked out into image. Placing platform SDKs in to ue4-build-prerequisites image allows to reuse the same SDKs for multiple engine builds. --- ue4docker/build.py | 47 +++++-- .../infrastructure/BuildConfiguration.py | 5 + ue4docker/infrastructure/DockerUtils.py | 4 +- ue4docker/infrastructure/ImageBuilder.py | 131 ++++++++++++------ 4 files changed, 130 insertions(+), 57 deletions(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index e42f4fc1..bc82e5da 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -88,14 +88,11 @@ def build(): # Create an auto-deleting temporary directory to hold our build context with tempfile.TemporaryDirectory() as tempDir: - # Copy our Dockerfiles to the temporary directory contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles") - contextRoot = join(tempDir, "dockerfiles") - shutil.copytree(contextOrig, contextRoot) # Create the builder instance to build the Docker images builder = ImageBuilder( - contextRoot, + join(tempDir, "dockerfiles"), config.containerPlatform, logger, config.rebuild, @@ -383,17 +380,37 @@ def build(): "VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber, ] - builder.build( - "ue4-build-prerequisites", - [config.prereqsTag], - commonArgs + config.platformArgs + prereqsArgs, - ) - builtImages.append("ue4-build-prerequisites") + custom_prerequisites_dockerfile = config.args.prerequisites_dockerfile + if custom_prerequisites_dockerfile is not None: + builder.build_builtin_image( + "ue4-base-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + builtin_name="ue4-build-prerequisites", + ) + builtImages.append("ue4-base-build-prerequisites") + else: + builder.build_builtin_image( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + ) prereqConsumerArgs = [ "--build-arg", "PREREQS_TAG={}".format(config.prereqsTag), ] + + if custom_prerequisites_dockerfile is not None: + builder.build( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqConsumerArgs, + dockerfile_template=custom_prerequisites_dockerfile, + context_dir=os.path.dirname(custom_prerequisites_dockerfile), + ) + + builtImages.append("ue4-build-prerequisites") else: logger.info("Skipping ue4-build-prerequisities image build.") @@ -418,11 +435,11 @@ def build(): "--build-arg", "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), ] - builder.build( + builder.build_builtin_image( "ue4-source", mainTags, commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, - secrets, + secrets=secrets, ) builtImages.append("ue4-source") else: @@ -436,7 +453,7 @@ def build(): # Build the UE4 Engine source build image, unless requested otherwise by the user if config.buildTargets["engine"]: - builder.build( + builder.build_builtin_image( "ue4-engine", mainTags, commonArgs + config.platformArgs + ue4BuildArgs, @@ -453,7 +470,7 @@ def build(): else [] ) - builder.build( + builder.build_builtin_image( "ue4-minimal", mainTags, commonArgs + config.platformArgs + ue4BuildArgs + minimalArgs, @@ -483,7 +500,7 @@ def build(): ) # Build the image - builder.build( + builder.build_builtin_image( "ue4-full", mainTags, commonArgs diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 6ccf2cbb..7d675b51 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -264,6 +264,11 @@ def addArguments(parser): default=None, help="Set a specific changelist number in the Unreal Engine's Build.version file", ) + parser.add_argument( + "--prerequisites-dockerfile", + default=None, + help="Specifies path to custom ue4-build-prerequisites dockerfile", + ) def __init__(self, parser, argv, logger): """ diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 1f1ce461..5570a28e 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -45,7 +45,7 @@ def exists(name): return False @staticmethod - def build(tags, context, args): + def build(tags: [str], context: str, args: [str]) -> [str]: """ Returns the `docker build` command to build an image """ @@ -58,7 +58,7 @@ def build(tags, context, args): ) @staticmethod - def buildx(tags, context, args, secrets): + def buildx(tags: [str], context: str, args: [str], secrets: [str]) -> [str]: """ Returns the `docker buildx` command to build an image with the BuildKit backend """ diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 609f6e93..5ddd122d 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -5,25 +5,34 @@ from .GlobalConfiguration import GlobalConfiguration import glob, humanfriendly, os, shutil, subprocess, tempfile, time from os.path import basename, exists, join -from jinja2 import Environment, Template +from jinja2 import Environment + + +class ImageBuildParams(object): + def __init__( + self, dockerfile: str, context_dir: str, env: Optional[Dict[str, str]] = None + ): + self.dockerfile = dockerfile + self.context_dir = context_dir + self.env = env class ImageBuilder(object): def __init__( self, - root, - platform, + tempDir: str, + platform: str, logger, - rebuild=False, - dryRun=False, - layoutDir=None, - templateContext=None, - combine=False, + rebuild: bool = False, + dryRun: bool = False, + layoutDir: str = None, + templateContext: Dict[str, str] = None, + combine: bool = False, ): """ Creates an ImageBuilder for the specified build parameters """ - self.root = root + self.tempDir = tempDir self.platform = platform self.logger = logger self.rebuild = rebuild @@ -32,17 +41,60 @@ def __init__( self.templateContext = templateContext if templateContext is not None else {} self.combine = combine - def build(self, name, tags, args, secrets=None): + def get_built_image_context(self, name): + """ + Resolve the full path to the build context for the specified image + """ + return os.path.normpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "dockerfiles", + basename(name), + self.platform, + ) + ) + + def build_builtin_image( + self, + name: str, + tags: [str], + args: [str], + builtin_name: str = None, + secrets: Dict[str, str] = None, + ): + context_dir = self.get_built_image_context( + name if builtin_name is None else builtin_name + ) + return self.build( + name, tags, args, join(context_dir, "Dockerfile"), context_dir, secrets + ) + + def build( + self, + name: str, + tags: [str], + args: [str], + dockerfile_template: str, + context_dir: str, + secrets: Dict[str, str] = None, + ): """ Builds the specified image if it doesn't exist or if we're forcing a rebuild """ + workdir = join(self.tempDir, basename(name), self.platform) + os.makedirs(workdir, exist_ok=True) + # Create a Jinja template environment and render the Dockerfile template environment = Environment( autoescape=False, trim_blocks=True, lstrip_blocks=True ) - dockerfile = join(self.context(name), "Dockerfile") - templateInstance = environment.from_string(FilesystemUtils.readFile(dockerfile)) + dockerfile = join(workdir, "Dockerfile") + + templateInstance = environment.from_string( + FilesystemUtils.readFile(dockerfile_template) + ) rendered = templateInstance.render(self.templateContext) # Compress excess whitespace introduced during Jinja rendering and save the contents back to disk @@ -70,7 +122,6 @@ def build(self, name, tags, args, secrets=None): # Determine whether we are building using `docker buildx` with build secrets imageTags = self._formatTags(name, tags) - command = DockerUtils.build(imageTags, self.context(name), args) if self.platform == "linux" and secrets is not None and len(secrets) > 0: @@ -82,9 +133,11 @@ def build(self, name, tags, args, secrets=None): secretFlags.append("id={},src={}".format(secret, secretFile)) # Generate the `docker buildx` command to use our build secrets - command = DockerUtils.buildx( - imageTags, self.context(name), args, secretFlags - ) + command = DockerUtils.buildx(imageTags, context_dir, args, secretFlags) + else: + command = DockerUtils.build(imageTags, context_dir, args) + + command += ["--file", dockerfile] env = os.environ.copy() if self.platform == "linux": @@ -97,29 +150,23 @@ def build(self, name, tags, args, secrets=None): command, "build", "built", - env=env, + ImageBuildParams(dockerfile, context_dir, env), ) - def context(self, name): - """ - Resolve the full path to the build context for the specified image - """ - return join(self.root, basename(name), self.platform) - - def pull(self, image): + def pull(self, image: str) -> None: """ Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version """ self._processImage(image, None, DockerUtils.pull(image), "pull", "pulled") - def willBuild(self, name, tags): + def willBuild(self, name: str, tags: [str]) -> bool: """ Determines if we will build the specified image, based on our build settings """ imageTags = self._formatTags(name, tags) return self._willProcess(imageTags[0]) - def _formatTags(self, name, tags): + def _formatTags(self, name: str, tags: [str]): """ Generates the list of fully-qualified tags that we will use when building an image """ @@ -127,11 +174,11 @@ def _formatTags(self, name, tags): "{}:{}".format(GlobalConfiguration.resolveTag(name), tag) for tag in tags ] - def _willProcess(self, image): + def _willProcess(self, image: [str]) -> bool: """ Determines if we will build or pull the specified image, based on our build settings """ - return self.rebuild == True or DockerUtils.exists(image) == False + return self.rebuild or not DockerUtils.exists(image) def _processImage( self, @@ -140,14 +187,14 @@ def _processImage( command: [str], actionPresentTense: str, actionPastTense: str, - env: Optional[Dict[str, str]] = None, - ): + build_params: Optional[ImageBuildParams] = None, + ) -> None: """ Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) """ # Determine if we are processing the image - if self._willProcess(image) == False: + if not self._willProcess(image): self.logger.info( 'Image "{}" exists and rebuild not requested, skipping {}.'.format( image, actionPresentTense @@ -159,7 +206,7 @@ def _processImage( self.logger.action( '{}ing image "{}"...'.format(actionPresentTense.capitalize(), image) ) - if self.dryRun == True: + if self.dryRun: print(command) self.logger.action( 'Completed dry run for image "{}".'.format(image), newline=False @@ -170,19 +217,19 @@ def _processImage( if self.layoutDir is not None: # Determine whether we're performing a simple copy or combining generated Dockerfiles - source = self.context(name) - if self.combine == True: + if self.combine: # Ensure the destination directory exists dest = join(self.layoutDir, "combined") self.logger.action( - 'Merging "{}" into "{}"...'.format(source, dest), newline=False + 'Merging "{}" into "{}"...'.format(build_params.context_dir, dest), + newline=False, ) os.makedirs(dest, exist_ok=True) # Merge the source Dockerfile with any existing Dockerfile contents in the destination directory # (Insert a single newline between merged file contents and ensure we have a single trailing newline) - sourceDockerfile = join(source, "Dockerfile") + sourceDockerfile = build_params.dockerfile destDockerfile = join(dest, "Dockerfile") dockerfileContents = ( FilesystemUtils.readFile(destDockerfile) @@ -199,7 +246,7 @@ def _processImage( # Copy any supplemental files from the source directory to the destination directory # (Exclude any extraneous files which are not referenced in the Dockerfile contents) - for file in glob.glob(join(source, "*.*")): + for file in glob.glob(join(build_params.context_dir, "*.*")): if basename(file) in dockerfileContents: shutil.copy(file, join(dest, basename(file))) @@ -213,9 +260,11 @@ def _processImage( # Copy the source directory to the destination dest = join(self.layoutDir, basename(name)) self.logger.action( - 'Copying "{}" to "{}"...'.format(source, dest), newline=False + 'Copying "{}" to "{}"...'.format(build_params.context_dir, dest), + newline=False, ) - shutil.copytree(source, dest) + shutil.copytree(build_params.context_dir, dest) + shutil.copy(build_params.dockerfile, dest) self.logger.action( 'Copied Dockerfile for image "{}".'.format(image), newline=False ) @@ -224,7 +273,9 @@ def _processImage( # Attempt to process the image using the supplied command startTime = time.time() - exitCode = subprocess.call(command, env=env) + exitCode = subprocess.call( + command, env=build_params.env if build_params else None + ) endTime = time.time() # Determine if processing succeeded From bdb03ce1551d2a778b05ddd8e7773e7fe3b0774c Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 8 Jun 2022 10:24:13 +0300 Subject: [PATCH 274/430] Remove maxsize diagnostic Motivation: 1. It tests for issue that only affects *EOLed* Windows 1903/1909 2. It doesn't work properly on Windows Server newer than 1909 because it tries to run incompatible container with process isolation 3. There's already check in `WindowsUtils.isBlacklistedWindowsHost` that will prevent user from running ue4-docker on affected systems resolves #263 --- ue4docker/diagnostics/__init__.py | 1 - ue4docker/diagnostics/diagnostic_all.py | 2 - ue4docker/diagnostics/diagnostic_maxsize.py | 90 --------------------- ue4docker/diagnostics_cmd.py | 1 - 4 files changed, 94 deletions(-) delete mode 100644 ue4docker/diagnostics/diagnostic_maxsize.py diff --git a/ue4docker/diagnostics/__init__.py b/ue4docker/diagnostics/__init__.py index 7b91b60c..f2c813bd 100644 --- a/ue4docker/diagnostics/__init__.py +++ b/ue4docker/diagnostics/__init__.py @@ -1,5 +1,4 @@ from .diagnostic_all import allDiagnostics from .diagnostic_8gig import diagnostic8Gig from .diagnostic_20gig import diagnostic20Gig -from .diagnostic_maxsize import diagnosticMaxSize from .diagnostic_network import diagnosticNetwork diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index 1a5dbef3..cada4ba1 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -1,7 +1,6 @@ from .base import DiagnosticBase from .diagnostic_8gig import diagnostic8Gig from .diagnostic_20gig import diagnostic20Gig -from .diagnostic_maxsize import diagnosticMaxSize from .diagnostic_network import diagnosticNetwork @@ -28,7 +27,6 @@ def run(self, logger, args=[]): diagnostics = [ diagnostic8Gig(), diagnostic20Gig(), - diagnosticMaxSize(), diagnosticNetwork(), ] for index, diagnostic in enumerate(diagnostics): diff --git a/ue4docker/diagnostics/diagnostic_maxsize.py b/ue4docker/diagnostics/diagnostic_maxsize.py deleted file mode 100644 index 787d32de..00000000 --- a/ue4docker/diagnostics/diagnostic_maxsize.py +++ /dev/null @@ -1,90 +0,0 @@ -from ..infrastructure import DockerUtils, WindowsUtils -from .base import DiagnosticBase -import platform - - -class diagnosticMaxSize(DiagnosticBase): - def getName(self): - """ - Returns the human-readable name of the diagnostic - """ - return "Check for Windows storage-opt bug" - - def getDescription(self): - """ - Returns a description of what the diagnostic does - """ - return "\n".join( - [ - "This diagnostic determines if the Windows host OS suffers from the bug that", - "prevents users from increasing the maximum allowed image size using Docker's", - "`storage-opt` configuration key, as reported here:", - "https://github.com/docker/for-win/issues/4100", - "", - "This bug is present in Windows Server and Windows 10 versions 1903 and 1909,", - "but a workaround was introduced in Docker CE version 19.03.6.", - ] - ) - - def getPrefix(self): - """ - Returns the short name of the diagnostic for use in log output - """ - return "maxsize" - - def run(self, logger, args=[]): - """ - Runs the diagnostic - """ - - # Verify that we are running under Windows and are in Windows container mode if using Docker Desktop - dockerPlatform = DockerUtils.info()["OSType"].lower() - hostPlatform = platform.system().lower() - if hostPlatform != "windows": - logger.info( - "[maxsize] This diagnostic only applies to Windows host systems.", False - ) - return True - elif dockerPlatform != "windows": - logger.error( - "[maxsize] Error: Docker Desktop is currently in Linux container mode.", - False, - ) - logger.error( - "[maxsize] Please switch to Windows container mode to run this diagnostic.", - False, - ) - return False - - # Verify that we are running Windows Server or Windows 10 version 1903 or newer - if WindowsUtils.getWindowsBuild() < 18362: - logger.info( - "[maxsize] This diagnostic only applies to Windows Server and Windows 10 version 1903 and newer.", - False, - ) - return True - - # Attempt to run a Windows Nanoserver 1903 container with the `storage-mode` configuration options set - # (The bug doesn't seem to be triggered when using older Windows images, presumably because they use an older host kernel) - try: - command = [ - "docker", - "run", - "--rm", - "--storage-opt", - "size=200GB", - "mcr.microsoft.com/windows/nanoserver:1903", - "cmd", - "exit", - ] - self._printAndRun(logger, "[maxsize] ", command, check=True) - logger.action( - "[maxsize] Diagnostic succeeded! The host platform can specify maximum image sizes using Docker's `storage-opt` configuration key.\n" - ) - return True - except: - logger.error( - "[maxsize] Diagnostic failed! The host platform cannot specify maximum image sizes using Docker's `storage-opt` configuration key.\n", - True, - ) - return False diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index 829ebbc9..8d40e5d1 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -10,7 +10,6 @@ def diagnostics(): "all": allDiagnostics(), "8gig": diagnostic8Gig(), "20gig": diagnostic20Gig(), - "maxsize": diagnosticMaxSize(), "network": diagnosticNetwork(), } From 1312b0e785aed5d7f65c880333d21959af159593 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 8 Jun 2022 10:37:08 +0300 Subject: [PATCH 275/430] Bump version to 0.0.99 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 12ca2072..c0fa484b 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.98" +__version__ = "0.0.99" From e6bfee3fa625cb0783292e30a785c1cb91138659 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Jun 2022 21:19:09 +0000 Subject: [PATCH 276/430] Bump actions/setup-python from 3 to 4 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 3 to 4. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0113818a..5417d3fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: "3.10" - name: Install dependencies @@ -50,7 +50,7 @@ jobs: - name: Checkout uses: actions/checkout@v3 - name: Setup Python - uses: actions/setup-python@v3 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - name: Install From b36516af8baf2bdc4979a3c0af795baefc4b456a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 22 Jun 2022 00:15:55 +0300 Subject: [PATCH 277/430] [Windows] Remove engine intermediate engine build products before committing ue4-minimal builder image (#265) These products are not used after build step, but they take massive amounts of disk space: up to 350GBs for multi-platform (Win64+PS4+PS5+XB1+XSX) engine build To make matters even worse, current Docker image commit algorithm on Windows additionally uses x2 of resulting image size on disk. --- ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 23bb53e8..fc2cc433 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -34,7 +34,8 @@ RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` -set:HostPlatformOnly=true ` -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` {{ buildgraph_args }} && ` - (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) + (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) && ` + rmdir /s /q C:\UnrealEngine\Engine # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers COPY split-components.py C:\split-components.py From 19695cd62caf2ed4f243595cee36c68a7ad632ce Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 8 Aug 2022 13:35:58 +0300 Subject: [PATCH 278/430] Bump version to 0.0.100 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index c0fa484b..1d6201ee 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.99" +__version__ = "0.0.100" From db499ae6c80a1ad7443b0767b20a81eead9e224a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 31 Oct 2022 19:13:19 +0300 Subject: [PATCH 279/430] Remove ue4-engine image (#273) Resolves #249 1. There is no visible evidence that _anyone_ uses ue4-engine image. 2. ue4-engine is broken for UE5 3. ue4-engine is not a prerequisite of ue4-minimal/ue4-full --- ue4docker/build.py | 13 +------------ ue4docker/clean.py | 10 ---------- .../dockerfiles/ue4-engine/linux/Dockerfile | 13 ------------- .../dockerfiles/ue4-engine/windows/Dockerfile | 15 --------------- ue4docker/infrastructure/BuildConfiguration.py | 17 +---------------- 5 files changed, 2 insertions(+), 66 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-engine/linux/Dockerfile delete mode 100644 ue4docker/dockerfiles/ue4-engine/windows/Dockerfile diff --git a/ue4docker/build.py b/ue4docker/build.py index bc82e5da..3327c657 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -445,23 +445,12 @@ def build(): else: logger.info("Skipping ue4-source image build.") - if config.buildTargets["engine"] or config.buildTargets["minimal"]: + if config.buildTargets["minimal"]: ue4BuildArgs = prereqConsumerArgs + [ "--build-arg", "TAG={}".format(mainTags[1]), ] - # Build the UE4 Engine source build image, unless requested otherwise by the user - if config.buildTargets["engine"]: - builder.build_builtin_image( - "ue4-engine", - mainTags, - commonArgs + config.platformArgs + ue4BuildArgs, - ) - builtImages.append("ue4-engine") - else: - logger.info("Skipping ue4-engine image build.") - # Build the minimal UE4 CI image, unless requested otherwise by the user if config.buildTargets["minimal"]: minimalArgs = ( diff --git a/ue4docker/clean.py b/ue4docker/clean.py index 377f0860..b8ad1869 100644 --- a/ue4docker/clean.py +++ b/ue4docker/clean.py @@ -29,7 +29,6 @@ def clean(): "-tag", default=None, help="Only clean images with the specified tag" ) parser.add_argument("--source", action="store_true", help="Clean ue4-source images") - parser.add_argument("--engine", action="store_true", help="Clean ue4-engine images") parser.add_argument( "--all", action="store_true", help="Clean all ue4-docker images" ) @@ -62,15 +61,6 @@ def clean(): args.dry_run, ) - # If requested, remove ue4-engine images - if args.engine == True: - _cleanMatching( - cleaner, - GlobalConfiguration.resolveTag("ue4-engine"), - args.tag, - args.dry_run, - ) - # If requested, remove everything if args.all == True: _cleanMatching( diff --git a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile b/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile deleted file mode 100644 index 3d4caa2d..00000000 --- a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -{% if combine %} -FROM source as engine -{% else %} -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} -{% endif %} - -# Build UBT and build the Engine -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -WaitMutex -RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -WaitMutex -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealPak Linux Development -WaitMutex diff --git a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile b/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile deleted file mode 100644 index 8d2bc70d..00000000 --- a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# escape=` -{% if combine %} -FROM source as engine -{% else %} -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} -{% endif %} - -# Build UBT and build the Engine -RUN GenerateProjectFiles.bat -RUN .\Engine\Build\BatchFiles\Build.bat UE4Editor Win64 Development -WaitMutex -RUN .\Engine\Build\BatchFiles\Build.bat ShaderCompileWorker Win64 Development -WaitMutex -RUN .\Engine\Build\BatchFiles\Build.bat UnrealPak Win64 Development -WaitMutex diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 7d675b51..119d8144 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -120,11 +120,6 @@ def addArguments(parser): action="store_true", help="Print `docker build` commands instead of running them", ) - parser.add_argument( - "--no-engine", - action="store_true", - help="Don't build the ue4-engine image (deprecated, use --target instead)", - ) parser.add_argument( "--no-minimal", action="store_true", @@ -283,9 +278,7 @@ def __init__(self, parser, argv, logger): self.changelist = self.args.changelist # Figure out what targets we have; this is needed to find out if we need --ue-version. - using_target_specifier_old = ( - self.args.no_engine or self.args.no_minimal or self.args.no_full - ) + using_target_specifier_old = self.args.no_minimal or self.args.no_full using_target_specifier_new = self.args.target is not None # If we specified nothing, it's the same as specifying `all` @@ -311,9 +304,6 @@ def __init__(self, parser, argv, logger): if not self.args.no_minimal: self.args.target += ["minimal"] - if not self.args.no_engine: - self.args.target += ["engine"] - # disabling these was never supported self.args.target += ["source"] self.args.target += ["build-prerequisites"] @@ -341,7 +331,6 @@ def __init__(self, parser, argv, logger): self.buildTargets = { "build-prerequisites": False, "source": False, - "engine": False, "minimal": False, "full": False, } @@ -361,10 +350,6 @@ def __init__(self, parser, argv, logger): self.buildTargets["minimal"] = True active_targets.add("source") - if "engine" in active_targets or "all" in active_targets: - self.buildTargets["engine"] = True - active_targets.add("source") - if "source" in active_targets or "all" in active_targets: self.buildTargets["source"] = True active_targets.add("build-prerequisites") From 23383dfb44fc8ab29077cd908a54fc73d6b95084 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 2 Nov 2022 14:02:07 +0300 Subject: [PATCH 280/430] Avoid downloading unneeded dependencies on Linux (#274) Before this commit, UE-5.0.3: * `ue4-source`: 67.2GB * `ue4-minimal`: 54.5GB After this commit, UE-5.0.3: * `ue4-source`: 47.5GB * `ue4-minimal`: 47GB --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 702a7044..a956c729 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -96,7 +96,11 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT # When running with BuildKit, we use a cache mount to cache the dependency data in `.git/ue4-gitdeps` across multiple build invocations WORKDIR /home/ue4/UnrealEngine RUN --mount=type=cache,target=/home/ue4/UnrealEngine/.git/ue4-gitdeps,uid=1000,gid=1000 sudo apt-get update && \ - ./Setup.sh && \ + ./Setup.sh \ + --exclude=Android \ + --exclude=Mac \ + --exclude=Win32 \ + --exclude=Win64 && \ sudo rm -rf /var/lib/apt/lists/* {% else %} @@ -104,7 +108,11 @@ RUN --mount=type=cache,target=/home/ue4/UnrealEngine/.git/ue4-gitdeps,uid=1000,g # When running without BuildKit, we use the `-no-cache` flag to disable caching of dependency data in `.git/ue4-gitdeps`, saving disk space WORKDIR /home/ue4/UnrealEngine RUN sudo apt-get update && \ - ./Setup.sh -no-cache && \ + ./Setup.sh -no-cache \ + --exclude=Android \ + --exclude=Mac \ + --exclude=Win32 \ + --exclude=Win64 && \ sudo rm -rf /var/lib/apt/lists/* {% endif %} From 2d0ef0df844b45bdc46d78285301834c5a21a16b Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 17 Nov 2022 09:34:53 +1000 Subject: [PATCH 281/430] Add XKB dependency required by UE5.1 under Linux --- ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index 322dc5ce..a4c37e71 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -51,6 +51,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libxcursor1 \ libxdamage1 \ libxi6 \ + libxkbcommon-x11-0 \ libxrandr2 \ libxrender1 \ libxss1 \ From 21b2f05b586a2d71c8f93a95b5a2472e3494a74d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 17 Nov 2022 10:12:31 +1000 Subject: [PATCH 282/430] Ensure consistent cache path across UE4/5 when using BuildKit cache mount --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index a956c729..38e70fdc 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -93,9 +93,14 @@ RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT # Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` {% if credential_mode == "secrets" %} -# When running with BuildKit, we use a cache mount to cache the dependency data in `.git/ue4-gitdeps` across multiple build invocations +# Ensure Setup.sh uses the same cache path when building either UE4 or UE5 +ENV UE_GITDEPS=/home/ue4/gitdeps +ENV UE4_GITDEPS=/home/ue4/gitdeps +RUN mkdir "$UE_GITDEPS" + +# When running with BuildKit, we use a cache mount to cache the dependency data across multiple build invocations WORKDIR /home/ue4/UnrealEngine -RUN --mount=type=cache,target=/home/ue4/UnrealEngine/.git/ue4-gitdeps,uid=1000,gid=1000 sudo apt-get update && \ +RUN --mount=type=cache,target=/home/ue4/gitdeps,uid=1000,gid=1000 sudo apt-get update && \ ./Setup.sh \ --exclude=Android \ --exclude=Mac \ From e423bde2713803e2facc061f2b79ec63335a32c8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 17 Nov 2022 10:34:39 +1000 Subject: [PATCH 283/430] Bump version to 0.0.101 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 1d6201ee..43028041 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.100" +__version__ = "0.0.101" From 3bda1a33bf8dfb39040e8e2c2ae0366a7fc79fb5 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 18 Nov 2022 14:09:14 +1000 Subject: [PATCH 284/430] Add CL number for Unreal Engine 5.1.0 release --- ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 119d8144..dede8a3e 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -38,6 +38,7 @@ "4.26.0": 14830424, "4.27.0": 17155196, "5.0.0": 19505902, + "5.1.0": 23058290, } From ae9f63e37c92846c09ada2b5ec623c9077121e85 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 23 Nov 2022 08:08:59 +0300 Subject: [PATCH 285/430] Prevent attempts to build UE with too old Visual Studios that are known to fail (#279) --- .../infrastructure/BuildConfiguration.py | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index dede8a3e..9284d245 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -53,16 +53,22 @@ class VisualStudio(object): VS2022: "17", } - MinSupportedUnreal = { + SupportedSince = { + # We do not support versions older than 4.20 + VS2017: semver.VersionInfo(4.20), # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa # # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 VS2019: semver.VersionInfo(4, 25), - VS2022: semver.VersionInfo(4, 27), + # Even though Epics claimed in 4.27 and 5.0 that they added VS2022 support, + # everything up to (including) 5.1.0 fails to build with it due to dependencies on .NET 4.5. + VS2022: semver.VersionInfo(5, 1, 1), } + UnsupportedSince = {VS2017: semver.VersionInfo(5, 0)} + class ExcludedComponent(object): """ @@ -576,16 +582,27 @@ def _generateWindowsConfig(self): if self.release is not None and not self.custom: # Check whether specified Unreal Engine release is compatible with specified Visual Studio - vsMinSupportedUnreal = VisualStudio.MinSupportedUnreal.get( + supportedSince = VisualStudio.SupportedSince.get(self.visualStudio, None) + if ( + supportedSince is not None + and semver.VersionInfo.parse(self.release) < supportedSince + ): + raise RuntimeError( + "specified version of Unreal Engine is too old for Visual Studio {}".format( + self.visualStudio + ) + ) + + unsupportedSince = VisualStudio.UnsupportedSince.get( self.visualStudio, None ) if ( - vsMinSupportedUnreal is not None - and semver.VersionInfo.parse(self.release) < vsMinSupportedUnreal + unsupportedSince is not None + and semver.VersionInfo.parse(self.release) >= unsupportedSince ): raise RuntimeError( - "specified version of Unreal Engine cannot be built with Visual Studio {}, oldest supported is {}".format( - self.visualStudio, vsMinSupportedUnreal + "Visual Studio {} is too old for specified version of Unreal Engine".format( + self.visualStudio ) ) From bdeb73c223f50b7c93b6743995ac70c0c46019f8 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 24 Nov 2022 15:02:35 +1000 Subject: [PATCH 286/430] Add `compute` and `video` NVIDIA driver capabilities under Linux --- .../ue4-build-prerequisites/linux/Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index a4c37e71..d16d5181 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -9,10 +9,18 @@ LABEL com.adamrehn.ue4-docker.sentinel="1" # Disable interactive prompts during package installation ENV DEBIAN_FRONTEND=noninteractive +# Enable CUDA support for NVIDIA GPUs (even when not using a CUDA base image), since evidently some versions of UE unconditionally assume +# `libcuda.so.1` exists when using the NVIDIA proprietary drivers, and will fail to initialise the Vulkan RHI if it is missing +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},compute + # Add the "display" driver capability for NVIDIA GPUs -# (This also allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) +# (This allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display +# Enable NVENC support for use by Unreal Engine plugins that depend on it (e.g. Pixel Streaming) +# (Note that adding `video` seems to implicitly enable `compute` as well, but we include separate directives here to clearly indicate the purpose of both) +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},video + # Install our build prerequisites RUN apt-get update && apt-get install -y --no-install-recommends \ build-essential \ From 46a3c1ac30046a7fe1dff8f424ce347a55e9ba0f Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 24 Nov 2022 15:38:24 +1000 Subject: [PATCH 287/430] Bump version to 0.0.102 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 43028041..945bce51 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.101" +__version__ = "0.0.102" From 0c95b864a298cb3f0557277cd2fa5a57be01c521 Mon Sep 17 00:00:00 2001 From: CanisHelix <7948640+CanisHelix@users.noreply.github.com> Date: Thu, 8 Dec 2022 12:54:27 +0000 Subject: [PATCH 288/430] feat(vs2022): improve vs2022 support (#284) * feat(vs2022): improve vs2022 support * fix(build): support vs2022 from 5.0.0 --- .../windows/install-prerequisites.bat | 11 ++++++++++- ue4docker/infrastructure/BuildConfiguration.py | 4 +--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat index d44e26e6..f7771054 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat @@ -59,6 +59,7 @@ if "%VISUAL_STUDIO_BUILD_NUMBER%" == "15" ( @rem NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks @rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5 @rem NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 +@rem NOTE: Microsoft.NetCore.Component.Runtime.3.1 is required by the AutomationTool tool and does not come installed with VS2022 so it needs targetting here. curl --progress-bar -L "https://aka.ms/vs/17/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error %TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ --installPath C:\BuildTools ^ @@ -75,10 +76,18 @@ curl --progress-bar -L "https://aka.ms/vs/17/release/vs_buildtools.exe" --output --add Microsoft.Net.Component.4.5.TargetingPack ^ --add Microsoft.Net.Component.4.6.2.TargetingPack ^ --add Microsoft.Net.ComponentGroup.DevelopmentPrerequisites ^ - --add Microsoft.NetCore.Component.SDK + --add Microsoft.NetCore.Component.SDK ^ + --add Microsoft.NetCore.Component.Runtime.3.1 python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error +@rem NOTE: Install the .Net 4.5 Framework Pack via nuget as this is not installed in VS2022 by the buildtools. +if "%VISUAL_STUDIO_BUILD_NUMBER%" == "17" ( +curl --progress-bar -L "https://www.nuget.org/api/v2/package/Microsoft.NETFramework.ReferenceAssemblies.net45/1.0.3" --output %TEMP%\DotNet45.zip && ^ +powershell -Command "Expand-Archive -Path \"$env:TEMP\DotNet45.zip\" -DestinationPath $env:TEMP" && ^ +powershell -Command "Copy-Item -Path \"$env:TEMP\build\.NETFramework\v4.5\*\" -Destination 'C:\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\' -Recurse -Force" || goto :error +) + @rem Clean up any temp files generated during prerequisite installation rmdir /S /Q \\?\%TEMP% mkdir %TEMP% diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 9284d245..f0271307 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -62,9 +62,7 @@ class VisualStudio(object): # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 VS2019: semver.VersionInfo(4, 25), - # Even though Epics claimed in 4.27 and 5.0 that they added VS2022 support, - # everything up to (including) 5.1.0 fails to build with it due to dependencies on .NET 4.5. - VS2022: semver.VersionInfo(5, 1, 1), + VS2022: semver.VersionInfo(5, 0, 0), } UnsupportedSince = {VS2017: semver.VersionInfo(5, 0)} From 16db23ee2b0c332fa18547adfce6ee4dd9fb88af Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 9 Dec 2022 14:12:17 +0300 Subject: [PATCH 289/430] Fix CI ubuntu-latest is now 22.04 and it doesn't have Python 3.6 anymore --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5417d3fe..5d103323 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,8 +31,9 @@ jobs: include: - { os: ubuntu-18.04, python: "3.6" } - { os: ubuntu-18.04, python: "3.10" } - - { os: ubuntu-latest, python: "3.6" } - - { os: ubuntu-latest, python: "3.10" } + - { os: ubuntu-20.04, python: "3.6" } + - { os: ubuntu-20.04, python: "3.10" } + - { os: ubuntu-22.04, python: "3.10" } - { os: windows-2019, python: "3.6", visual-studio: 2017 } - { os: windows-2019, python: "3.10", visual-studio: 2017 } - { os: windows-2019, python: "3.6", visual-studio: 2019 } From 85fdb97e14e9bc81ad6a9e9e8cef8c154e35262e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 9 Dec 2022 15:14:42 +0300 Subject: [PATCH 290/430] Stop using Ubuntu 18.04 on CI It is deprecated. See https://github.com/actions/virtual-environments/issues/6002 --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d103323..97c2aab2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,6 @@ jobs: strategy: matrix: include: - - { os: ubuntu-18.04, python: "3.6" } - - { os: ubuntu-18.04, python: "3.10" } - { os: ubuntu-20.04, python: "3.6" } - { os: ubuntu-20.04, python: "3.10" } - { os: ubuntu-22.04, python: "3.10" } From 5861435544a353aba0fa8957fa7368c8d24ab891 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 10 Dec 2022 00:36:00 +0300 Subject: [PATCH 291/430] Rewrite install-prerequisites.bat in PowerShell (#287) This is a preparation step before increasing script complexity --- .../windows/Dockerfile | 4 +- .../windows/buildtools-exitcode.py | 7 - .../windows/install-prerequisites.bat | 111 ---------------- .../windows/install-prerequisites.ps1 | 124 ++++++++++++++++++ 4 files changed, 126 insertions(+), 120 deletions(-) delete mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py delete mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat create mode 100644 ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index a0e37518..72bc765b 100644 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -57,6 +57,6 @@ RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v Lo RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Install the rest of our build prerequisites and clean up afterwards to minimise image size -COPY buildtools-exitcode.py install-prerequisites.bat C:\ +COPY install-prerequisites.ps1 C:\ ARG VISUAL_STUDIO_BUILD_NUMBER -RUN C:\install-prerequisites.bat %VISUAL_STUDIO_BUILD_NUMBER% +RUN powershell -ExecutionPolicy Bypass -File C:\install-prerequisites.ps1 %VISUAL_STUDIO_BUILD_NUMBER% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py deleted file mode 100644 index 70ef15eb..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -import sys - -# Propagate any exit codes from the VS Build Tools installer except for 3010 -code = int(sys.argv[1]) -code = 0 if code == 3010 else code -sys.exit(code) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat deleted file mode 100644 index f7771054..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ /dev/null @@ -1,111 +0,0 @@ -@rem Install the chocolatey packages we need -choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error -@rem pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg || goto :error - -@rem Reload our environment variables from the registry so the `git` command works -call refreshenv -@echo on - -@rem Forcibly disable the git credential manager -git config --system credential.helper "" || goto :error - -@rem Gather the required DirectX runtime files, since Windows Server Core does not include them -curl --progress-bar -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe && ^ -start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ^ -expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\Windows\System32\ && ^ -expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\Windows\System32\ && ^ -expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\Windows\System32\ && ^ -expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\Windows\System32\ && ^ -expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\Windows\System32\ || goto :error - -@rem Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) -curl --progress-bar -L "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" --output %TEMP%\dxc.zip && ^ -powershell -Command "Expand-Archive -Path \"$env:TEMP\dxc.zip\" -DestinationPath $env:TEMP" && ^ -xcopy /y %TEMP%\bin\x64\dxcompiler.dll C:\Windows\System32\ && ^ -xcopy /y %TEMP%\bin\x64\dxil.dll C:\Windows\System32\ || goto :error - -@rem Gather the Vulkan runtime library -curl --progress-bar -L "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" --output %TEMP%\vulkan-runtime-components.zip && ^ -powershell -Command "Expand-Archive -Path \"$env:TEMP\vulkan-runtime-components.zip\" -DestinationPath $env:TEMP" && ^ -powershell -Command "Copy-Item -Path \"*\x64\vulkan-1.dll\" -Destination C:\Windows\System32" || goto :error - -set VISUAL_STUDIO_BUILD_NUMBER=%~1 - -@rem Use the latest available Windows SDK. The motivation behind this is: -@rem 1. Newer SDKs allow developers to use newer APIs. Developers can guard that API usage with runtime version checks if they want to continue to support older Windows releases. -@rem 2. Unreal Engine slowly moves to newer Windows SDK. 4.27.0 no longer compiles with SDKs older than 18362 and even if it will be fixed in 4.27.x, -@rem this is just a question of a time when older SDKs support will be dropped completely -@rem 3. UE5 doesn't support VS2017 at all, so in the future that argument for continuing to use Windows SDK 17763 from VS2017 era will be weaker and weaker. -@rem -@rem We can't use newer SDK for VS2017 that is used to compile older engines because 18362 SDK support was only added in UE-4.23. -@rem -@rem See https://github.com/adamrehn/ue4-docker/issues/192 -@rem See https://forums.unrealengine.com/t/ndis_miniport_major_version-is-not-defined-error/135058 -@rem See https://github.com/EpicGames/UnrealEngine/blame/4.23.0-release/Engine/Source/Programs/UnrealBuildTool/Platform/Windows/UEBuildWindows.cs#L1822-L1823 -@rem See https://github.com/EpicGames/UnrealEngine/commit/ecc4872c3269e75a24adc40734cc8bcc9bbed1ca -@rem See https://udn.unrealengine.com/s/question/0D54z000079HcjJCAS/d3d12h427-error-c4668-winapipartitiongames-is-not-defined-as-a-preprocessor-macro-replacing-with-0-for-ifelif -@rem -@rem Keywords for Google: -@rem error C4668: 'NDIS_MINIPORT_MAJOR_VERSION' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif -@rem d3d12.h(427): error C4668: 'WINAPI_PARTITION_GAMES' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' -if "%VISUAL_STUDIO_BUILD_NUMBER%" == "15" ( - set WINDOWS_SDK_VERSION=17763 -) else ( - set WINDOWS_SDK_VERSION=20348 -) - -@rem Install the Visual Studio Build Tools workloads and components we need -@rem NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks -@rem NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5 -@rem NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 -@rem NOTE: Microsoft.NetCore.Component.Runtime.3.1 is required by the AutomationTool tool and does not come installed with VS2022 so it needs targetting here. -curl --progress-bar -L "https://aka.ms/vs/17/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error -%TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ - --installPath C:\BuildTools ^ - --channelUri "https://aka.ms/vs/%VISUAL_STUDIO_BUILD_NUMBER%/release/channel" ^ - --installChannelUri "https://aka.ms/vs/%VISUAL_STUDIO_BUILD_NUMBER%/release/channel" ^ - --channelId VisualStudio.%VISUAL_STUDIO_BUILD_NUMBER%.Release ^ - --productId Microsoft.VisualStudio.Product.BuildTools ^ - --locale en-US ^ - --add Microsoft.VisualStudio.Workload.VCTools ^ - --add Microsoft.VisualStudio.Workload.MSBuildTools ^ - --add Microsoft.VisualStudio.Component.NuGet ^ - --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ^ - --add Microsoft.VisualStudio.Component.Windows10SDK.%WINDOWS_SDK_VERSION% ^ - --add Microsoft.Net.Component.4.5.TargetingPack ^ - --add Microsoft.Net.Component.4.6.2.TargetingPack ^ - --add Microsoft.Net.ComponentGroup.DevelopmentPrerequisites ^ - --add Microsoft.NetCore.Component.SDK ^ - --add Microsoft.NetCore.Component.Runtime.3.1 - -python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error - -@rem NOTE: Install the .Net 4.5 Framework Pack via nuget as this is not installed in VS2022 by the buildtools. -if "%VISUAL_STUDIO_BUILD_NUMBER%" == "17" ( -curl --progress-bar -L "https://www.nuget.org/api/v2/package/Microsoft.NETFramework.ReferenceAssemblies.net45/1.0.3" --output %TEMP%\DotNet45.zip && ^ -powershell -Command "Expand-Archive -Path \"$env:TEMP\DotNet45.zip\" -DestinationPath $env:TEMP" && ^ -powershell -Command "Copy-Item -Path \"$env:TEMP\build\.NETFramework\v4.5\*\" -Destination 'C:\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\' -Recurse -Force" || goto :error -) - -@rem Clean up any temp files generated during prerequisite installation -rmdir /S /Q \\?\%TEMP% -mkdir %TEMP% - -@rem This shaves off ~300MB as of 2021-08-31 -choco-cleaner || goto :error - -@rem Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file -@rem Just remove it, so a proper one will be generated on next NuGet run -@rem See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034 -if exist %APPDATA%\NuGet rmdir /s /q %APPDATA%\NuGet - -@rem Display a human-readable completion message -@echo off -@echo Finished installing build prerequisites and cleaning up temporary files. -goto :EOF - -@rem If any of our essential commands fail, propagate the error code -:error -@echo off -exit /b %ERRORLEVEL% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 new file mode 100644 index 00000000..35eda990 --- /dev/null +++ b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -0,0 +1,124 @@ +$ErrorActionPreference = "stop" + +# TODO: Why `Update-SessionEnvironment` doesn't Just Work without this? +# Taken from https://stackoverflow.com/a/46760714 +# Make `Update-SessionEnvironment` available right away, by defining the $env:ChocolateyInstall +# variable and importing the Chocolatey profile module. +$env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." +Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +# Install the chocolatey packages we need +choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" + +# pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine +choco install -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg + +# Reload our environment variables from the registry so the `git` command works +Update-SessionEnvironment + +# Forcibly disable the git credential manager +git config --system credential.helper "" + +# Gather the required DirectX runtime files, since Windows Server Core does not include them +Invoke-WebRequest -Uri "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" -OutFile "$env:TEMP\directx_redist.exe" +Start-Process -FilePath "$env:TEMP\directx_redist.exe" -ArgumentList "/Q", "/T:$env:TEMP" -Wait +expand "$env:TEMP\APR2007_xinput_x64.cab" -F:xinput1_3.dll C:\Windows\System32\ +expand "$env:TEMP\Jun2010_D3DCompiler_43_x64.cab" -F:D3DCompiler_43.dll C:\Windows\System32\ +expand "$env:TEMP\Feb2010_X3DAudio_x64.cab" -F:X3DAudio1_7.dll C:\Windows\System32\ +expand "$env:TEMP\Jun2010_XAudio_x64.cab" -F:XAPOFX1_5.dll C:\Windows\System32\ +expand "$env:TEMP\Jun2010_XAudio_x64.cab" -F:XAudio2_7.dll C:\Windows\System32\ + +# Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) +Invoke-WebRequest -Uri "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" -OutFile "$env:TEMP\dxc.zip" +Expand-Archive -Path "$env:TEMP\dxc.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "$env:TEMP\bin\x64\dxcompiler.dll" C:\Windows\System32\ +Copy-Item -Path "$env:TEMP\bin\x64\dxil.dll" C:\Windows\System32\ + +# Gather the Vulkan runtime library +Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" -OutFile "$env:TEMP\vulkan-runtime-components.zip" +Expand-Archive -Path "$env:TEMP\vulkan-runtime-components.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "*\x64\vulkan-1.dll" -Destination C:\Windows\System32\ + +$visual_studio_build = $args[0] + +# Use the latest available Windows SDK. The motivation behind this is: +# 1. Newer SDKs allow developers to use newer APIs. Developers can guard that API usage with runtime version checks if they want to continue to support older Windows releases. +# 2. Unreal Engine slowly moves to newer Windows SDK. 4.27.0 no longer compiles with SDKs older than 18362 and even if it will be fixed in 4.27.x, +# this is just a question of a time when older SDKs support will be dropped completely +# 3. UE5 doesn't support VS2017 at all, so in the future that argument for continuing to use Windows SDK 17763 from VS2017 era will be weaker and weaker. +# +# We can't use newer SDK for VS2017 that is used to compile older engines because 18362 SDK support was only added in UE-4.23. +# +# See https://github.com/adamrehn/ue4-docker/issues/192 +# See https://forums.unrealengine.com/t/ndis_miniport_major_version-is-not-defined-error/135058 +# See https://github.com/EpicGames/UnrealEngine/blame/4.23.0-release/Engine/Source/Programs/UnrealBuildTool/Platform/Windows/UEBuildWindows.cs#L1822-L1823 +# See https://github.com/EpicGames/UnrealEngine/commit/ecc4872c3269e75a24adc40734cc8bcc9bbed1ca +# See https://udn.unrealengine.com/s/question/0D54z000079HcjJCAS/d3d12h427-error-c4668-winapipartitiongames-is-not-defined-as-a-preprocessor-macro-replacing-with-0-for-ifelif +# +# Keywords for Google: +# error C4668: 'NDIS_MINIPORT_MAJOR_VERSION' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif +# d3d12.h(427): error C4668: 'WINAPI_PARTITION_GAMES' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' +if ($visual_studio_build -eq "15") +{ + $windows_sdk_version = 17763 +} +else +{ + $windows_sdk_version = 20348 +} + +# NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks +Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "$env:TEMP\vs_buildtools.exe" + +# NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5 +# NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 +# NOTE: Microsoft.NetCore.Component.Runtime.3.1 is required by the AutomationTool tool and does not come installed with VS2022 so it needs targetting here. +$vs_args = @( + "--quiet", + "--wait", + "--norestart", + "--nocache", + "--installPath", "C:\BuildTools", + "--channelUri", "https://aka.ms/vs/$visual_studio_build/release/channel", + "--installChannelUri", "https://aka.ms/vs/$visual_studio_build/release/channel", + "--channelId", "VisualStudio.$visual_studio_build.Release", + "--productId", "Microsoft.VisualStudio.Product.BuildTools", + "--locale", "en-US", + "--add", "Microsoft.VisualStudio.Workload.VCTools", + "--add", "Microsoft.VisualStudio.Workload.MSBuildTools", + "--add", "Microsoft.VisualStudio.Component.NuGet", + "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "--add", "Microsoft.VisualStudio.Component.Windows10SDK.$windows_sdk_version", + "--add", "Microsoft.Net.Component.4.5.TargetingPack", + "--add", "Microsoft.Net.Component.4.6.2.TargetingPack", + "--add", "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites", + "--add", "Microsoft.NetCore.Component.SDK", + "--add", "Microsoft.NetCore.Component.Runtime.3.1" +) + +# Install the Visual Studio Build Tools workloads and components we need +Start-Process -FilePath "$env:TEMP\vs_buildtools.exe" -ArgumentList $vs_args -Wait + +# NOTE: Install the .Net 4.5 Framework Pack via NuGet as it is no longer available via Visual Studio, but still needed +# NOTE: some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 +Invoke-WebRequest -Uri "https://www.nuget.org/api/v2/package/Microsoft.NETFramework.ReferenceAssemblies.net45/1.0.3" -OutFile "$env:TEMP\DotNet45.zip" +Expand-Archive -Path "$env:TEMP\DotNet45.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "$env:TEMP\build\.NETFramework\v4.5\*" -Destination "C:\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\" -Recurse -Force + +# Clean up any temp files generated during prerequisite installation +Remove-Item -LiteralPath "$env:TEMP" -Recurse -Force +New-Item -Type directory -Path "$env:TEMP" + +# This shaves off ~300MB as of 2021-08-31 +choco-cleaner + +# Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file +# Just remove it, so a proper one will be generated on next NuGet run +# See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034 +if (Test-Path "$env:APPDATA\NuGet") +{ + Remove-Item -LiteralPath "$env:APPDATA\NuGet" -Recurse -Force +} + +# Display a human-readable completion message +Write-Output "Finished installing build prerequisites and cleaning up temporary files." From 4cc57b374195b5d90bfbd747b1ea6f8f0f407266 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 10 Dec 2022 21:37:55 +0300 Subject: [PATCH 292/430] resolves #290 tell users to enter access token --- ue4docker/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 3327c657..5a3e063b 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -28,7 +28,7 @@ def _getUsername(args): def _getPassword(args): return _getCredential( - args, "password", "UE4DOCKER_PASSWORD", lambda: getpass.getpass("Password: ") + args, "password", "UE4DOCKER_PASSWORD", lambda: getpass.getpass("Access token or password: ") ) From c4e8ecec9b0f1dfdee0d6fa96cd4592c6a016644 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 12:51:56 +0300 Subject: [PATCH 293/430] Fix codestyle --- ue4docker/build.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ue4docker/build.py b/ue4docker/build.py index 5a3e063b..af034845 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -28,7 +28,10 @@ def _getUsername(args): def _getPassword(args): return _getCredential( - args, "password", "UE4DOCKER_PASSWORD", lambda: getpass.getpass("Access token or password: ") + args, + "password", + "UE4DOCKER_PASSWORD", + lambda: getpass.getpass("Access token or password: "), ) From 407172e31d84a39adea893ac5ce69c98ae373593 Mon Sep 17 00:00:00 2001 From: CanisHelix <7948640+CanisHelix@users.noreply.github.com> Date: Sun, 11 Dec 2022 20:33:20 +0900 Subject: [PATCH 294/430] fix(package): patch filters to exclude hololens due to package error (#288) --- .../windows/patch-broken-releases.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index 177f8247..02fe54a2 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -53,3 +53,30 @@ def writeFile(filename, data): print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) else: print("PATCHED {}".format(gitdepsFile), file=sys.stderr) + + +# Determine if we are building UE 5.1.0 +if ( + versionDetails["MajorVersion"] == 5 + and versionDetails["MinorVersion"] == 1 + and versionDetails["PatchVersion"] == 0 +): + + # Hack InstalledEngineFilters.xml with the changes from CL 23300641 + # (See: ) + buildFile = join(engineRoot, "Engine", "Build", "InstalledEngineFilters.xml") + buildXml = readFile(buildFile) + if "HoloLens.Automation.json" not in buildXml: + + buildXml = buildXml.replace( + '', + '\n' + + " Engine\Saved\CsTools\Engine\Intermediate\ScriptModules\HoloLens.Automation.json\n", + ) + + writeFile(buildFile, buildXml) + + if verboseOutput == True: + print("PATCHED {}:\n\n{}".format(buildFile, buildFile), file=sys.stderr) + else: + print("PATCHED {}".format(buildFile), file=sys.stderr) From 4a332480efcb026cbeca56a1dcbee012cc5118f6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 15:30:17 +0300 Subject: [PATCH 295/430] Move CONTRIBING.md to repo root It is still consumable by GitHub, but now more discoverable by humans --- .github/CONTRIBUTING.md => CONTRIBUTING.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename .github/CONTRIBUTING.md => CONTRIBUTING.md (96%) diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 96% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md index 96ff826b..88752841 100644 --- a/.github/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,5 +25,5 @@ ue4-docker uses [Black](https://github.com/psf/black) for its code formatting. If you are submitting changes, make sure to install and run Black: - `pip install --user black` -- `black .` (invoked in repository root) +- `python -m black .` (invoke in repository root) - Now your code is properly formatted diff --git a/README.md b/README.md index 2b26e498..d40ec9c5 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Resources: ## Contributing -See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/.github/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. +See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. ## Legal From d384589276efb070f3da129b634f790e09b8088f Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 15:36:32 +0300 Subject: [PATCH 296/430] Import docs from docs.adamrehn.com --- .../building-images/advanced-build-options.md | 231 ++++++++++++++++++ .../available-container-images.md | 41 ++++ .../troubleshooting-build-issues.md | 118 +++++++++ docs/commands/build.md | 45 ++++ docs/commands/clean.md | 22 ++ docs/commands/diagnostics.md | 49 ++++ docs/commands/export.md | 54 ++++ docs/commands/info.md | 23 ++ docs/commands/list-of-commands.md | 15 ++ docs/commands/setup.md | 27 ++ docs/commands/test.md | 15 ++ docs/commands/version.md | 13 + docs/configuration/configuring-linux.md | 68 ++++++ docs/configuration/configuring-macos.md | 51 ++++ docs/configuration/configuring-windows-10.md | 71 ++++++ .../configuring-windows-server.md | 114 +++++++++ .../supported-host-configurations.md | 33 +++ .../frequently-asked-questions.md | 22 ++ .../introduction-to-ue4-docker.md | 47 ++++ .../large-container-images-primer.md | 20 ++ docs/read-these-first/nvidia-docker-primer.md | 6 + .../windows-container-primer.md | 14 ++ docs/use-cases/continuous-integration.md | 9 + docs/use-cases/linux-installed-builds.md | 11 + docs/use-cases/microservices.md | 10 + docs/use-cases/use-cases-overview.md | 15 ++ 26 files changed, 1144 insertions(+) create mode 100644 docs/building-images/advanced-build-options.md create mode 100644 docs/building-images/available-container-images.md create mode 100644 docs/building-images/troubleshooting-build-issues.md create mode 100644 docs/commands/build.md create mode 100644 docs/commands/clean.md create mode 100644 docs/commands/diagnostics.md create mode 100644 docs/commands/export.md create mode 100644 docs/commands/info.md create mode 100644 docs/commands/list-of-commands.md create mode 100644 docs/commands/setup.md create mode 100644 docs/commands/test.md create mode 100644 docs/commands/version.md create mode 100644 docs/configuration/configuring-linux.md create mode 100644 docs/configuration/configuring-macos.md create mode 100644 docs/configuration/configuring-windows-10.md create mode 100644 docs/configuration/configuring-windows-server.md create mode 100644 docs/configuration/supported-host-configurations.md create mode 100644 docs/read-these-first/frequently-asked-questions.md create mode 100644 docs/read-these-first/introduction-to-ue4-docker.md create mode 100644 docs/read-these-first/large-container-images-primer.md create mode 100644 docs/read-these-first/nvidia-docker-primer.md create mode 100644 docs/read-these-first/windows-container-primer.md create mode 100644 docs/use-cases/continuous-integration.md create mode 100644 docs/use-cases/linux-installed-builds.md create mode 100644 docs/use-cases/microservices.md create mode 100644 docs/use-cases/use-cases-overview.md diff --git a/docs/building-images/advanced-build-options.md b/docs/building-images/advanced-build-options.md new file mode 100644 index 00000000..3133d0b0 --- /dev/null +++ b/docs/building-images/advanced-build-options.md @@ -0,0 +1,231 @@ +--- +title: Advanced build options +pagenum: 2 +--- + +## General options + + +### Performing a dry run + +If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the [ue4-docker build](../commands/build) command. Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. + + +### Specifying a custom namespace for the image tags + +If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. **Note that this setting does not effect the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image**, which always retains its default name in order to maintain compatibility with the [prebuilt version from Docker Hub](#pulling-a-prebuilt-version-of-the-ue4-build-prerequisites-image). + + +### Specifying Git credentials + +The [ue4-docker build](../commands/build) command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: + +- **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. + +- **Environment variables**: the `UE4DOCKER_USERNAME` and `UE4DOCKER_PASSWORD` environment variables can be used to specify the username and password, respectively. Note that credentials specified via command-line arguments will take precedence over values defined in environment variables. + +- **Standard input**: if either the username or password has not been specified via a command-line argument or environment variable then the build command will prompt the user to enter the credential(s) for which values have not already been specified. + +Note that the username and password are handled independently, which means you can use different methods to specify the two credentials (e.g. username specified via command-line argument and password supplied via standard input.) + +Users who have enabled [Two-Factor Authentication (2FA)](https://help.github.com/en/articles/about-two-factor-authentication) for their GitHub account will need to generate a [personal access token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) and use that in place of their password. + + +### Building a custom version of the Unreal Engine + +If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. When building a custom Engine version, **both the repository URL and branch/tag must be specified**: + +{% highlight bash %} +ue4-docker build custom -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +{% endhighlight %} + +This will produce images tagged `adamrehn/ue4-source:custom`, `adamrehn/ue4-engine:custom`, etc. + +If you are performing multiple custom builds and wish to differentiate between them, it is recommended to also specify a name for the custom build: + +{% highlight bash %} +ue4-docker build custom:my-custom-build -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +{% endhighlight %} + +This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn/ue4-engine:my-custom-build`, etc. + + +### Pulling a prebuilt version of the `ue4-build-prerequisites` image + +If you would like to pull the [prebuilt version](https://hub.docker.com/r/adamrehn/ue4-build-prerequisites) of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image from Docker Hub instead of building it locally, simply specify the `--pull-prerequisites` flag when invoking the [ue4-docker build](../commands/build) command. This is primarily useful when building images under versions of Windows Server Core prior to Windows Server 2019, since using the prebuilt image allows you to avoid [copying the required DLL files from Windows 10](../configuration/configuring-windows-server#step-5-copy-required-dll-files-windows-server-core-version-1803-and-older-only). Note however that prebuilt versions of the image are available only for [LTSC versions of Windows](https://docs.microsoft.com/en-us/windows-server/get-started-19/servicing-channels-19), not for SAC versions. + + +### Excluding Engine components to reduce the final image size + +Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the [ue4-docker build](../commands/build) command to specify that certain Engine components should be excluded from the [ue4-minimal](../building-images/available-container-images#ue4-minimal) and [ue4-full](../building-images/available-container-images#ue4-full) images. The following components can be excluded: + +- `ddc`: disables building the DDC for the Engine. This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. + +- `debug`: removes all debug symbols from the built images. (When building Windows containers the files are actually truncated instead of removed, so they still exist but have a size of zero bytes. This is done for compatibility reasons.) + +- `templates`: removes the template projects and samples that ship with the Engine. + +You can specify the `--exclude` flag multiple times to exclude as many components as you like. For example: + +{% highlight bash %} +# Excludes both debug symbols and template projects +ue4-docker build 4.21.2 --exclude debug --exclude templates +{% endhighlight %} + + +### Enabling system resource monitoring during builds + +Starting in ue4-docker version 0.0.46, you can use the `--monitor` flag to enable a background thread that will log information about system resource usage (available disk space and memory, CPU usage, etc.) at intervals during the build. You can also use the `-interval` flag to override the default interval of 20 seconds: + +{% highlight bash %} +# Logs system resource levels every 20 seconds +ue4-docker build 4.24.2 --monitor + +# Logs system resource levels every 5 seconds +ue4-docker build 4.24.2 --monitor -interval=5 +{% endhighlight %} + + +### Exporting generated Dockerfiles + +Since ue4-docker version 0.0.78, the [ue4-docker build](../commands/build) command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a [multi-stage build](https://docs.docker.com/develop/develop-images/multistage-build/). You can use these flags like so: + +{% highlight bash %} +# Exports Dockerfiles for all images to the specified filesystem directory +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" + +# Exports Dockerfiles for all images except the ue4-engine image +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --no-engine + +# Exports Dockerfiles for all images except the ue4-engine image and combines them into a single Dockerfile +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --no-engine --combine +{% endhighlight %} + +Exporting Dockerfiles is useful for debugging or contributing to the development of ue4-docker itself. You can also use the generated Dockerfiles to build container images independently of ue4-docker, but only under the following circumstances: + +- When building Windows container images, you must specify the [advanced option](#advanced-options-for-dockerfile-generation) `source_mode` and set it to `copy`. This generates Dockerfiles that copy the Unreal Engine source code from the host filesystem rather than cloning it from a git repository, thus eliminating the dependency on ue4-docker's credential endpoint to securely provide git credentials and allowing container images to be built without the need for ue4-docker itself. + +- When building Linux container images, you must either set the [advanced option](#advanced-options-for-dockerfile-generation) `source_mode` to `copy` as detailed above, or else specify the `credential_mode` option and set it to `secrets`. This generates Dockerfiles that use the Linux-only [BuildKit build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) functionality to securely provide git credentials, eliminating the dependency on ue4-docker's credential endpoint whilst still facilitating the use of a git repository to provide the Unreal Engine source code. + + +### Advanced options for Dockerfile generation + +{% include alerts/info.html content="Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in [Jinja identifiers](https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers)." title="Title" %} + +Since ue4-docker version 0.0.78, the [ue4-docker build](../commands/build) command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying [Jinja templating engine](https://jinja.palletsprojects.com/) used to generate Dockerfiles. Some of these options (such as `source_mode`) can only be used when [exporting generated Dockerfiles](#exporting-generated-dockerfiles), whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: + +- **`source_mode`**: *(string)* controls how the [ue4-source](../building-images/available-container-images#ue4-source) Dockerfile obtains the source code for the Unreal Engine. Valid options are: + + - `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. This is the only mode that can be used when not [exporting generated Dockerfiles](#exporting-generated-dockerfiles). + + - `copy`: copies the Unreal Engine source code from the host filesystem. The filesystem path can be specified using the `SOURCE_LOCATION` Docker build argument, and of course must be a child path of the build context. + +- **`credential_mode`**: *(string)* controls how the [ue4-source](../building-images/available-container-images#ue4-source) Dockerfile securely obtains credentials for authenticating with remote git repositories when `source_mode` is set to `git`. Valid options are: + + - `endpoint`: the default mode, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the [ue4-source](../building-images/available-container-images#ue4-source) container during the build process by way of a Docker build argument. This mode will not work when [exporting generated Dockerfiles](#exporting-generated-dockerfiles), since the credential endpoint will not be available during the build process. + + - `secrets`: **(Linux containers only)** uses [BuildKit build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to securely inject the git credentials into the [ue4-source](../building-images/available-container-images#ue4-source) container during the build process. + +- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the [BuildGraph system](https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html) when creating an Installed Build of the Unreal Engine in the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image. + +- **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. This includes the labels which specify the [components excluded from the ue4-minimal image](#excluding-engine-components-to-reduce-the-final-image-size) as well as the sentinel labels that the [ue4-docker clean](../commands/clean) command uses to identify container images, and will therefore break the functionality of that command. + +- **`disable_all_patches`**: *(boolean)* disables all of the patches that ue4-docker ordinarily applies to the Unreal Engine source code. This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. It is typically safer to disable individual patches using the specific flag for each patch instead of simply disabling everything: + + - **`disable_release_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to versions of the Unreal Engine which are known to contain bugs, such as Unreal Engine 4.25.4. This will obviously break the build process when building these known broken releases, but will have no effect when building other versions of the Unreal Engine. + + - **`disable_windows_setup_patch`**: *(boolean)* prevents ue4-docker from patching `Setup.bat` under Windows to comment out the calls to the Unreal Engine prerequisites installer and UnrealVersionSelector, both of which are known to cause issues during the build process for Windows containers. + + - **`disable_linker_fixup`**: *(boolean)* prevents ue4-docker from replacing the linker in the Unreal Engine's bundled toolchain with a symbolic link to the system linker under Linux. + + - **`disable_example_platform_cleanup`**: *(boolean)* prevents ue4-docker from removing the `Engine/Platforms/XXX` directory that was introduced in Unreal Engine 4.24.0 and subsequently removed in Unreal Engine 4.26.0. This directory represents a "dummy" target platform for demonstration purposes, and the presence of this directory will typically break the build process. + + - **`disable_ubt_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix bugs in UnrealBuildTool (UBT) under various versions of the Unreal Engine. + + - **`disable_opengl_patch`**: *(boolean)* prevents ue4-docker from attempting to re-enable the OpenGL RHI under Linux for versions of the Unreal Engine in which it is present but deprecated. + + - **`disable_buildgraph_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to the BuildGraph XML files used to create an Installed Build of the Unreal Engine. These patches fix various bugs under both Windows and Linux across multiple versions of the Unreal Engine. + + - **`disable_target_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix broken `PlatformType` fields for client and server targets in `BaseEngine.ini` under Unreal Engine versions where these values are set incorrectly. + + - **`disable_unrealpak_copy`**: *(boolean)* prevents ue4-docker from ensuring the UnrealPak tool is correctly copied into Installed Builds of the Unreal Engine under Linux. Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. + + - **`disable_toolchain_copy`**: *(boolean)* prevents ue4-docker from ensuring the bundled clang toolchain is correctly copied into Installed Builds of the Unreal Engine under Linux. Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. + + +## Windows-specific options + + +### Specifying the Windows Server Core base image tag + +{% capture _alert_content %} +The `-basetag` flag controls how the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image is built and tagged, which has a flow-on effect to all of the other images. If you are building multiple related images over separate invocations of the build command (e.g. building the [ue4-source](../building-images/available-container-images#ue4-source) image in one command and then subsequently building the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +{% endcapture %} +{% include alerts/info.html content=_alert_content %} + +By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. (See the [Windows Container Version Compatibility](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility) page for a table detailing which configurations are supported.) + +If you are building images with the intention of subsequently running them under an older version of Windows 10 or Windows Server, you will need to build images based on the same kernel version as the target system (or older.) The kernel version can be specified by providing the appropriate base OS image tag via the `-basetag=TAG` flag when invoking the build command: + +{% highlight bash %} +ue4-docker build 4.20.3 -basetag=ltsc2016 # Uses Windows Server 2016 (Long Term Support Channel) +{% endhighlight %} + +For a list of supported base image tags, see the [Windows Server Core base image on Docker Hub](https://hub.docker.com/r/microsoft/windowsservercore/). + + +### Specifying the isolation mode under Windows + +The isolation mode can be explicitly specified via the `-isolation=MODE` flag when invoking the build command. Valid values are `process` (supported under Windows Server and [Windows 10 version 1809 or newer](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional)) or `hyperv` (supported under both Windows 10 and Windows Server.) If you do not explicitly specify an isolation mode then the appropriate default for the host system will be used. + +### Specifying Visual Studio Build Tools version under Windows + +By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. +Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. +To do so, pass `--visual-studio=2019` flag when invoking the build command. + +### Specifying the directory from which to copy required Windows DLL files + +By default, DLL files are copied from `%SystemRoot%\System32`. However, when building container images with an older kernel version than the host, the copied DLL files will be too new and the container OS will refuse to load them. A custom directory containing the correct DLL files for the container kernel version can be specified via the `-dlldir=DIR` flag when invoking the build command. + + +### Keeping or excluding Installed Build debug symbols under Windows + +{% capture _alert_content %} +Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. See [this section of the Troubleshooting Build Issues page](./troubleshooting-build-issues#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image) for further details on this issue. +{% endcapture %} +{% include alerts/warning.html content=_alert_content %} + +Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the [ue4-minimal](../building-images/available-container-images#ue4-minimal) Windows image. This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. (This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) + +Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section [Excluding Engine components to reduce the final image size](#excluding-engine-components-to-reduce-the-final-image-size). + + +### Building Linux container images under Windows + +By default, Windows container images are built when running the build command under Windows. To build Linux container images instead, simply specify the `--linux` flag when invoking the build command. + + +## Linux-specific options + + +### Enabling CUDA support for GPU-enabled Linux images + +{% capture _alert_content %} +The `--cuda` flag controls how the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image is built and tagged, which has a flow-on effect to all of the other images. If you are building multiple related images over separate invocations of the build command (e.g. building the [ue4-source](../building-images/available-container-images#ue4-source) image in one command and then subsequently building the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +{% endcapture %} +{% include alerts/info.html content=_alert_content %} + +By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. + +You can also control the version of the CUDA base image that is used by appending a version number when specifying the `--cuda` flag, as demonstrated below: + +{% highlight bash %} +# Uses the default CUDA base image (currently CUDA 9.2) +ue4-docker build RELEASE --cuda + +# Uses the CUDA 10.0 base image +ue4-docker build RELEASE --cuda=10.0 +{% endhighlight %} + +For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the [nvidia/cudagl](https://hub.docker.com/r/nvidia/cudagl/) base image. diff --git a/docs/building-images/available-container-images.md b/docs/building-images/available-container-images.md new file mode 100644 index 00000000..a66eef46 --- /dev/null +++ b/docs/building-images/available-container-images.md @@ -0,0 +1,41 @@ +--- +title: List of available container images +pagenum: 1 +--- + +You can build the following images using the [ue4-docker build](../commands/build) command: + +* TOC +{:toc} + +By default, all available images will be built. You can prevent unwanted images from being built by appending the relevant image-specific flag to the build command (see the *"Flag to disable"* entry for each image below.) + + +{% for image in site.data.ue4-docker.images %} +## {{ image.name | escape }} + +**Tags:** + +{% for tag in image.tags %} +- {{ tag }} +{% endfor %} + +{::nomarkdown} +

+ Dockerfiles: + Windows + | + Linux +

+{:/} + +**Flag to disable:** {{ image.disable }} + +**Contents:** {{ image.contents }} + +**Uses:** + +{% for use in image.uses %}- {{ use }} +{% endfor %} + +{% endfor %} diff --git a/docs/building-images/troubleshooting-build-issues.md b/docs/building-images/troubleshooting-build-issues.md new file mode 100644 index 00000000..69574c72 --- /dev/null +++ b/docs/building-images/troubleshooting-build-issues.md @@ -0,0 +1,118 @@ +--- +title: Troubleshooting build issues +pagenum: 3 +--- + +## General issues + +### Building the `ue4-build-prerequisites` image fails with a network-related error + +This indicates an underlying network or proxy server issue outside of ue4-docker itself that you will need to troubleshoot. You can use the [ue4-docker diagnostics](../commands/diagnostics) command to test container network connectivity during the troubleshooting process. Here are some steps to try: + +- If your host system accesses the network through a proxy server, make sure that [Docker is configured to use the correct proxy server settings](https://docs.docker.com/network/proxy/). +- If DNS resolution is failing then try adding the following entry to your [Docker daemon configuration file](https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file) (accessed through the Docker Engine settings pane if you're using Docker Desktop) and restarting the daemon: + +```json +"dns": ["8.8.8.8"] +``` + + +### Cloning the UnrealEngine Git repository fails with the message `error: unable to read askpass response from 'C:\git-credential-helper.bat'` (for Windows containers) or `'/tmp/git-credential-helper.sh'` (for Linux containers) + +This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. (Under Windows Server, the [ue4-docker setup](../commands/setup) command can configure the firewall rule for you automatically.) + + +### Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file + +This is typically caused by insufficient available disk space. To fix this, simply free up some disk space and run the build again. Running [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/) can be helpful for freeing up space occupied by untagged images. Note that restarting the Docker daemon and/or rebooting the host system may also help, since some versions of Docker have a bug that results in the amount of required disk space slowly increasing as more and more builds are run. + + +### Building Windows containers fails with the message `hcsshim::ImportLayer failed in Win32: The system cannot find the path specified` or building Linux containers fails with a message about insufficient disk space + +Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. + +- For Windows containers, follow [the instructions provided by Microsoft](https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size), making sure you restart the Docker daemon after you've modified the config JSON. (Under Windows Server, the [ue4-docker setup](../commands/setup) command can configure this for you automatically.) +- For Linux containers, use the [Docker for Windows "Advanced" settings tab](https://docs.docker.com/docker-for-windows/#advanced) under Windows or the [Docker for Mac "Disk" settings tab](https://docs.docker.com/docker-for-mac/#disk) under macOS. + + +### Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image + +{% capture _alert_content %} +Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. Since ue4-docker version 0.0.47, you can use the [ue4-docker diagnostics](../commands/diagnostics) command to check whether the Docker daemon on your system suffers from this issue. If it does, you may need to [exclude debug symbols](./advanced-build-options#excluding-engine-components-to-reduce-the-final-image-size) when building Windows images. +{% endcapture %} +{% include alerts/warning.html content=_alert_content %} + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: + +- (affects all platforms) +- (affects Windows containers only) + +[#37581](https://github.com/moby/moby/issues/37581) was [fixed](https://github.com/moby/moby/pull/37771) in Docker CE 18.09.0, whilst [#40444](https://github.com/moby/moby/issues/40444) was [fixed](https://github.com/moby/moby/pull/41430) in Docker CE 20.10.0. + +If you are using a version of Docker that contains one of these bugs then you will need to [exclude debug symbols](./advanced-build-options#excluding-engine-components-to-reduce-the-final-image-size), which reduces the size of the Installed Build below the 8GiB threshold. If debug symbols are required then it will be necessary to upgrade or downgrade to a version of Docker that does not suffer from the 8GiB size limit issue (although finding such a version under Windows may prove quite difficult.) + + +## Linux-specific issues + +### Building the Engine in a Linux container fails with an error indicating that a compatible version of clang cannot be found or the file `ToolchainVersion.txt` is missing + +This is typically caused by the download of the Unreal Engine's toolchain archive from the Epic Games CDN becoming corrupted and failing to extract correctly. This issue can occur both inside containers and when running directly on a host system, and the fix is to simply delete the corrupted files and try again: + +1. Untag the [ue4-source](./available-container-images#ue4-source) image. +2. Clear the Docker filesystem layer cache by running [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/). +3. Re-run the [ue4-docker build](../commands/build) command. + + +## Windows-specific issues + + +### Building the `ue4-build-prerequisites` image fails with an unknown error + +Microsoft issued a security update in February 2020 that [broke container compatibility for all versions of Windows Server and caused 32-bit applications to fail silently when run](https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t). The issue is resolved by ensuring that both the host system and the container image are using versions of Windows that incorporate the fix: + +- Make sure your host system is up-to-date and all available Windows updates are installed. +- Make sure you are using the latest version of ue4-docker, which automatically uses container images that incorporate the fix. + + +### Building Windows containers fails with the message `hcsshim: timeout waiting for notification extra info` or the message `This operation ended because the timeout has expired` + +Recent versions of Docker under Windows may sometimes encounter the error [hcsshim: timeout waiting for notification extra info](https://github.com/Microsoft/hcsshim/issues/152) when building or running Windows containers. This is a known issue when using Windows containers in [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container). At the time of writing, Microsoft have stated that they are aware of the problem, but an official fix is yet to be released. + +As a workaround until a proper fix is issued, it seems that altering the memory limit for containers between subsequent invocations of the `docker` command can reduce the frequency with which this error occurs. (Changing the memory limit when using Hyper-V isolation likely forces Docker to provision a new Hyper-V VM, preventing it from re-using an existing one that has become unresponsive.) Please note that this workaround has been devised based on my own testing under Windows 10 and may not hold true when using Hyper-V isolation under Windows Server. + +To enable the workaround, specify the `--random-memory` flag when invoking the build command. This will set the container memory limit to a random value between 10GB and 12GB when the build command starts. If a build fails with the `hcsshim` timeout error, simply re-run the build command and in most cases the build will continue successfully, even if only for a short while. Restarting the Docker daemon may also help. + +Note that some older versions of UnrealBuildTool will crash with an error stating *"The process cannot access the file because it is being used by another process"* when using a memory limit that is not a multiple of 4GB. If this happens, simply run the build command again with an appropriate memory limit (e.g. `-m 8GB` or `-m 12GB`.) If the access error occurs even when using an appropriate memory limit, this likely indicates that Windows is unable to allocate the full amount of memory to the container. Rebooting the host system may help to alleviate this issue. + + +### Building or running Windows containers fails with the message `The operating system of the container does not match the operating system of the host` + +This error is shown in two situations: + +- The host system is running an **older kernel version** than the container image. In this case, you will need to build the images using the same kernel version as the host system or older. See [Specifying the Windows Server Core base image tag](./advanced-build-options#specifying-the-windows-server-core-base-image-tag) for details on specifying the correct kernel version when building Windows container images. +- The host system is running a **newer kernel version** than the container image and you are attempting to use process isolation mode instead of Hyper-V isolation mode. (Process isolation mode is the default under Windows Server.) In this case, you will need to use Hyper-V isolation mode instead. See [Specifying the isolation mode under Windows](./advanced-build-options#specifying-the-isolation-mode-under-windows) for details on how to do this. + + +### Pulling the .NET Framework base image fails with the message `ProcessUtilityVMImage \\?\`(long path here)`\UtilityVM: The system cannot find the path specified` + +This is a known issue when the host system is running an older kernel version than the container image. Just like in the case of the *"The operating system of the container does not match the operating system of the host"* error mentioned above, you will need to build the images using the same kernel version as the host system or older. See [Specifying the Windows Server Core base image tag](./advanced-build-options#specifying-the-windows-server-core-base-image-tag) for details on specifying the correct kernel version when building Windows container images. + + +### Building the Engine in a Windows container fails with the message `The process cannot access the file because it is being used by another process` + +This is a known bug in some older versions of UnrealBuildTool when using a memory limit that is not a multiple of 4GB. To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the [`hcsshim` timeout issues](#building-windows-containers-fails-with-the-message-hcsshim-timeout-waiting-for-notification-extra-info-or-the-message-this-operation-ended-because-the-timeout-has-expired) section. + + +### Building the Engine in a Windows container fails with the message `fatal error LNK1318: Unexpected PDB error; OK (0)` + +This is a known bug in some versions of Visual Studio which only appears to occur intermittently. The simplest fix is to simply reboot the host system and then re-run the build command. Insufficient available memory may also contribute to triggering this bug. Note that a linker wrapper [was added in Unreal Engine 4.24.0](https://docs.unrealengine.com/en-US/Support/Builds/ReleaseNotes/4_24/index.html) to automatically retry link operations in the event that this bug occurs, so it shouldn't be an issue when building version 4.24.0 or newer. + + +### Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` + +This error typically occurs when the Windows pagefile size is not large enough. As stated in the [Windows containers primer](../read-these-first/windows-container-primer#hyper-v-isolation-mode-issues), there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. However, containers running in process isolation mode will use the pagefile settings of the host system. When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. (Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) + + +### Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system + +As described in the [Windows containers primer](../read-these-first/windows-container-primer#hyper-v-isolation-mode-issues), the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/commands/build.md b/docs/commands/build.md new file mode 100644 index 00000000..9d833934 --- /dev/null +++ b/docs/commands/build.md @@ -0,0 +1,45 @@ +--- +title: ue4-docker build +blurb: Builds container images for a specific version of UE4. +pagenum: 2 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker build [-h] [--linux] [--rebuild] [--dry-run] + [--pull-prerequisites] [--no-engine] [--no-minimal] [--no-full] + [--no-cache] [--random-memory] + [--exclude {debug,templates}] [--opt OPT] [--cuda VERSION] + [-username USERNAME] [-password PASSWORD] + [-repo REPO] [-branch BRANCH] [-isolation ISOLATION] + [-basetag BASETAG] [-dlldir DLLDIR] [-suffix SUFFIX] + [-m M] + [-ue4cli UE4CLI] [-conan-ue4cli CONAN_UE4CLI] + [-layout LAYOUT] [--combine] + [--monitor] [-interval INTERVAL] + [--ignore-blacklist] + [--visual-studio VISUAL_STUDIO] + [-v] + release +{% endhighlight %} + + +## Basic usage + +To build container images for a specific version of the Unreal Engine, simply specify the UE4 release that you would like to build using full [semver](https://semver.org/) version syntax. For example, to build Unreal Engine 4.20.3, run: + +{% highlight bash %} +ue4-docker build 4.20.3 +{% endhighlight %} + +You will be prompted for the Git credentials to be used when cloning the UE4 GitHub repository (this will be the GitHub username and password you normally use when cloning .) The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. + +By default, all available images will be built. See the [List of available container images](../building-images/available-container-images) page for details on customising which images are built. + + +## Advanced usage + +See the [Advanced build options](../building-images/advanced-build-options) page for details on all of the supported flags for customising build behaviour. diff --git a/docs/commands/clean.md b/docs/commands/clean.md new file mode 100644 index 00000000..6b95e771 --- /dev/null +++ b/docs/commands/clean.md @@ -0,0 +1,22 @@ +--- +title: ue4-docker clean +blurb: Cleans built container images. +pagenum: 3 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker clean [-tag TAG] [--source] [--engine] [--all] [--dry-run] +{% endhighlight %} + +By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed. You can customise this behaviour using these flags: + +- `-tag TAG`: Applies a filter for the three flags below, restricting them to removing only images with the specified tag (e.g. `-tag 4.21.0` will only remove images for 4.21.0.) +- `--source`: Removes [ue4-source](../building-images/available-container-images#ue4-source) images, applying the tag filter if one was specified +- `--engine`: Removes [ue4-engine](../building-images/available-container-images#ue4-engine) images, applying the tag filter if one was specified +- `--all`: Removes all ue4-docker images, applying the tag filter if one was specified + +If you're unsure as to exactly what images will be removed by a given invocation of the command, append the `--dry-run` flag to have ue4-docker print the generated `docker rmi` commands instead of running them. \ No newline at end of file diff --git a/docs/commands/diagnostics.md b/docs/commands/diagnostics.md new file mode 100644 index 00000000..afe623c5 --- /dev/null +++ b/docs/commands/diagnostics.md @@ -0,0 +1,49 @@ +--- +title: ue4-docker diagnostics +blurb: Runs diagnostics to detect issues with the host system configuration. +pagenum: 4 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker diagnostics DIAGNOSTIC +{% endhighlight %} + +This command can be used to run the following diagnostics: + +* TOC +{:toc} + + +## Checking for the Docker 8GiB filesystem layer bug + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, [breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols](../building-images/troubleshooting-build-issues#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image). + +This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: + +{% highlight bash %} +ue4-docker diagnostics 8gig +{% endhighlight %} + + +## Checking for the Windows Host Compute Service (HCS) `storage-opt` bug + +Windows Server versions 1903 and 1909 and Windows 10 versions 1903 and 1909 contain [a bug in the Host Compute Service (HCS)](https://github.com/docker/for-win/issues/4100) that prevents users from increasing the maximum allowed image size using Docker's `storage-opt` configuration key. Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the [ue4-docker build](./build) command from functioning correctly on affected systems. + +This diagnostic tests whether the host system is affected this bug, by attempting to run a container with a non-default `storage-opt` value: + +{% highlight bash %} +ue4-docker diagnostics maxsize +{% endhighlight %} + + +## Checking for container network connectivity issues + +This diagnostic tests whether running containers are able to access the internet, resolve DNS entries, and download remote files: + +{% highlight bash %} +ue4-docker diagnostics network +{% endhighlight %} diff --git a/docs/commands/export.md b/docs/commands/export.md new file mode 100644 index 00000000..4df4ff62 --- /dev/null +++ b/docs/commands/export.md @@ -0,0 +1,54 @@ +--- +title: ue4-docker export +blurb: Exports components from built container images to the host system. +pagenum: 5 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker export COMPONENT TAG DESTINATION +{% endhighlight %} + +This command can be used to export the following components: + +* TOC +{:toc} + + +## Exporting Installed Builds under Linux + +Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the [ue4-minimal](../building-images/available-container-images#ue4-minimal) or [ue4-full](../building-images/available-container-images#ue4-full) image for the UE4 version that you want to export, you can export it to the host system like so: + +{% highlight bash %} +# Example: specify a version without a full image tag (assumes `adamrehn/ue4-full`) +# Exports the Installed Build from `adamrehn/ue4-full:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "4.27.0" ~/UnrealInstalled + +# Example: specify a full image tag +# Exports the Installed Build from `adamrehn/ue4-minimal:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "adamrehn/ue4-minimal:4.27.0" ~/UnrealInstalled + +# Example: use a container image based on ue4-minimal with a completely different tag +# Exports the Installed Build from `ghcr.io/epicgames/unreal-engine:dev-4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/UnrealInstalled +{% endhighlight %} + + +## Exporting Conan packages + +The Conan wrapper packages generated by [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}) can be exported from the [ue4-full](../building-images/available-container-images#ue4-full) image to the local Conan package cache on the host system like so: + +{% highlight bash %} +ue4-docker export packages 4.27.0 cache +{% endhighlight %} + +[Conan](https://conan.io/) will need to be installed on the host system for this to work. To use the exported packages for development on the host system, you will also need to generate the accompanying profile-wide packages by running the command: + +{% highlight bash %} +ue4 conan generate --profile-only +{% endhighlight %} + +This will require both [ue4cli]({{ site.data.common.projects.ue4cli.repo }}) and [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}) to be installed on the host system. diff --git a/docs/commands/info.md b/docs/commands/info.md new file mode 100644 index 00000000..7a82c9f3 --- /dev/null +++ b/docs/commands/info.md @@ -0,0 +1,23 @@ +--- +title: ue4-docker info +blurb: Displays information about the host system and Docker daemon. +pagenum: 6 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker info +{% endhighlight %} + +The command will output the following information: + +- The ue4-docker version +- The host OS version +- The Docker daemon version +- Whether or not the NVIDIA Container Toolkit is supported under the current host configuration +- The detected configuration value for the maximum image size for Windows containers +- The total amount of detected system memory +- The number of detected physical and logical CPUs diff --git a/docs/commands/list-of-commands.md b/docs/commands/list-of-commands.md new file mode 100644 index 00000000..f2110f06 --- /dev/null +++ b/docs/commands/list-of-commands.md @@ -0,0 +1,15 @@ +--- +title: List of ue4-docker commands +pagenum: 1 +--- + +The ue4-docker command line interface exposes the following commands: + +{::nomarkdown} +{% assign commands = site.documents | where: "subsite", page.subsite | where: "chapter", page.chapter | where_exp: "page", "page.pagenum != null" | where_exp: "page", "page.pagenum > 1" | sort: "pagenum" %} + +{:/} diff --git a/docs/commands/setup.md b/docs/commands/setup.md new file mode 100644 index 00000000..0947bc32 --- /dev/null +++ b/docs/commands/setup.md @@ -0,0 +1,27 @@ +--- +title: ue4-docker setup +blurb: Automatically configures the host system where possible. +pagenum: 7 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker setup +{% endhighlight %} + +This command will automatically configure a Linux or Windows Server host system with the settings required in order to build and run containers produced by ue4-docker. + +**Under Linux:** + +- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image. + +**Under Windows Server:** + +- The Docker daemon will be configured to set the maximum image size for Windows containers to 400GB. +- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image. +- Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the [full Windows base image](https://hub.docker.com/_/microsoft-windows). Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. + +**Under Windows 10 and macOS** this command will print a message informing the user that automatic configuration is not supported under their platform and that they will need to configure the system manually. diff --git a/docs/commands/test.md b/docs/commands/test.md new file mode 100644 index 00000000..50bc2da2 --- /dev/null +++ b/docs/commands/test.md @@ -0,0 +1,15 @@ +--- +title: ue4-docker test +blurb: Runs tests to verify the correctness of built container images. +pagenum: 8 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker test TAG +{% endhighlight %} + +This command runs a suite of tests to verify that built [ue4-full](../building-images/available-container-images#ue4-full) container images are functioning correctly and can be used to build and package Unreal projects and plugins. This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/commands/version.md b/docs/commands/version.md new file mode 100644 index 00000000..01f3e41e --- /dev/null +++ b/docs/commands/version.md @@ -0,0 +1,13 @@ +--- +title: ue4-docker version +blurb: Prints the ue4-docker version number. +pagenum: 9 +--- + +{{ page.blurb | escape }} + +**Usage syntax:** + +{% highlight bash %} +ue4-docker version +{% endhighlight %} diff --git a/docs/configuration/configuring-linux.md b/docs/configuration/configuring-linux.md new file mode 100644 index 00000000..551e15fd --- /dev/null +++ b/docs/configuration/configuring-linux.md @@ -0,0 +1,68 @@ +--- +title: Configuring Linux +pagenum: 2 +--- + +## Requirements + +- 64-bit version of one of Docker's [supported Linux distributions](https://docs.docker.com/install/#supported-platforms) (CentOS 7+, Debian 7.7+, Fedora 26+, Ubuntu 14.04+) +- Minimum 8GB of RAM +- Minimum {{ site.data.ue4-docker.common.diskspace_linux | escape }} available disk space for building container images + + +## Step 1: Install Docker CE + +Follow the official installation instructions from the Docker Documentation for your distribution: + +- [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) +- [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) +- [Fedora](https://docs.docker.com/install/linux/docker-ce/fedora/) +- [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) + +Once Docker is installed, follow the instructions from the [Post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) page of the Docker Documentation to allow Docker commands to be run by a non-root user. This step is required in order to enable audio support when performing cloud rendering using the NVIDIA Container Toolkit. + + +## Step 2: Install Python 3.6 or newer + +{% capture _alert_content %} +Note that older versions of these Linux distributions may not have Python 3.6 available in their system repositories by default. When working with an older distribution it may be necessary to configure community repositories that provide newer versions of Python. +{% endcapture %} +{% include alerts/info.html content=_alert_content %} + +Under CentOS, run: + +{% highlight bash %} +sudo yum install python3 python3-devel python3-pip +{% endhighlight %} + +Under Debian and Ubuntu, run: + +{% highlight bash %} +sudo apt-get install python3 python3-dev python3-pip +{% endhighlight %} + +Under Fedora, run: + +{% highlight bash %} +sudo dnf install python3 python3-devel python3-pip +{% endhighlight %} + + +## Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command: + +{% highlight console %} +sudo pip3 install ue4-docker +{% endhighlight %} + + +## Step 4: Use ue4-docker to automatically configure the Linux firewall + +If the host system is running an active firewall that blocks access to port 9876 (required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image) then it is necessary to create a firewall rule to permit access to this port. The [ue4-docker setup](../commands/setup) command will detect this scenario and perform the appropriate firewall configuration automatically. Simply run: + +{% highlight console %} +sudo ue4-docker setup +{% endhighlight %} + +Note that the `iptables-persistent` service will need to be installed for the newly-created firewall rule to persist after the host system reboots. diff --git a/docs/configuration/configuring-macos.md b/docs/configuration/configuring-macos.md new file mode 100644 index 00000000..a4ff7fcc --- /dev/null +++ b/docs/configuration/configuring-macos.md @@ -0,0 +1,51 @@ +--- +title: Configuring macOS +pagenum: 5 +--- + +{% capture _alert_content %} +macOS provides a sub-optimal experience when running Linux containers, due to the following factors: + +- Linux containers are unable to use GPU acceleration via the [NVIDIA Container Toolkit](../read-these-first/nvidia-docker-primer). +{% endcapture %} +{% include alerts/warning.html content=_alert_content %} + + +## Requirements + +- 2010 or newer model Mac hardware +- macOS 10.10.3 Yosemite or newer +- Minimum 8GB of RAM +- Minimum {{ site.data.ue4-docker.common.diskspace_linux | escape }} available disk space for building container images + + +## Step 1: Install Docker CE for Mac + +Download and install [Docker CE for Mac from the Docker Store](https://store.docker.com/editions/community/docker-ce-desktop-mac). + + +## Step 2: Install Python 3 via Homebrew + +The simplest way to install Python 3 and pip under macOS is to use the [Homebrew package manager](https://brew.sh/). To do so, run the following commands from a Terminal prompt: + +{% highlight bash %} +# Install Homebrew +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" + +# Install Python +brew install python +{% endhighlight %} + + +## Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from a Terminal prompt: + +{% highlight console %} +sudo pip3 install ue4-docker +{% endhighlight %} + + +## Step 4: Manually configure Docker daemon settings + +Use [the Advanced section under the Resources tab of the Docker Desktop settings pane](https://docs.docker.com/desktop/mac/#resources) to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to {{ site.data.ue4-docker.common.diskspace_linux | escape }}. diff --git a/docs/configuration/configuring-windows-10.md b/docs/configuration/configuring-windows-10.md new file mode 100644 index 00000000..12d0b8f8 --- /dev/null +++ b/docs/configuration/configuring-windows-10.md @@ -0,0 +1,71 @@ +--- +title: Configuring Windows 10 and 11 +pagenum: 4 +--- + +{% capture _alert_content %} +Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. Using [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container) will result in a sub-optimal experience due to [several issues that impact performance and stability](../read-these-first/windows-container-primer). The default isolation mode depends on the specific version of Windows being used: + +- Under Windows 10, Hyper-V isolation mode is the default isolation mode and [process isolation mode must manually enabled](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-) each time a container is built or run. The [ue4-docker build](../commands/build) command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** + +- Under Windows 11, process isolation mode is the default isolation mode. +{% endcapture %} +{% include alerts/warning.html content=_alert_content %} + + +## Requirements + +- 64-bit Windows 10 Pro, Enterprise, or Education (Version 1607 or newer) +- Hardware-accelerated virtualisation enabled in the system BIOS/EFI +- Minimum 8GB of RAM +- Minimum {{ site.data.ue4-docker.common.diskspace_windows | escape }} available disk space for building container images + + +## Step 1: Install Docker CE for Windows + +Download and install [Docker CE for Windows from the Docker Store](https://store.docker.com/editions/community/docker-ce-desktop-windows). + + +## Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the [Chocolatey package manager](https://chocolatey.org/). To do so, run the following command from an elevated PowerShell prompt: + +{% highlight powershell %} +Set-ExecutionPolicy Bypass -Scope Process -Force; +iex ((New-Object System.Net.WebClient).DownloadString( + 'https://chocolatey.org/install.ps1' +)) +{% endhighlight %} + +You may need to restart your shell for it to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +{% highlight powershell %} +choco install -y python +{% endhighlight %} + + +## Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +{% highlight console %} +pip install ue4-docker +{% endhighlight %} + + +## Step 4: Manually configure Docker daemon settings + +For building and running Windows containers: + +- Configure the Docker daemon to [use Windows containers](https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers) rather than Linux containers. +- Configure the Docker daemon to increase the maximum container disk size from the default 20GB limit by following [the instructions provided by Microsoft](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits). The 120GB limit specified in the instructions is not quite enough, so set a 400GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** + +{% include alerts/warning.html content="**The ue4-docker maintainers do not provide support for building and running Linux containers under Windows**, due to the various technical limitations of the Hyper-V and WSL2 backends for Docker Desktop. (See [this issue](https://github.com/adamrehn/ue4-docker/issues/205) for details of these limitations.) This functionality is still present in ue4-docker for those who choose to use it, but users are solely responsible for troubleshooting any issues they encounter when doing so." %} + +For building and running Linux containers: + +- Configure the Docker daemon to [use Linux containers](https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers) rather than Windows containers. + +- **If you are using the Hyper-V backend** then use [the Advanced section under the Resources tab of the Docker Desktop settings pane](https://docs.docker.com/desktop/windows/#resources) to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to {{ site.data.ue4-docker.common.diskspace_linux | escape }}. + +- **If you are using the WSL2 backend** then [expand the WSL2 virtual hard disk](https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk) to at least {{ site.data.ue4-docker.common.diskspace_linux | escape }}. diff --git a/docs/configuration/configuring-windows-server.md b/docs/configuration/configuring-windows-server.md new file mode 100644 index 00000000..9226ade1 --- /dev/null +++ b/docs/configuration/configuring-windows-server.md @@ -0,0 +1,114 @@ +--- +title: Configuring Windows Server +pagenum: 3 +--- + +{% capture _alert_content %} +Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. Using [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container) will result in a sub-optimal experience due to [several issues that impact performance and stability](../read-these-first/windows-container-primer). Process isolation mode is the default isolation mode under Windows Server. +{% endcapture %} +{% include alerts/warning.html content=_alert_content %} + + +## Requirements + +- Windows Server 2016 or newer +- Minimum 8GB of RAM +- Minimum {{ site.data.ue4-docker.common.diskspace_windows | escape }} available disk space for building container images + + +## Step 1: Install Docker EE + +As per the instructions provided by the [Install Docker Engine - Enterprise on Windows Servers](https://docs.docker.com/install/windows/docker-ee/) page of the Docker Documentation, run the following commands from an elevated PowerShell prompt: + +{% highlight powershell %} +# Add the Docker provider to the PowerShell package manager +Install-Module DockerMsftProvider -Force + +# Install Docker EE +Install-Package Docker -ProviderName DockerMsftProvider -Force + +# Restart the computer to enable the containers feature +Restart-Computer +{% endhighlight %} + + +## Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the [Chocolatey package manager](https://chocolatey.org/). To do so, run the following command from an elevated PowerShell prompt: + +{% highlight powershell %} +Set-ExecutionPolicy Bypass -Scope Process -Force; +iex ((New-Object System.Net.WebClient).DownloadString( + 'https://chocolatey.org/install.ps1' +)) +{% endhighlight %} + +You may need to restart the system for your shell to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +{% highlight powershell %} +choco install -y python +{% endhighlight %} + + +## Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +{% highlight console %} +pip install ue4-docker +{% endhighlight %} + + +## Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall + +To automatically configure the required system settings, run the [ue4-docker setup](../commands/setup) command from an elevated Command Prompt: + +{% highlight console %} +ue4-docker setup +{% endhighlight %} + +This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image), and download any required DLL files under Windows Server version 1809 and newer. + + +## Step 5: Copy required DLL files (Windows Server Core version 1803 and older only) + +{% capture _alert_content %} +**This step is only required under when building the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image locally under older versions of Windows Server.** + +Starting from Windows Server version 1809 (base image tag `ltsc2019`), the [ue4-docker setup](../commands/setup) command will automatically extract the required DLL files from the [full Windows base image](https://hub.docker.com/_/microsoft-windowsfamily-windows). Manual configuration is only required under Windows Server version 1803 and older when you are building the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image locally rather than [pulling the prebuilt version](../building-images/advanced-build-options#pulling-a-prebuilt-version-of-the-ue4-build-prerequisites-image). +{% endcapture %} +{% include alerts/info.html content=_alert_content %} + +Under Windows Server Core, you will need to copy a number of DLL files from a copy of Windows 10 (or Windows Server [with the Desktop Experience feature](https://docs.microsoft.com/en-us/windows-server/get-started/getting-started-with-server-with-desktop-experience)) and place them in the `C:\Windows\System32` directory. Note that these DLL files must be copied from the **same version of Windows** as the Windows Server Core host system (e.g. Windows Server 1709 needs DLLs from Windows 10 1709, Windows Server 1803 needs DLLs from Windows 10 1803, etc.) Although DLLs from an older system version may potentially work, Windows containers will refuse to load these DLL files if they have been copied from a version of Windows that is newer than the container. + +The required DLL files for each version of Windows Server Core are listed in the tables below, along with the minimum and maximum versions of these DLLs that have been tested and are known to work. + +{% for server in site.data.ue4-docker.dlls %} +### {{ server.name | escape }} (base image tag `{{ server.basetag | escape }}`) + +{::nomarkdown} +{% if server.notes != "" %} + {% assign content = server.notes | markdownify %} + {% include alerts/info.html content=content %} +{% endif %} + + + + + + + + + + + {% for dll in server.dlls %} + + + + + + {% endfor %} + +
DLL FileMin VersionMax Version
{{ dll.name | escape }}{{ dll.minVersion | escape }}{{ dll.maxVersion | escape }}
+{:/} +{% endfor %} diff --git a/docs/configuration/supported-host-configurations.md b/docs/configuration/supported-host-configurations.md new file mode 100644 index 00000000..bdf22da6 --- /dev/null +++ b/docs/configuration/supported-host-configurations.md @@ -0,0 +1,33 @@ +--- +title: Supported host configurations +pagenum: 1 +--- + +The table below lists the host operating systems can be used to build and run the container images produced by ue4-docker, as well as which features are supported under each system. **Click on an operating system's name to view the configuration instructions for that platform.** + +{::nomarkdown} + + + + + + + + + + + + {% for host in site.data.ue4-docker.hosts %} + + + + + + + + {% endfor %} + +
Host OSLinux containersWindows containersNVIDIA Container ToolkitOptimality
{{ host.name | escape }}{{ host.linux | escape }}{{ host.windows | escape }}{{ host.gpu | escape }}{{ host.optimality | escape }}
+{:/} + +The ***Optimality*** column indicates whether a given host operating system provides the best experience for running the container types that it supports. The configuration instructions page for each operating system provides further details regarding the factors that make it either optimal or sub-optimal. diff --git a/docs/read-these-first/frequently-asked-questions.md b/docs/read-these-first/frequently-asked-questions.md new file mode 100644 index 00000000..873974d5 --- /dev/null +++ b/docs/read-these-first/frequently-asked-questions.md @@ -0,0 +1,22 @@ +--- +title: Frequently Asked Questions +pagenum: 5 +--- + +## Why are the Dockerfiles written in such an inefficient manner? There are a large number of `RUN` directives that could be combined to improve both build efficiency and overall image size. + +With the exception of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) and [ue4-minimal](../building-images/available-container-images#ue4-minimal) images, the Dockerfiles have been deliberately written in an inefficient way because doing so serves two very important purposes. + +The first purpose is self-documentation. These Docker images are the first publicly-available Windows and Linux images to provide comprehensive build capabilities for Unreal Engine 4. Along with the supporting documentation and [articles on adamrehn.com](https://adamrehn.com/articles/tag/Unreal%20Engine/), the code in this repository represents an important source of information regarding the steps that must be taken to get UE4 working correctly inside a container. The readability of the Dockerfiles is key, which is why they contain so many individual `RUN` directives with explanatory comments. Combining `RUN` directives would reduce readability and potentially obfuscate the significance of critical steps. + +The second purpose is debuggability. Updating the Dockerfiles to ensure compatibility with new Unreal Engine releases is an extremely involved process that typically requires building the Engine many times over. By breaking the Dockerfiles into many fine-grained `RUN` directives, the Docker build cache can be leveraged to ensure only the failing steps need to be repeated when rebuilding the images during debugging. Combining `RUN` directives would increase the amount of processing that needs to be redone each time one of the commands in a given directive fails, significantly increasing overall debugging times. + + +## Can Windows containers be used to perform cloud rendering in the same manner as Linux containers with the NVIDIA Container Toolkit? + +See the answer here: + + +## Is it possible to build Unreal projects for macOS or iOS using the Docker containers? + +Building projects for macOS or iOS requires a copy of macOS and Xcode. Since macOS cannot run inside a Docker container, there is unfortunately no way to perform macOS or iOS builds using Docker containers. diff --git a/docs/read-these-first/introduction-to-ue4-docker.md b/docs/read-these-first/introduction-to-ue4-docker.md new file mode 100644 index 00000000..52aa6cb6 --- /dev/null +++ b/docs/read-these-first/introduction-to-ue4-docker.md @@ -0,0 +1,47 @@ +--- +title: Introduction to ue4-docker +pagenum: 1 +--- + +## Overview + +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli]({{ site.data.common.projects.ue4cli.repo }}), [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}), and [ue4-ci-helpers]({{ site.data.common.projects.ue4-ci-helpers.repo }}) to facilitate a wide variety of use cases. + +Key features include: + +- Unreal Engine 4.20.0 and newer is supported. +- Both Windows containers and Linux containers are supported. +- Building and packaging UE4 projects is supported. +- Running automation tests is supported. +- Running built UE4 projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. + + +## Important legal notice + +**With the exception of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. As per Section 1A of the [Unreal Engine EULA](https://www.unrealengine.com/eula), Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games UE4 GitHub repository.** {::nomarkdown}Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.{:/} **It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** + +For more details, see the [Unreal Engine EULA Restrictions](https://unrealcontainers.com/docs/obtaining-images/eula-restrictions) page on the [Unreal Containers community hub](https://unrealcontainers.com/). + + +## Getting started + +Multi-purpose Docker images for large, cross-platform projects such as the Unreal Engine involve a great deal more complexity than most typical Docker images. Before you start using ue4-docker it may be helpful to familiarise yourself with some of these complexities: + +- If you've never built large (multi-gigabyte) Docker images before, be sure to read the [Large container images primer](./large-container-images-primer). +- If you've never used Windows containers before, be sure to read the [Windows containers primer](./windows-container-primer). +- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the [NVIDIA Container Toolkit primer](./nvidia-docker-primer). + +Once you're familiar with all of the relevant background material, you can dive right in: + +1. First up, head to the [Configuration](../configuration) section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. +2. Next, check out the [Use Cases](../use-cases) section for details on the various scenarios in which the Docker images can be used. Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. +3. If you run into any issues or want to customise your build with advanced options, the [Building Images](../building-images) section provides all of the relevant details. +4. For more information, check out the [FAQ](./frequently-asked-questions) and the [Command Reference](../commands) section. + + +## Links + +- [ue4-docker GitHub repository]({{ site.data.common.projects.ue4-docker.repo }}) +- [ue4-docker package on PyPI](https://pypi.org/project/ue4-docker/) +- [Related articles on adamrehn.com](https://adamrehn.com/articles/tag/Unreal%20Engine/) +- [Unreal Containers community hub](https://unrealcontainers.com/) diff --git a/docs/read-these-first/large-container-images-primer.md b/docs/read-these-first/large-container-images-primer.md new file mode 100644 index 00000000..e6c49211 --- /dev/null +++ b/docs/read-these-first/large-container-images-primer.md @@ -0,0 +1,20 @@ +--- +title: Large container images primer +pagenum: 2 +--- + +Although large container images are in no way different to smaller container images at a technical level, there are several aspects of the Docker build process that impact large images to a far greater extent than smaller images. This page provides an overview for users who have never built large (multi-gigabyte) container images before and may therefore be unfamiliar with these impacts. This information applies equally to both Linux containers and Windows containers. + + +## Filesystem layer commit performance + +The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. For some of the larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. (The Installed Build layer in the the multi-stage build of the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) + +Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. There is no need for concern, as none of the steps in the ue4-docker Dockerfiles can run indefinitely without failing and emitting an error. When a build step ceases to produce output, it is merely a matter of waiting for the filesystem layer commit to complete. + + +## Disk space consumption during the build process + +Due to overheads associated with temporary layers in multi-stage builds and layer difference computation, the Docker build process for an image will consume more disk space than is required to hold the final built image. These overheads are relatively modest when building smaller container images. However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. + +Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least {{ site.data.ue4-docker.common.diskspace_linux | escape }} of available disk space under Linux and at least {{ site.data.ue4-docker.common.diskspace_windows | escape }} of available disk space under Windows. Once a build is complete, the [ue4-docker clean](../commands/clean) command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all of the disk space not occupied by the final built images. The [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/) command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/read-these-first/nvidia-docker-primer.md b/docs/read-these-first/nvidia-docker-primer.md new file mode 100644 index 00000000..80f00a08 --- /dev/null +++ b/docs/read-these-first/nvidia-docker-primer.md @@ -0,0 +1,6 @@ +--- +title: NVIDIA Container Toolkit primer +pagenum: 4 +--- + +{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Key Concepts: NVIDIA Container Toolkit](https://unrealcontainers.com/docs/concepts/nvidia-docker)." %} diff --git a/docs/read-these-first/windows-container-primer.md b/docs/read-these-first/windows-container-primer.md new file mode 100644 index 00000000..58a0977e --- /dev/null +++ b/docs/read-these-first/windows-container-primer.md @@ -0,0 +1,14 @@ +--- +title: Windows containers primer +pagenum: 3 +--- + +{% include alerts/info.html content="The implementation-agnostic information from this page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Key Concepts: Windows Containers](https://unrealcontainers.com/docs/concepts/windows-containers)." %} + +**Details specific to ue4-docker:** + +Due to the performance and stability issues currently associated with containers running in Hyper-V isolation mode, it is strongly recommended that process isolation mode be used for building and running Windows containers. This necessitates the use of Windows Server as the host system ([or Windows 10 version 1809 or newer for development and testing purposes](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional)) and requires that all container images use the same Windows version as the host system. A number of ue4-docker commands provide specific functionality to facilitate this: + +- The [ue4-docker build](../commands/build) command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. Hyper-V isolation mode will still be used if the user explicitly [specifies a different kernel version](../building-images/advanced-build-options#specifying-the-windows-server-core-base-image-tag) than that of the host system or [explicitly requests Hyper-V isolation mode](../building-images/advanced-build-options#specifying-the-isolation-mode-under-windows). + +- The [ue4-docker setup](../commands/setup) command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. diff --git a/docs/use-cases/continuous-integration.md b/docs/use-cases/continuous-integration.md new file mode 100644 index 00000000..c1fbe767 --- /dev/null +++ b/docs/use-cases/continuous-integration.md @@ -0,0 +1,9 @@ +--- +title: Continuous Integration (CI) +blurb: > + Use a controlled and reproducible environment to build, + test, and package Unreal projects. +pagenum: 2 +--- + +{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Continuous Integration and Deployment (CI/CD)](https://unrealcontainers.com/docs/use-cases/continuous-integration)." %} diff --git a/docs/use-cases/linux-installed-builds.md b/docs/use-cases/linux-installed-builds.md new file mode 100644 index 00000000..7ace6a59 --- /dev/null +++ b/docs/use-cases/linux-installed-builds.md @@ -0,0 +1,11 @@ +--- +title: Linux Installed Builds +blurb: > + Under Windows and macOS, Engine licensees can easily download and manage + Installed Builds of UE4 through the Epic Games Launcher. Since version 4.21.0 + of the Engine, ue4-docker provides an alternative source of Installed Builds + under Linux in lieu of a native version of the launcher. +pagenum: 4 +--- + +{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Linux Installed Builds](https://unrealcontainers.com/docs/use-cases/linux-installed-builds)." %} diff --git a/docs/use-cases/microservices.md b/docs/use-cases/microservices.md new file mode 100644 index 00000000..98ab8eec --- /dev/null +++ b/docs/use-cases/microservices.md @@ -0,0 +1,10 @@ +--- +title: Microservices +blurb: > + Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full + image makes it easy to build UE4-powered microservices with Google's + popular gRPC framework. +pagenum: 3 +--- + +{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Microservices](https://unrealcontainers.com/docs/use-cases/microservices)." %} diff --git a/docs/use-cases/use-cases-overview.md b/docs/use-cases/use-cases-overview.md new file mode 100644 index 00000000..d0a950a3 --- /dev/null +++ b/docs/use-cases/use-cases-overview.md @@ -0,0 +1,15 @@ +--- +title: Use cases overview +pagenum: 1 +--- + +The container images produced by ue4-docker incorporate infrastructure to facilitate a wide variety of use cases. A number of key use cases are listed below. Select a use case to see detailed instructions on how to build and run the appropriate container images. + +{::nomarkdown} +{% assign cases = site.documents | where: "subsite", page.subsite | where: "chapter", page.chapter | where_exp: "page", "page.pagenum != null" | where_exp: "page", "page.pagenum > 1" | sort: "pagenum" %} + +{:/} From 55191a78aa966010604da5ed1dd5b9fb2492f016 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 23:15:11 +0300 Subject: [PATCH 297/430] Convert docs to Asciidoc * Remove all info about ue4-engine * Remove docs about prebuilt ue4-build-prerequisites * Format command docs as mapages --- CONTRIBUTING.adoc | 27 ++ CONTRIBUTING.md | 29 -- README.adoc | 39 +++ README.md | 39 --- .../advanced-build-options.adoc | 266 ++++++++++++++++++ .../building-images/advanced-build-options.md | 231 --------------- .../available-container-images.adoc | 95 +++++++ .../available-container-images.md | 41 --- .../troubleshooting-build-issues.adoc | 144 ++++++++++ .../troubleshooting-build-issues.md | 118 -------- docs/commands/build.md | 45 --- docs/commands/clean.md | 22 -- docs/commands/diagnostics.md | 49 ---- docs/commands/export.md | 54 ---- docs/commands/info.md | 23 -- docs/commands/list-of-commands.md | 15 - docs/commands/test.md | 15 - docs/commands/ue4-docker-build.adoc | 122 ++++++++ docs/commands/ue4-docker-clean.adoc | 32 +++ docs/commands/ue4-docker-diagnostics.adoc | 52 ++++ docs/commands/ue4-docker-export.adoc | 62 ++++ docs/commands/ue4-docker-info.adoc | 26 ++ .../{setup.md => ue4-docker-setup.adoc} | 30 +- docs/commands/ue4-docker-test.adoc | 20 ++ docs/commands/ue4-docker-version.adoc | 18 ++ docs/commands/version.md | 13 - docs/configuration/configuring-linux.adoc | 72 +++++ docs/configuration/configuring-linux.md | 68 ----- docs/configuration/configuring-macos.adoc | 51 ++++ docs/configuration/configuring-macos.md | 51 ---- .../configuration/configuring-windows-10.adoc | 75 +++++ docs/configuration/configuring-windows-10.md | 71 ----- .../configuring-windows-server.adoc | 73 +++++ .../configuring-windows-server.md | 114 -------- .../supported-host-configurations.adoc | 43 +++ .../supported-host-configurations.md | 33 --- .../frequently-asked-questions.adoc | 30 ++ .../frequently-asked-questions.md | 22 -- .../introduction-to-ue4-docker.adoc | 51 ++++ .../introduction-to-ue4-docker.md | 47 ---- .../large-container-images-primer.adoc | 32 +++ .../large-container-images-primer.md | 20 -- .../nvidia-docker-primer.adoc | 9 + docs/read-these-first/nvidia-docker-primer.md | 6 - .../windows-container-primer.adoc | 20 ++ .../windows-container-primer.md | 14 - docs/use-cases/continuous-integration.adoc | 11 + docs/use-cases/continuous-integration.md | 9 - docs/use-cases/linux-installed-builds.adoc | 12 + docs/use-cases/linux-installed-builds.md | 11 - docs/use-cases/microservices.adoc | 11 + docs/use-cases/microservices.md | 10 - docs/use-cases/use-cases-overview.adoc | 14 + docs/use-cases/use-cases-overview.md | 15 - .../infrastructure/BuildConfiguration.py | 2 +- 55 files changed, 1425 insertions(+), 1199 deletions(-) create mode 100644 CONTRIBUTING.adoc delete mode 100644 CONTRIBUTING.md create mode 100644 README.adoc delete mode 100644 README.md create mode 100644 docs/building-images/advanced-build-options.adoc delete mode 100644 docs/building-images/advanced-build-options.md create mode 100644 docs/building-images/available-container-images.adoc delete mode 100644 docs/building-images/available-container-images.md create mode 100644 docs/building-images/troubleshooting-build-issues.adoc delete mode 100644 docs/building-images/troubleshooting-build-issues.md delete mode 100644 docs/commands/build.md delete mode 100644 docs/commands/clean.md delete mode 100644 docs/commands/diagnostics.md delete mode 100644 docs/commands/export.md delete mode 100644 docs/commands/info.md delete mode 100644 docs/commands/list-of-commands.md delete mode 100644 docs/commands/test.md create mode 100644 docs/commands/ue4-docker-build.adoc create mode 100644 docs/commands/ue4-docker-clean.adoc create mode 100644 docs/commands/ue4-docker-diagnostics.adoc create mode 100644 docs/commands/ue4-docker-export.adoc create mode 100644 docs/commands/ue4-docker-info.adoc rename docs/commands/{setup.md => ue4-docker-setup.adoc} (52%) create mode 100644 docs/commands/ue4-docker-test.adoc create mode 100644 docs/commands/ue4-docker-version.adoc delete mode 100644 docs/commands/version.md create mode 100644 docs/configuration/configuring-linux.adoc delete mode 100644 docs/configuration/configuring-linux.md create mode 100644 docs/configuration/configuring-macos.adoc delete mode 100644 docs/configuration/configuring-macos.md create mode 100644 docs/configuration/configuring-windows-10.adoc delete mode 100644 docs/configuration/configuring-windows-10.md create mode 100644 docs/configuration/configuring-windows-server.adoc delete mode 100644 docs/configuration/configuring-windows-server.md create mode 100644 docs/configuration/supported-host-configurations.adoc delete mode 100644 docs/configuration/supported-host-configurations.md create mode 100644 docs/read-these-first/frequently-asked-questions.adoc delete mode 100644 docs/read-these-first/frequently-asked-questions.md create mode 100644 docs/read-these-first/introduction-to-ue4-docker.adoc delete mode 100644 docs/read-these-first/introduction-to-ue4-docker.md create mode 100644 docs/read-these-first/large-container-images-primer.adoc delete mode 100644 docs/read-these-first/large-container-images-primer.md create mode 100644 docs/read-these-first/nvidia-docker-primer.adoc delete mode 100644 docs/read-these-first/nvidia-docker-primer.md create mode 100644 docs/read-these-first/windows-container-primer.adoc delete mode 100644 docs/read-these-first/windows-container-primer.md create mode 100644 docs/use-cases/continuous-integration.adoc delete mode 100644 docs/use-cases/continuous-integration.md create mode 100644 docs/use-cases/linux-installed-builds.adoc delete mode 100644 docs/use-cases/linux-installed-builds.md create mode 100644 docs/use-cases/microservices.adoc delete mode 100644 docs/use-cases/microservices.md create mode 100644 docs/use-cases/use-cases-overview.adoc delete mode 100644 docs/use-cases/use-cases-overview.md diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc new file mode 100644 index 00000000..5667ba89 --- /dev/null +++ b/CONTRIBUTING.adoc @@ -0,0 +1,27 @@ += Guidelines for contributing to ue4-docker +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Creating issues + +When creating an issue to report a bug, please follow the provided issue template. +Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. + +If you are creating an issue for a feature request, you can disregard the default issue template. +However, please make sure you have thoroughly read https://docs.adamrehn.com/ue4-docker/[the documentation] to ensure the requested feature does not already exist before creating an issue. + +== Creating pull requests + +Before creating a pull request, please ensure sure you have tested your changes on all platforms to which the change applies (Linux / Windows Server / Windows 10 / macOS). + +== Autoformatting your changes + +ue4-docker uses https://github.com/psf/black[Black] for its code formatting. +If you are submitting changes, make sure to install and run Black: + +* `pip install --user black` +* `python -m black .` (invoke in repository root) +* Now your code is properly formatted diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 88752841..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,29 +0,0 @@ -Guidelines for contributing to ue4-docker -========================================= - - -## Contents - -- [Creating issues](#creating-issues) -- [Creating pull requests](#creating-pull-requests) - - -## Creating issues - -When creating an issue to report a bug, please follow the provided issue template. Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. - -If you are creating an issue for a feature request, you can disregard the default issue template. However, please make sure you have thoroughly read [the documentation](https://docs.adamrehn.com/ue4-docker/) to ensure the requested feature does not already exist before creating an issue. - - -## Creating pull requests - -Before creating a pull request, please ensure sure you have tested your changes on all platforms to which the change applies (Linux / Windows Server / Windows 10 / macOS). - -## Autoformatting your changes - -ue4-docker uses [Black](https://github.com/psf/black) for its code formatting. -If you are submitting changes, make sure to install and run Black: - -- `pip install --user black` -- `python -m black .` (invoke in repository root) -- Now your code is properly formatted diff --git a/README.adoc b/README.adoc new file mode 100644 index 00000000..55d51231 --- /dev/null +++ b/README.adoc @@ -0,0 +1,39 @@ += Unreal Engine 4 Docker Containers + +image:https://raw.githubusercontent.com/adamrehn/ue4-docker/master/resources/images/banner.svg?sanitize=true[Unreal Engine and Docker Logos,align="center"] + +*Looking for a place to start? +Check out the https://unrealcontainers.com/[Unreal Containers community hub] for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the https://docs.adamrehn.com/ue4-docker/[comprehensive ue4-docker documentation] to view details specific to using the ue4-docker project.* + +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' https://www.unrealengine.com/[Unreal Engine 4]. +The images also incorporate the infrastructure from https://github.com/adamrehn/ue4cli[ue4cli], https://github.com/adamrehn/conan-ue4cli[conan-ue4cli], and https://github.com/adamrehn/ue4-ci-helpers[ue4-ci-helpers] to facilitate a wide variety of use cases. + +Key features include: + +* Unreal Engine 4.20.0 and newer is supported. +* Both Windows containers and Linux containers are supported. +* Building and packaging UE4 projects is supported. +* Running automation tests is supported. +* Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. + +Resources: + +* *Documentation:* https://docs.adamrehn.com/ue4-docker/ +* *GitHub repository:* https://github.com/adamrehn/ue4-docker +* *Package on PyPI:* https://pypi.org/project/ue4-docker/ +* *Related articles:* https://adamrehn.com/articles/tag/Unreal%20Engine/ +* *Unreal Containers community hub:* https://unrealcontainers.com/ +* *Development channel on the Unreal Containers Community Discord server*: https://discord.gg/46CJg9fyJ9 + +== Contributing + +See the file https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md[CONTRIBUTING.md] for guidelines on how to contribute to the development of ue4-docker. + +== Legal + +Copyright © 2018 - 2022, Adam Rehn. +Licensed under the MIT License, see the file https://github.com/adamrehn/ue4-docker/blob/master/LICENSE[LICENSE] for details. + +Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. + +Docker and the Docker logo are trademarks or registered trademarks of Docker in the United States and other countries. diff --git a/README.md b/README.md deleted file mode 100644 index d40ec9c5..00000000 --- a/README.md +++ /dev/null @@ -1,39 +0,0 @@ -

Unreal Engine and Docker Logos

-

Unreal Engine 4 Docker Containers

-

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

-

 

- -**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://docs.adamrehn.com/ue4-docker/) to view details specific to using the ue4-docker project.** - -The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. - -Key features include: - -- Unreal Engine 4.20.0 and newer is supported. -- Both Windows containers and Linux containers are supported. -- Building and packaging UE4 projects is supported. -- Running automation tests is supported. -- Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. - -Resources: - -- **Documentation:** -- **GitHub repository:** -- **Package on PyPI:** -- **Related articles:** -- **Unreal Containers community hub:** -- **Development channel on the Unreal Containers Community Discord server**: - - -## Contributing - -See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. - - -## Legal - -Copyright © 2018 - 2021, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. - -Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. - -Docker and the Docker logo are trademarks or registered trademarks of Docker in the United States and other countries. diff --git a/docs/building-images/advanced-build-options.adoc b/docs/building-images/advanced-build-options.adoc new file mode 100644 index 00000000..7096cf09 --- /dev/null +++ b/docs/building-images/advanced-build-options.adoc @@ -0,0 +1,266 @@ += Advanced build options +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== General options + +=== Performing a dry run + +If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the link:../commands/ue4-docker-build.adoc[ue4-docker build] command. +Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. + +=== Specifying a custom namespace for the image tags + +If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. + +=== Specifying Git credentials + +The link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: + +- **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. + +- **Environment variables**: the `UE4DOCKER_USERNAME` and `UE4DOCKER_PASSWORD` environment variables can be used to specify the username and password, respectively. +Note that credentials specified via command-line arguments will take precedence over values defined in environment variables. + +- **Standard input**: if either the username or password has not been specified via a command-line argument or environment variable then the build command will prompt the user to enter the credential(s) for which values have not already been specified. + +Note that the username and password are handled independently, which means you can use different methods to specify the two credentials (e.g. username specified via command-line argument and password supplied via standard input.) + +Users who have enabled https://help.github.com/en/articles/about-two-factor-authentication[Two-Factor Authentication (2FA)] for their GitHub account will need to generate a https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line[personal access token] and use that in place of their password. + +=== Building a custom version of the Unreal Engine + +If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +When building a custom Engine version, **both the repository URL and branch/tag must be specified**: + +If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +When building a custom Engine version, **both the repository URL and branch/tag must be specified**: + +[source,shell] +---- +ue4-docker build custom -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +---- + +This will produce images tagged `adamrehn/ue4-source:custom`, `adamrehn/ue4-minimal:custom`, etc. + +If you are performing multiple custom builds and wish to differentiate between them, it is recommended to also specify a name for the custom build: + +[source,shell] +---- +ue4-docker build custom:my-custom-build -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +---- + +This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn/ue4-minimal:my-custom-build`, etc. + +=== Excluding Engine components to reduce the final image size + +Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the link:../commands/ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] and link:../building-images/available-container-images.adoc#ue4-full[ue4-full] images. +The following components can be excluded: + +- `ddc`: disables building the DDC for the Engine. +This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. + +- `debug`: removes all debug symbols from the built images. +(When building Windows containers the files are actually truncated instead of removed, so they still exist but have a size of zero bytes. +This is done for compatibility reasons.) + +- `templates`: removes the template projects and samples that ship with the Engine. + +You can specify the `--exclude` flag multiple times to exclude as many components as you like. +For example: + +[source,shell] +---- +# Excludes both debug symbols and template projects +ue4-docker build 4.21.2 --exclude debug --exclude templates +---- + +=== Enabling system resource monitoring during builds + +Starting in ue4-docker version 0.0.46, you can use the `--monitor` flag to enable a background thread that will log information about system resource usage (available disk space and memory, CPU usage, etc.) at intervals during the build. +You can also use the `-interval` flag to override the default interval of 20 seconds: + +[source,shell] +---- +# Logs system resource levels every 20 seconds +ue4-docker build 4.24.2 --monitor +---- + +[source,shell] +---- +# Logs system resource levels every 20 seconds +ue4-docker build 4.24.2 --monitor +---- + +[source,shell] +---- +# Logs system resource levels every 5 seconds +ue4-docker build 4.24.2 --monitor -interval=5 +---- + +=== Exporting generated Dockerfiles + +Since ue4-docker version 0.0.78, the link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. +In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a https://docs.docker.com/develop/develop-images/multistage-build/[multi-stage build]. +You can use these flags like so: + +[source,shell] +---- +# Exports Dockerfiles for all images to the specified filesystem directory +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" +---- + +[source,shell] +---- +# Exports Dockerfiles for all images +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" +---- + +[source,shell] +---- +# Exports Dockerfiles for all images and combines them into a single Dockerfile +ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --combine +---- + +Exporting Dockerfiles is useful for debugging or contributing to the development of ue4-docker itself. +You can also use the generated Dockerfiles to build container images independently of ue4-docker, but only under the following circumstances: + +- When building Windows container images, you must specify the <> `source_mode` and set it to `copy`. +This generates Dockerfiles that copy the Unreal Engine source code from the host filesystem rather than cloning it from a git repository, thus eliminating the dependency on ue4-docker's credential endpoint to securely provide git credentials and allowing container images to be built without the need for ue4-docker itself. + +- When building Linux container images, you must either set the <> `source_mode` to `copy` as detailed above, or else specify the `credential_mode` option and set it to `secrets`. +This generates Dockerfiles that use the Linux-only https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] functionality to securely provide git credentials, eliminating the dependency on ue4-docker's credential endpoint whilst still facilitating the use of a git repository to provide the Unreal Engine source code. + +=== Advanced options for Dockerfile generation + +NOTE: Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers[Jinja identifiers]. + +Since ue4-docker version 0.0.78, the link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. +Some of these options (such as `source_mode`) can only be used when <>, whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: + +- **`source_mode`**: *(string)* controls how the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. +Valid options are: + +- `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. +This is the only mode that can be used when not <>. + +- `copy`: copies the Unreal Engine source code from the host filesystem. +The filesystem path can be specified using the `SOURCE_LOCATION` Docker build argument, and of course must be a child path of the build context. + +- **`credential_mode`**: *(string)* controls how the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] Dockerfile securely obtains credentials for authenticating with remote git repositories when `source_mode` is set to `git`. +Valid options are: + +- `endpoint`: the default mode, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] container during the build process by way of a Docker build argument. +This mode will not work when <>, since the credential endpoint will not be available during the build process. + +- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] container during the build process. + +- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image. + +- **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. +This includes the labels which specify the <> as well as the sentinel labels that the link:../commands/ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. + +- **`disable_all_patches`**: *(boolean)* disables all the patches that ue4-docker ordinarily applies to the Unreal Engine source code. +This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. +It is typically safer to disable individual patches using the specific flag for each patch instead of simply disabling everything: + +- **`disable_release_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to versions of the Unreal Engine which are known to contain bugs, such as Unreal Engine 4.25.4. This will obviously break the build process when building these known broken releases, but will have no effect when building other versions of the Unreal Engine. + +- **`disable_windows_setup_patch`**: *(boolean)* prevents ue4-docker from patching `Setup.bat` under Windows to comment out the calls to the Unreal Engine prerequisites installer and UnrealVersionSelector, both of which are known to cause issues during the build process for Windows containers. + +- **`disable_linker_fixup`**: *(boolean)* prevents ue4-docker from replacing the linker in the Unreal Engine's bundled toolchain with a symbolic link to the system linker under Linux. + +- **`disable_example_platform_cleanup`**: *(boolean)* prevents ue4-docker from removing the `Engine/Platforms/XXX` directory that was introduced in Unreal Engine 4.24.0 and subsequently removed in Unreal Engine 4.26.0. This directory represents a "dummy" target platform for demonstration purposes, and the presence of this directory will typically break the build process. + +- **`disable_ubt_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix bugs in UnrealBuildTool (UBT) under various versions of the Unreal Engine. + +- **`disable_opengl_patch`**: *(boolean)* prevents ue4-docker from attempting to re-enable the OpenGL RHI under Linux for versions of the Unreal Engine in which it is present but deprecated. + +- **`disable_buildgraph_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to the BuildGraph XML files used to create an Installed Build of the Unreal Engine. +These patches fix various bugs under both Windows and Linux across multiple versions of the Unreal Engine. + +- **`disable_target_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix broken `PlatformType` fields for client and server targets in `BaseEngine.ini` under Unreal Engine versions where these values are set incorrectly. + +- **`disable_unrealpak_copy`**: *(boolean)* prevents ue4-docker from ensuring the UnrealPak tool is correctly copied into Installed Builds of the Unreal Engine under Linux. +Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. + +- **`disable_toolchain_copy`**: *(boolean)* prevents ue4-docker from ensuring the bundled clang toolchain is correctly copied into Installed Builds of the Unreal Engine under Linux. +Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. + +== Windows-specific options + +=== Specifying the Windows Server Core base image tag + +NOTE: The `-basetag` flag controls how the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. + +By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. +However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. +(See the https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility[Windows Container Version Compatibility] page for a table detailing which configurations are supported.) + +If you are building images with the intention of subsequently running them under an older version of Windows 10 or Windows Server, you will need to build images based on the same kernel version as the target system (or older.) The kernel version can be specified by providing the appropriate base OS image tag via the `-basetag=TAG` flag when invoking the build command: + +[source,shell] +---- +ue4-docker build 4.20.3 -basetag=ltsc2016 # Uses Windows Server 2016 (Long Term Support Channel) +---- + +For a list of supported base image tags, see the https://hub.docker.com/r/microsoft/windowsservercore/[Windows Server Core base image on Docker Hub]. + +=== Specifying the isolation mode under Windows + +The isolation mode can be explicitly specified via the `-isolation=MODE` flag when invoking the build command. +Valid values are `process` (supported under Windows Server and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[Windows 10 version 1809 or newer]) or `hyperv` (supported under both Windows 10 and Windows Server.) If you do not explicitly specify an isolation mode then the appropriate default for the host system will be used. + +=== Specifying Visual Studio Build Tools version under Windows + +By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. +Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. +To do so, pass `--visual-studio=2019` flag when invoking the build command. + +=== Keeping or excluding Installed Build debug symbols under Windows + +WARNING: Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. +See link:troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy--frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[this section of the Troubleshooting Build Issues page] for further details on this issue. + +Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. +This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. +However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. +(This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) + +Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section <>. + +=== Building Linux container images under Windows + +By default, Windows container images are built when running the build command under Windows. +To build Linux container images instead, simply specify the `--linux` flag when invoking the build command. + +== Linux-specific options + +=== Enabling CUDA support for GPU-enabled Linux images + +IMPORTANT: The `--cuda` flag controls how the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. + +By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. +If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. + +You can also control the version of the CUDA base image that is used by appending a version number when specifying the `--cuda` flag, as demonstrated below: + +[source,shell] +---- +# Uses the default CUDA base image (currently CUDA 9.2) +ue4-docker build RELEASE --cuda +---- + +[source,shell] +---- +# Uses the CUDA 10.0 base image +ue4-docker build RELEASE --cuda=10.0 +{% endhighlight %} +---- + +For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the https://hub.docker.com/r/nvidia/cudagl/[nvidia/cudagl] base image. diff --git a/docs/building-images/advanced-build-options.md b/docs/building-images/advanced-build-options.md deleted file mode 100644 index 3133d0b0..00000000 --- a/docs/building-images/advanced-build-options.md +++ /dev/null @@ -1,231 +0,0 @@ ---- -title: Advanced build options -pagenum: 2 ---- - -## General options - - -### Performing a dry run - -If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the [ue4-docker build](../commands/build) command. Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. - - -### Specifying a custom namespace for the image tags - -If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. **Note that this setting does not effect the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image**, which always retains its default name in order to maintain compatibility with the [prebuilt version from Docker Hub](#pulling-a-prebuilt-version-of-the-ue4-build-prerequisites-image). - - -### Specifying Git credentials - -The [ue4-docker build](../commands/build) command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: - -- **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. - -- **Environment variables**: the `UE4DOCKER_USERNAME` and `UE4DOCKER_PASSWORD` environment variables can be used to specify the username and password, respectively. Note that credentials specified via command-line arguments will take precedence over values defined in environment variables. - -- **Standard input**: if either the username or password has not been specified via a command-line argument or environment variable then the build command will prompt the user to enter the credential(s) for which values have not already been specified. - -Note that the username and password are handled independently, which means you can use different methods to specify the two credentials (e.g. username specified via command-line argument and password supplied via standard input.) - -Users who have enabled [Two-Factor Authentication (2FA)](https://help.github.com/en/articles/about-two-factor-authentication) for their GitHub account will need to generate a [personal access token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) and use that in place of their password. - - -### Building a custom version of the Unreal Engine - -If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. When building a custom Engine version, **both the repository URL and branch/tag must be specified**: - -{% highlight bash %} -ue4-docker build custom -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch -{% endhighlight %} - -This will produce images tagged `adamrehn/ue4-source:custom`, `adamrehn/ue4-engine:custom`, etc. - -If you are performing multiple custom builds and wish to differentiate between them, it is recommended to also specify a name for the custom build: - -{% highlight bash %} -ue4-docker build custom:my-custom-build -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch -{% endhighlight %} - -This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn/ue4-engine:my-custom-build`, etc. - - -### Pulling a prebuilt version of the `ue4-build-prerequisites` image - -If you would like to pull the [prebuilt version](https://hub.docker.com/r/adamrehn/ue4-build-prerequisites) of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image from Docker Hub instead of building it locally, simply specify the `--pull-prerequisites` flag when invoking the [ue4-docker build](../commands/build) command. This is primarily useful when building images under versions of Windows Server Core prior to Windows Server 2019, since using the prebuilt image allows you to avoid [copying the required DLL files from Windows 10](../configuration/configuring-windows-server#step-5-copy-required-dll-files-windows-server-core-version-1803-and-older-only). Note however that prebuilt versions of the image are available only for [LTSC versions of Windows](https://docs.microsoft.com/en-us/windows-server/get-started-19/servicing-channels-19), not for SAC versions. - - -### Excluding Engine components to reduce the final image size - -Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the [ue4-docker build](../commands/build) command to specify that certain Engine components should be excluded from the [ue4-minimal](../building-images/available-container-images#ue4-minimal) and [ue4-full](../building-images/available-container-images#ue4-full) images. The following components can be excluded: - -- `ddc`: disables building the DDC for the Engine. This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. - -- `debug`: removes all debug symbols from the built images. (When building Windows containers the files are actually truncated instead of removed, so they still exist but have a size of zero bytes. This is done for compatibility reasons.) - -- `templates`: removes the template projects and samples that ship with the Engine. - -You can specify the `--exclude` flag multiple times to exclude as many components as you like. For example: - -{% highlight bash %} -# Excludes both debug symbols and template projects -ue4-docker build 4.21.2 --exclude debug --exclude templates -{% endhighlight %} - - -### Enabling system resource monitoring during builds - -Starting in ue4-docker version 0.0.46, you can use the `--monitor` flag to enable a background thread that will log information about system resource usage (available disk space and memory, CPU usage, etc.) at intervals during the build. You can also use the `-interval` flag to override the default interval of 20 seconds: - -{% highlight bash %} -# Logs system resource levels every 20 seconds -ue4-docker build 4.24.2 --monitor - -# Logs system resource levels every 5 seconds -ue4-docker build 4.24.2 --monitor -interval=5 -{% endhighlight %} - - -### Exporting generated Dockerfiles - -Since ue4-docker version 0.0.78, the [ue4-docker build](../commands/build) command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a [multi-stage build](https://docs.docker.com/develop/develop-images/multistage-build/). You can use these flags like so: - -{% highlight bash %} -# Exports Dockerfiles for all images to the specified filesystem directory -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" - -# Exports Dockerfiles for all images except the ue4-engine image -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --no-engine - -# Exports Dockerfiles for all images except the ue4-engine image and combines them into a single Dockerfile -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --no-engine --combine -{% endhighlight %} - -Exporting Dockerfiles is useful for debugging or contributing to the development of ue4-docker itself. You can also use the generated Dockerfiles to build container images independently of ue4-docker, but only under the following circumstances: - -- When building Windows container images, you must specify the [advanced option](#advanced-options-for-dockerfile-generation) `source_mode` and set it to `copy`. This generates Dockerfiles that copy the Unreal Engine source code from the host filesystem rather than cloning it from a git repository, thus eliminating the dependency on ue4-docker's credential endpoint to securely provide git credentials and allowing container images to be built without the need for ue4-docker itself. - -- When building Linux container images, you must either set the [advanced option](#advanced-options-for-dockerfile-generation) `source_mode` to `copy` as detailed above, or else specify the `credential_mode` option and set it to `secrets`. This generates Dockerfiles that use the Linux-only [BuildKit build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) functionality to securely provide git credentials, eliminating the dependency on ue4-docker's credential endpoint whilst still facilitating the use of a git repository to provide the Unreal Engine source code. - - -### Advanced options for Dockerfile generation - -{% include alerts/info.html content="Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in [Jinja identifiers](https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers)." title="Title" %} - -Since ue4-docker version 0.0.78, the [ue4-docker build](../commands/build) command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying [Jinja templating engine](https://jinja.palletsprojects.com/) used to generate Dockerfiles. Some of these options (such as `source_mode`) can only be used when [exporting generated Dockerfiles](#exporting-generated-dockerfiles), whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: - -- **`source_mode`**: *(string)* controls how the [ue4-source](../building-images/available-container-images#ue4-source) Dockerfile obtains the source code for the Unreal Engine. Valid options are: - - - `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. This is the only mode that can be used when not [exporting generated Dockerfiles](#exporting-generated-dockerfiles). - - - `copy`: copies the Unreal Engine source code from the host filesystem. The filesystem path can be specified using the `SOURCE_LOCATION` Docker build argument, and of course must be a child path of the build context. - -- **`credential_mode`**: *(string)* controls how the [ue4-source](../building-images/available-container-images#ue4-source) Dockerfile securely obtains credentials for authenticating with remote git repositories when `source_mode` is set to `git`. Valid options are: - - - `endpoint`: the default mode, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the [ue4-source](../building-images/available-container-images#ue4-source) container during the build process by way of a Docker build argument. This mode will not work when [exporting generated Dockerfiles](#exporting-generated-dockerfiles), since the credential endpoint will not be available during the build process. - - - `secrets`: **(Linux containers only)** uses [BuildKit build secrets](https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information) to securely inject the git credentials into the [ue4-source](../building-images/available-container-images#ue4-source) container during the build process. - -- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the [BuildGraph system](https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html) when creating an Installed Build of the Unreal Engine in the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image. - -- **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. This includes the labels which specify the [components excluded from the ue4-minimal image](#excluding-engine-components-to-reduce-the-final-image-size) as well as the sentinel labels that the [ue4-docker clean](../commands/clean) command uses to identify container images, and will therefore break the functionality of that command. - -- **`disable_all_patches`**: *(boolean)* disables all of the patches that ue4-docker ordinarily applies to the Unreal Engine source code. This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. It is typically safer to disable individual patches using the specific flag for each patch instead of simply disabling everything: - - - **`disable_release_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to versions of the Unreal Engine which are known to contain bugs, such as Unreal Engine 4.25.4. This will obviously break the build process when building these known broken releases, but will have no effect when building other versions of the Unreal Engine. - - - **`disable_windows_setup_patch`**: *(boolean)* prevents ue4-docker from patching `Setup.bat` under Windows to comment out the calls to the Unreal Engine prerequisites installer and UnrealVersionSelector, both of which are known to cause issues during the build process for Windows containers. - - - **`disable_linker_fixup`**: *(boolean)* prevents ue4-docker from replacing the linker in the Unreal Engine's bundled toolchain with a symbolic link to the system linker under Linux. - - - **`disable_example_platform_cleanup`**: *(boolean)* prevents ue4-docker from removing the `Engine/Platforms/XXX` directory that was introduced in Unreal Engine 4.24.0 and subsequently removed in Unreal Engine 4.26.0. This directory represents a "dummy" target platform for demonstration purposes, and the presence of this directory will typically break the build process. - - - **`disable_ubt_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix bugs in UnrealBuildTool (UBT) under various versions of the Unreal Engine. - - - **`disable_opengl_patch`**: *(boolean)* prevents ue4-docker from attempting to re-enable the OpenGL RHI under Linux for versions of the Unreal Engine in which it is present but deprecated. - - - **`disable_buildgraph_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to the BuildGraph XML files used to create an Installed Build of the Unreal Engine. These patches fix various bugs under both Windows and Linux across multiple versions of the Unreal Engine. - - - **`disable_target_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to fix broken `PlatformType` fields for client and server targets in `BaseEngine.ini` under Unreal Engine versions where these values are set incorrectly. - - - **`disable_unrealpak_copy`**: *(boolean)* prevents ue4-docker from ensuring the UnrealPak tool is correctly copied into Installed Builds of the Unreal Engine under Linux. Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. - - - **`disable_toolchain_copy`**: *(boolean)* prevents ue4-docker from ensuring the bundled clang toolchain is correctly copied into Installed Builds of the Unreal Engine under Linux. Some older versions of the Unreal Engine did not copy this correctly, breaking the functionality of created Installed Builds. - - -## Windows-specific options - - -### Specifying the Windows Server Core base image tag - -{% capture _alert_content %} -The `-basetag` flag controls how the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image is built and tagged, which has a flow-on effect to all of the other images. If you are building multiple related images over separate invocations of the build command (e.g. building the [ue4-source](../building-images/available-container-images#ue4-source) image in one command and then subsequently building the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. -{% endcapture %} -{% include alerts/info.html content=_alert_content %} - -By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. (See the [Windows Container Version Compatibility](https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility) page for a table detailing which configurations are supported.) - -If you are building images with the intention of subsequently running them under an older version of Windows 10 or Windows Server, you will need to build images based on the same kernel version as the target system (or older.) The kernel version can be specified by providing the appropriate base OS image tag via the `-basetag=TAG` flag when invoking the build command: - -{% highlight bash %} -ue4-docker build 4.20.3 -basetag=ltsc2016 # Uses Windows Server 2016 (Long Term Support Channel) -{% endhighlight %} - -For a list of supported base image tags, see the [Windows Server Core base image on Docker Hub](https://hub.docker.com/r/microsoft/windowsservercore/). - - -### Specifying the isolation mode under Windows - -The isolation mode can be explicitly specified via the `-isolation=MODE` flag when invoking the build command. Valid values are `process` (supported under Windows Server and [Windows 10 version 1809 or newer](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional)) or `hyperv` (supported under both Windows 10 and Windows Server.) If you do not explicitly specify an isolation mode then the appropriate default for the host system will be used. - -### Specifying Visual Studio Build Tools version under Windows - -By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. -Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. -To do so, pass `--visual-studio=2019` flag when invoking the build command. - -### Specifying the directory from which to copy required Windows DLL files - -By default, DLL files are copied from `%SystemRoot%\System32`. However, when building container images with an older kernel version than the host, the copied DLL files will be too new and the container OS will refuse to load them. A custom directory containing the correct DLL files for the container kernel version can be specified via the `-dlldir=DIR` flag when invoking the build command. - - -### Keeping or excluding Installed Build debug symbols under Windows - -{% capture _alert_content %} -Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. See [this section of the Troubleshooting Build Issues page](./troubleshooting-build-issues#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image) for further details on this issue. -{% endcapture %} -{% include alerts/warning.html content=_alert_content %} - -Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the [ue4-minimal](../building-images/available-container-images#ue4-minimal) Windows image. This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. (This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) - -Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section [Excluding Engine components to reduce the final image size](#excluding-engine-components-to-reduce-the-final-image-size). - - -### Building Linux container images under Windows - -By default, Windows container images are built when running the build command under Windows. To build Linux container images instead, simply specify the `--linux` flag when invoking the build command. - - -## Linux-specific options - - -### Enabling CUDA support for GPU-enabled Linux images - -{% capture _alert_content %} -The `--cuda` flag controls how the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image is built and tagged, which has a flow-on effect to all of the other images. If you are building multiple related images over separate invocations of the build command (e.g. building the [ue4-source](../building-images/available-container-images#ue4-source) image in one command and then subsequently building the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. -{% endcapture %} -{% include alerts/info.html content=_alert_content %} - -By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. - -You can also control the version of the CUDA base image that is used by appending a version number when specifying the `--cuda` flag, as demonstrated below: - -{% highlight bash %} -# Uses the default CUDA base image (currently CUDA 9.2) -ue4-docker build RELEASE --cuda - -# Uses the CUDA 10.0 base image -ue4-docker build RELEASE --cuda=10.0 -{% endhighlight %} - -For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the [nvidia/cudagl](https://hub.docker.com/r/nvidia/cudagl/) base image. diff --git a/docs/building-images/available-container-images.adoc b/docs/building-images/available-container-images.adoc new file mode 100644 index 00000000..55fa3ca3 --- /dev/null +++ b/docs/building-images/available-container-images.adoc @@ -0,0 +1,95 @@ += List of available container images +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +You can build the following images using the link:../commands/ue4-docker-build.adoc[ue4-docker build] command: + +- <> +- <> +- <> +- <> + +By default, all available images will be built. +You can prevent unwanted images from being built by appending the relevant image-specific flag to the build command (see the *"Flag to disable"* entry for each image below.) + +== ue4-build-prerequisites + +**Tags:** + +* "**Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Windows Server Core base image tag] + +* **Linux containers**: `adamrehn/ue4-build-prerequisites:CONFIGURATION` where `CONFIGURATION` is as follows: + +** `opengl` if CUDA support is not enabled + +** `cudaglVERSION` where `VERSION` is the CUDA version if link:advanced-build-options.adoc#enabling-cuda-support-for-gpu-enabled-linux-images[CUDA support is enabled] (e.g. `cudagl9.2`, `cudagl10.0`, etc.) + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the build prerequisites common to all Engine versions. + +**Uses:** + +* Keep this image on disk to speed up subsequent container image builds. + +== ue4-source + +**Tags:** + +* `adamrehn/ue4-source:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-source:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the cloned source code for UE4, along with its downloaded dependency data. +The ue4-minimal image uses this source code as the starting point for its build. + +**Uses:** + +* Only needed during the build process. +Afterwards, this image can be removed using `ue4-docker clean --source` to save disk space. + +== ue4-minimal + +**Tags:** + +* `adamrehn/ue4-minimal:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-minimal:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the absolute minimum set of components required for use in a Continuous Integration (CI) pipeline, consisting of only the build prerequisites and an Installed Build of the Engine. + +**Uses:** + +* Use this image for link:../use-cases/continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. + +== ue4-full + +**Tags:** + +* `adamrehn/ue4-full:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-full:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains everything from the `ue4-minimal` image, and adds the following: + +* ue4cli +* conan-ue4cli +* ue4-ci-helpers +* UE4Capture (Linux image only) +* PulseAudio support (Linux image only) +* X11 support (Linux image only) + +**Uses:** + +* link:../use-cases/continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers +* Packaging or running cloud rendering projects that optionally utilise UE4Capture under Linux (**note that the (https://hub.docker.com/r/adamrehn/ue4-runtime[ue4-runtime]) image is the recommended base image for running cloud rendering workloads, since this avoids including the multi-gigabyte UE4 developer tools**) +* Packaging link:../use-cases/microservices.adoc[UE4-powered microservices] diff --git a/docs/building-images/available-container-images.md b/docs/building-images/available-container-images.md deleted file mode 100644 index a66eef46..00000000 --- a/docs/building-images/available-container-images.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: List of available container images -pagenum: 1 ---- - -You can build the following images using the [ue4-docker build](../commands/build) command: - -* TOC -{:toc} - -By default, all available images will be built. You can prevent unwanted images from being built by appending the relevant image-specific flag to the build command (see the *"Flag to disable"* entry for each image below.) - - -{% for image in site.data.ue4-docker.images %} -## {{ image.name | escape }} - -**Tags:** - -{% for tag in image.tags %} -- {{ tag }} -{% endfor %} - -{::nomarkdown} -

- Dockerfiles: - Windows - | - Linux -

-{:/} - -**Flag to disable:** {{ image.disable }} - -**Contents:** {{ image.contents }} - -**Uses:** - -{% for use in image.uses %}- {{ use }} -{% endfor %} - -{% endfor %} diff --git a/docs/building-images/troubleshooting-build-issues.adoc b/docs/building-images/troubleshooting-build-issues.adoc new file mode 100644 index 00000000..caf6c301 --- /dev/null +++ b/docs/building-images/troubleshooting-build-issues.adoc @@ -0,0 +1,144 @@ += Troubleshooting build issues +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== General issues + +=== Building the `ue4-build-prerequisites` image fails with a network-related error + +This indicates an underlying network or proxy server issue outside ue4-docker itself that you will need to troubleshoot. +You can use the link:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. +Here are some steps to try: + +- If your host system accesses the network through a proxy server, make sure that https://docs.docker.com/network/proxy/[Docker is configured to use the correct proxy server settings]. +- If DNS resolution is failing then try adding the following entry to your https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file[Docker daemon configuration file] (accessed through the Docker Engine settings pane if you're using Docker Desktop) and restarting the daemon: + +[source,json] +---- +{ + "dns": ["8.8.8.8"] +} +---- + +=== Cloning the UnrealEngine Git repository fails with the message `error: unable to read askpass response from 'C:\git-credential-helper.bat'` (for Windows containers) or `'/tmp/git-credential-helper.sh'` (for Linux containers) + +This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. +This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. +The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. +(Under Windows Server, the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) + +=== Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file + +This is typically caused by insufficient available disk space. +To fix this, simply free up some disk space and run the build again. +Running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] can be helpful for freeing up space occupied by untagged images. +Note that restarting the Docker daemon and/or rebooting the host system may also help, since some versions of Docker have a bug that results in the amount of required disk space slowly increasing as more and more builds are run. + +=== Building Windows containers fails with the message `hcsshim::ImportLayer failed in Win32: The system cannot find the path specified` or building Linux containers fails with a message about insufficient disk space + +Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. + +- For Windows containers, follow https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size[the instructions provided by Microsoft], making sure you restart the Docker daemon after you've modified the config JSON. +(Under Windows Server, the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) +- For Linux containers, use the https://docs.docker.com/docker-for-windows/#advanced[Docker for Windows "Advanced" settings tab] under Windows or the https://docs.docker.com/docker-for-mac/#disk[Docker for Mac "Disk" settings tab] under macOS. + +=== Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image + +WARNING: Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. +Since ue4-docker version 0.0.47, you can use the link:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. +If it does, you may need to link:advanced-build-options.adoc#excluding-engine-components-to-reduce-the-final-image-size[exclude debug symbols] when building Windows images. + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: + +- https://github.com/moby/moby/issues/37581 (affects all platforms) +- https://github.com/moby/moby/issues/40444 (affects Windows containers only) + +https://github.com/moby/moby/issues/37581[#37581] was https://github.com/moby/moby/pull/37771[fixed] in Docker CE 18.09.0, whilst https://github.com/moby/moby/issues/40444[#40444] was https://github.com/moby/moby/pull/41430[fixed] in Docker CE 20.10.0. + +If you are using a version of Docker that contains one of these bugs then you will need to link:advanced-build-options.adoc#excluding-engine-components-to-reduce-the-final-image-size[exclude debug symbols], which reduces the size of the Installed Build below the 8GiB threshold. +If debug symbols are required then it will be necessary to upgrade or downgrade to a version of Docker that does not suffer from the 8GiB size limit issue (although finding such a version under Windows may prove quite difficult.) + +== Linux-specific issues + +=== Building the Engine in a Linux container fails with an error indicating that a compatible version of clang cannot be found or the file `ToolchainVersion.txt` is missing + +This is typically caused by the download of the Unreal Engine's toolchain archive from the Epic Games CDN becoming corrupted and failing to extract correctly. +This issue can occur both inside containers and when running directly on a host system, and the fix is to simply delete the corrupted files and try again: + +. Untag the available-container-images.adoc#ue4-source[ue4-source] image. +. Clear the Docker filesystem layer cache by running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune]. +. Re-run the link:../commands/ue4-docker-build.adoc[ue4-docker build] command. + +== Windows-specific issues + +=== Building the `ue4-build-prerequisites` image fails with an unknown error + +Microsoft issued a security update in February 2020 that https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t[broke container compatibility for all versions of Windows Server and caused 32-bit applications to fail silently when run]. +The issue is resolved by ensuring that both the host system and the container image are using versions of Windows that incorporate the fix: + +- Make sure your host system is up-to-date and all available Windows updates are installed. +- Make sure you are using the latest version of ue4-docker, which automatically uses container images that incorporate the fix. + +=== Building Windows containers fails with the message `hcsshim: timeout waiting for notification extra info` or the message `This operation ended because the timeout has expired` + +Recent versions of Docker under Windows may sometimes encounter the error https://github.com/Microsoft/hcsshim/issues/152[hcsshim: timeout waiting for notification extra info] when building or running Windows containers. +This is a known issue when using Windows containers in https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode]. +At the time of writing, Microsoft have stated that they are aware of the problem, but an official fix is yet to be released. + +As a workaround until a proper fix is issued, it seems that altering the memory limit for containers between subsequent invocations of the `docker` command can reduce the frequency with which this error occurs. +(Changing the memory limit when using Hyper-V isolation likely forces Docker to provision a new Hyper-V VM, preventing it from re-using an existing one that has become unresponsive.) Please note that this workaround has been devised based on my own testing under Windows 10 and may not hold true when using Hyper-V isolation under Windows Server. + +To enable the workaround, specify the `--random-memory` flag when invoking the build command. +This will set the container memory limit to a random value between 10GB and 12GB when the build command starts. +If a build fails with the `hcsshim` timeout error, simply re-run the build command and in most cases the build will continue successfully, even if only for a short while. +Restarting the Docker daemon may also help. + +Note that some older versions of UnrealBuildTool will crash with an error stating *"The process cannot access the file because it is being used by another process"* when using a memory limit that is not a multiple of 4GB. +If this happens, simply run the build command again with an appropriate memory limit (e.g. `-m 8GB` or `-m 12GB`.) If the access error occurs even when using an appropriate memory limit, this likely indicates that Windows is unable to allocate the full amount of memory to the container. +Rebooting the host system may help to alleviate this issue. + +=== Building or running Windows containers fails with the message `The operating system of the container does not match the operating system of the host` + +This error is shown in two situations: + +- The host system is running an **older kernel version** than the container image. +In this case, you will need to build the images using the same kernel version as the host system or older. +See link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. +- The host system is running a **newer kernel version** than the container image and you are attempting to use process isolation mode instead of Hyper-V isolation mode. +(Process isolation mode is the default under Windows Server.) In this case, you will need to use Hyper-V isolation mode instead. +See link:advanced-build-options.adoc#specifying-the-isolation-mode-under-windows[Specifying the isolation mode under Windows] for details on how to do this. + +=== Pulling the .NET Framework base image fails with the message `ProcessUtilityVMImage \\?\`(long path here)`\UtilityVM: The system cannot find the path specified` + +This is a known issue when the host system is running an older kernel version than the container image. +Just like in the case of *"The operating system of the container does not match the operating system of the host"* error mentioned above, you will need to build the images using the same kernel version as the host system or older. +See link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. + +=== Building the Engine in a Windows container fails with the message `The process cannot access the file because it is being used by another process` + +This is a known bug in some older versions of UnrealBuildTool when using a memory limit that is not a multiple of 4GB. +To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the <> section. + +=== Building the Engine in a Windows container fails with the message `fatal error LNK1318: Unexpected PDB error; OK (0)` + +This is a known bug in some versions of Visual Studio which only appears to occur intermittently. +The simplest fix is to simply reboot the host system and then re-run the build command. +Insufficient available memory may also contribute to triggering this bug. +Note that a linker wrapper https://docs.unrealengine.com/en-US/Support/Builds/ReleaseNotes/4_24/index.html[was added in Unreal Engine 4.24.0] to automatically retry link operations in the event that this bug occurs, so it shouldn't be an issue when building version 4.24.0 or newer. + +=== Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` + +This error typically occurs when the Windows pagefile size is not large enough. +As stated in the link:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. +However, containers running in process isolation mode will use the pagefile settings of the host system. +When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. +(Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) + +=== Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system + +As described in the link:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. +As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. +The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/building-images/troubleshooting-build-issues.md b/docs/building-images/troubleshooting-build-issues.md deleted file mode 100644 index 69574c72..00000000 --- a/docs/building-images/troubleshooting-build-issues.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -title: Troubleshooting build issues -pagenum: 3 ---- - -## General issues - -### Building the `ue4-build-prerequisites` image fails with a network-related error - -This indicates an underlying network or proxy server issue outside of ue4-docker itself that you will need to troubleshoot. You can use the [ue4-docker diagnostics](../commands/diagnostics) command to test container network connectivity during the troubleshooting process. Here are some steps to try: - -- If your host system accesses the network through a proxy server, make sure that [Docker is configured to use the correct proxy server settings](https://docs.docker.com/network/proxy/). -- If DNS resolution is failing then try adding the following entry to your [Docker daemon configuration file](https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file) (accessed through the Docker Engine settings pane if you're using Docker Desktop) and restarting the daemon: - -```json -"dns": ["8.8.8.8"] -``` - - -### Cloning the UnrealEngine Git repository fails with the message `error: unable to read askpass response from 'C:\git-credential-helper.bat'` (for Windows containers) or `'/tmp/git-credential-helper.sh'` (for Linux containers) - -This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. (Under Windows Server, the [ue4-docker setup](../commands/setup) command can configure the firewall rule for you automatically.) - - -### Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file - -This is typically caused by insufficient available disk space. To fix this, simply free up some disk space and run the build again. Running [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/) can be helpful for freeing up space occupied by untagged images. Note that restarting the Docker daemon and/or rebooting the host system may also help, since some versions of Docker have a bug that results in the amount of required disk space slowly increasing as more and more builds are run. - - -### Building Windows containers fails with the message `hcsshim::ImportLayer failed in Win32: The system cannot find the path specified` or building Linux containers fails with a message about insufficient disk space - -Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. - -- For Windows containers, follow [the instructions provided by Microsoft](https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size), making sure you restart the Docker daemon after you've modified the config JSON. (Under Windows Server, the [ue4-docker setup](../commands/setup) command can configure this for you automatically.) -- For Linux containers, use the [Docker for Windows "Advanced" settings tab](https://docs.docker.com/docker-for-windows/#advanced) under Windows or the [Docker for Mac "Disk" settings tab](https://docs.docker.com/docker-for-mac/#disk) under macOS. - - -### Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image - -{% capture _alert_content %} -Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. Since ue4-docker version 0.0.47, you can use the [ue4-docker diagnostics](../commands/diagnostics) command to check whether the Docker daemon on your system suffers from this issue. If it does, you may need to [exclude debug symbols](./advanced-build-options#excluding-engine-components-to-reduce-the-final-image-size) when building Windows images. -{% endcapture %} -{% include alerts/warning.html content=_alert_content %} - -Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: - -- (affects all platforms) -- (affects Windows containers only) - -[#37581](https://github.com/moby/moby/issues/37581) was [fixed](https://github.com/moby/moby/pull/37771) in Docker CE 18.09.0, whilst [#40444](https://github.com/moby/moby/issues/40444) was [fixed](https://github.com/moby/moby/pull/41430) in Docker CE 20.10.0. - -If you are using a version of Docker that contains one of these bugs then you will need to [exclude debug symbols](./advanced-build-options#excluding-engine-components-to-reduce-the-final-image-size), which reduces the size of the Installed Build below the 8GiB threshold. If debug symbols are required then it will be necessary to upgrade or downgrade to a version of Docker that does not suffer from the 8GiB size limit issue (although finding such a version under Windows may prove quite difficult.) - - -## Linux-specific issues - -### Building the Engine in a Linux container fails with an error indicating that a compatible version of clang cannot be found or the file `ToolchainVersion.txt` is missing - -This is typically caused by the download of the Unreal Engine's toolchain archive from the Epic Games CDN becoming corrupted and failing to extract correctly. This issue can occur both inside containers and when running directly on a host system, and the fix is to simply delete the corrupted files and try again: - -1. Untag the [ue4-source](./available-container-images#ue4-source) image. -2. Clear the Docker filesystem layer cache by running [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/). -3. Re-run the [ue4-docker build](../commands/build) command. - - -## Windows-specific issues - - -### Building the `ue4-build-prerequisites` image fails with an unknown error - -Microsoft issued a security update in February 2020 that [broke container compatibility for all versions of Windows Server and caused 32-bit applications to fail silently when run](https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t). The issue is resolved by ensuring that both the host system and the container image are using versions of Windows that incorporate the fix: - -- Make sure your host system is up-to-date and all available Windows updates are installed. -- Make sure you are using the latest version of ue4-docker, which automatically uses container images that incorporate the fix. - - -### Building Windows containers fails with the message `hcsshim: timeout waiting for notification extra info` or the message `This operation ended because the timeout has expired` - -Recent versions of Docker under Windows may sometimes encounter the error [hcsshim: timeout waiting for notification extra info](https://github.com/Microsoft/hcsshim/issues/152) when building or running Windows containers. This is a known issue when using Windows containers in [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container). At the time of writing, Microsoft have stated that they are aware of the problem, but an official fix is yet to be released. - -As a workaround until a proper fix is issued, it seems that altering the memory limit for containers between subsequent invocations of the `docker` command can reduce the frequency with which this error occurs. (Changing the memory limit when using Hyper-V isolation likely forces Docker to provision a new Hyper-V VM, preventing it from re-using an existing one that has become unresponsive.) Please note that this workaround has been devised based on my own testing under Windows 10 and may not hold true when using Hyper-V isolation under Windows Server. - -To enable the workaround, specify the `--random-memory` flag when invoking the build command. This will set the container memory limit to a random value between 10GB and 12GB when the build command starts. If a build fails with the `hcsshim` timeout error, simply re-run the build command and in most cases the build will continue successfully, even if only for a short while. Restarting the Docker daemon may also help. - -Note that some older versions of UnrealBuildTool will crash with an error stating *"The process cannot access the file because it is being used by another process"* when using a memory limit that is not a multiple of 4GB. If this happens, simply run the build command again with an appropriate memory limit (e.g. `-m 8GB` or `-m 12GB`.) If the access error occurs even when using an appropriate memory limit, this likely indicates that Windows is unable to allocate the full amount of memory to the container. Rebooting the host system may help to alleviate this issue. - - -### Building or running Windows containers fails with the message `The operating system of the container does not match the operating system of the host` - -This error is shown in two situations: - -- The host system is running an **older kernel version** than the container image. In this case, you will need to build the images using the same kernel version as the host system or older. See [Specifying the Windows Server Core base image tag](./advanced-build-options#specifying-the-windows-server-core-base-image-tag) for details on specifying the correct kernel version when building Windows container images. -- The host system is running a **newer kernel version** than the container image and you are attempting to use process isolation mode instead of Hyper-V isolation mode. (Process isolation mode is the default under Windows Server.) In this case, you will need to use Hyper-V isolation mode instead. See [Specifying the isolation mode under Windows](./advanced-build-options#specifying-the-isolation-mode-under-windows) for details on how to do this. - - -### Pulling the .NET Framework base image fails with the message `ProcessUtilityVMImage \\?\`(long path here)`\UtilityVM: The system cannot find the path specified` - -This is a known issue when the host system is running an older kernel version than the container image. Just like in the case of the *"The operating system of the container does not match the operating system of the host"* error mentioned above, you will need to build the images using the same kernel version as the host system or older. See [Specifying the Windows Server Core base image tag](./advanced-build-options#specifying-the-windows-server-core-base-image-tag) for details on specifying the correct kernel version when building Windows container images. - - -### Building the Engine in a Windows container fails with the message `The process cannot access the file because it is being used by another process` - -This is a known bug in some older versions of UnrealBuildTool when using a memory limit that is not a multiple of 4GB. To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the [`hcsshim` timeout issues](#building-windows-containers-fails-with-the-message-hcsshim-timeout-waiting-for-notification-extra-info-or-the-message-this-operation-ended-because-the-timeout-has-expired) section. - - -### Building the Engine in a Windows container fails with the message `fatal error LNK1318: Unexpected PDB error; OK (0)` - -This is a known bug in some versions of Visual Studio which only appears to occur intermittently. The simplest fix is to simply reboot the host system and then re-run the build command. Insufficient available memory may also contribute to triggering this bug. Note that a linker wrapper [was added in Unreal Engine 4.24.0](https://docs.unrealengine.com/en-US/Support/Builds/ReleaseNotes/4_24/index.html) to automatically retry link operations in the event that this bug occurs, so it shouldn't be an issue when building version 4.24.0 or newer. - - -### Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` - -This error typically occurs when the Windows pagefile size is not large enough. As stated in the [Windows containers primer](../read-these-first/windows-container-primer#hyper-v-isolation-mode-issues), there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. However, containers running in process isolation mode will use the pagefile settings of the host system. When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. (Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) - - -### Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system - -As described in the [Windows containers primer](../read-these-first/windows-container-primer#hyper-v-isolation-mode-issues), the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/commands/build.md b/docs/commands/build.md deleted file mode 100644 index 9d833934..00000000 --- a/docs/commands/build.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -title: ue4-docker build -blurb: Builds container images for a specific version of UE4. -pagenum: 2 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker build [-h] [--linux] [--rebuild] [--dry-run] - [--pull-prerequisites] [--no-engine] [--no-minimal] [--no-full] - [--no-cache] [--random-memory] - [--exclude {debug,templates}] [--opt OPT] [--cuda VERSION] - [-username USERNAME] [-password PASSWORD] - [-repo REPO] [-branch BRANCH] [-isolation ISOLATION] - [-basetag BASETAG] [-dlldir DLLDIR] [-suffix SUFFIX] - [-m M] - [-ue4cli UE4CLI] [-conan-ue4cli CONAN_UE4CLI] - [-layout LAYOUT] [--combine] - [--monitor] [-interval INTERVAL] - [--ignore-blacklist] - [--visual-studio VISUAL_STUDIO] - [-v] - release -{% endhighlight %} - - -## Basic usage - -To build container images for a specific version of the Unreal Engine, simply specify the UE4 release that you would like to build using full [semver](https://semver.org/) version syntax. For example, to build Unreal Engine 4.20.3, run: - -{% highlight bash %} -ue4-docker build 4.20.3 -{% endhighlight %} - -You will be prompted for the Git credentials to be used when cloning the UE4 GitHub repository (this will be the GitHub username and password you normally use when cloning .) The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. - -By default, all available images will be built. See the [List of available container images](../building-images/available-container-images) page for details on customising which images are built. - - -## Advanced usage - -See the [Advanced build options](../building-images/advanced-build-options) page for details on all of the supported flags for customising build behaviour. diff --git a/docs/commands/clean.md b/docs/commands/clean.md deleted file mode 100644 index 6b95e771..00000000 --- a/docs/commands/clean.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: ue4-docker clean -blurb: Cleans built container images. -pagenum: 3 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker clean [-tag TAG] [--source] [--engine] [--all] [--dry-run] -{% endhighlight %} - -By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed. You can customise this behaviour using these flags: - -- `-tag TAG`: Applies a filter for the three flags below, restricting them to removing only images with the specified tag (e.g. `-tag 4.21.0` will only remove images for 4.21.0.) -- `--source`: Removes [ue4-source](../building-images/available-container-images#ue4-source) images, applying the tag filter if one was specified -- `--engine`: Removes [ue4-engine](../building-images/available-container-images#ue4-engine) images, applying the tag filter if one was specified -- `--all`: Removes all ue4-docker images, applying the tag filter if one was specified - -If you're unsure as to exactly what images will be removed by a given invocation of the command, append the `--dry-run` flag to have ue4-docker print the generated `docker rmi` commands instead of running them. \ No newline at end of file diff --git a/docs/commands/diagnostics.md b/docs/commands/diagnostics.md deleted file mode 100644 index afe623c5..00000000 --- a/docs/commands/diagnostics.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: ue4-docker diagnostics -blurb: Runs diagnostics to detect issues with the host system configuration. -pagenum: 4 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker diagnostics DIAGNOSTIC -{% endhighlight %} - -This command can be used to run the following diagnostics: - -* TOC -{:toc} - - -## Checking for the Docker 8GiB filesystem layer bug - -Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, [breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols](../building-images/troubleshooting-build-issues#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image). - -This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: - -{% highlight bash %} -ue4-docker diagnostics 8gig -{% endhighlight %} - - -## Checking for the Windows Host Compute Service (HCS) `storage-opt` bug - -Windows Server versions 1903 and 1909 and Windows 10 versions 1903 and 1909 contain [a bug in the Host Compute Service (HCS)](https://github.com/docker/for-win/issues/4100) that prevents users from increasing the maximum allowed image size using Docker's `storage-opt` configuration key. Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the [ue4-docker build](./build) command from functioning correctly on affected systems. - -This diagnostic tests whether the host system is affected this bug, by attempting to run a container with a non-default `storage-opt` value: - -{% highlight bash %} -ue4-docker diagnostics maxsize -{% endhighlight %} - - -## Checking for container network connectivity issues - -This diagnostic tests whether running containers are able to access the internet, resolve DNS entries, and download remote files: - -{% highlight bash %} -ue4-docker diagnostics network -{% endhighlight %} diff --git a/docs/commands/export.md b/docs/commands/export.md deleted file mode 100644 index 4df4ff62..00000000 --- a/docs/commands/export.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: ue4-docker export -blurb: Exports components from built container images to the host system. -pagenum: 5 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker export COMPONENT TAG DESTINATION -{% endhighlight %} - -This command can be used to export the following components: - -* TOC -{:toc} - - -## Exporting Installed Builds under Linux - -Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the [ue4-minimal](../building-images/available-container-images#ue4-minimal) or [ue4-full](../building-images/available-container-images#ue4-full) image for the UE4 version that you want to export, you can export it to the host system like so: - -{% highlight bash %} -# Example: specify a version without a full image tag (assumes `adamrehn/ue4-full`) -# Exports the Installed Build from `adamrehn/ue4-full:4.27.0` to the directory `~/UnrealInstalled` on the host system -ue4-docker export installed "4.27.0" ~/UnrealInstalled - -# Example: specify a full image tag -# Exports the Installed Build from `adamrehn/ue4-minimal:4.27.0` to the directory `~/UnrealInstalled` on the host system -ue4-docker export installed "adamrehn/ue4-minimal:4.27.0" ~/UnrealInstalled - -# Example: use a container image based on ue4-minimal with a completely different tag -# Exports the Installed Build from `ghcr.io/epicgames/unreal-engine:dev-4.27.0` to the directory `~/UnrealInstalled` on the host system -ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/UnrealInstalled -{% endhighlight %} - - -## Exporting Conan packages - -The Conan wrapper packages generated by [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}) can be exported from the [ue4-full](../building-images/available-container-images#ue4-full) image to the local Conan package cache on the host system like so: - -{% highlight bash %} -ue4-docker export packages 4.27.0 cache -{% endhighlight %} - -[Conan](https://conan.io/) will need to be installed on the host system for this to work. To use the exported packages for development on the host system, you will also need to generate the accompanying profile-wide packages by running the command: - -{% highlight bash %} -ue4 conan generate --profile-only -{% endhighlight %} - -This will require both [ue4cli]({{ site.data.common.projects.ue4cli.repo }}) and [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}) to be installed on the host system. diff --git a/docs/commands/info.md b/docs/commands/info.md deleted file mode 100644 index 7a82c9f3..00000000 --- a/docs/commands/info.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: ue4-docker info -blurb: Displays information about the host system and Docker daemon. -pagenum: 6 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker info -{% endhighlight %} - -The command will output the following information: - -- The ue4-docker version -- The host OS version -- The Docker daemon version -- Whether or not the NVIDIA Container Toolkit is supported under the current host configuration -- The detected configuration value for the maximum image size for Windows containers -- The total amount of detected system memory -- The number of detected physical and logical CPUs diff --git a/docs/commands/list-of-commands.md b/docs/commands/list-of-commands.md deleted file mode 100644 index f2110f06..00000000 --- a/docs/commands/list-of-commands.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: List of ue4-docker commands -pagenum: 1 ---- - -The ue4-docker command line interface exposes the following commands: - -{::nomarkdown} -{% assign commands = site.documents | where: "subsite", page.subsite | where: "chapter", page.chapter | where_exp: "page", "page.pagenum != null" | where_exp: "page", "page.pagenum > 1" | sort: "pagenum" %} - -{:/} diff --git a/docs/commands/test.md b/docs/commands/test.md deleted file mode 100644 index 50bc2da2..00000000 --- a/docs/commands/test.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: ue4-docker test -blurb: Runs tests to verify the correctness of built container images. -pagenum: 8 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker test TAG -{% endhighlight %} - -This command runs a suite of tests to verify that built [ue4-full](../building-images/available-container-images#ue4-full) container images are functioning correctly and can be used to build and package Unreal projects and plugins. This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/commands/ue4-docker-build.adoc b/docs/commands/ue4-docker-build.adoc new file mode 100644 index 00000000..aee923c7 --- /dev/null +++ b/docs/commands/ue4-docker-build.adoc @@ -0,0 +1,122 @@ += ue4-docker-build (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-build - build container image for a specific version of Unreal Engine + +== Synopsis + +*ue4-docker build* [_OPTION_]... _VERSION_ + +== Description + +To build container images for a specific version of the Unreal Engine, simply specify the release that you would like to build using full https://semver.org/[semver] version syntax. +For example, to build Unreal Engine 4.20.3, run: + +[source,shell] +---- +ue4-docker build 4.20.3 +---- + +You will be prompted for the Git credentials to be used when cloning the Unreal Engine GitHub repository (this will be the GitHub username and password you normally use when cloning . +The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. + +By default, all available images will be built. +See the link:../building-images/available-container-images.adoc[List of available container images] for details on customising which images are built. + +== Options + +*-h, --help*:: +Print help and exit + +*--rebuild**:: +Rebuild images even if they already exist + +*--dry-run*:: +Print `docker build` commands instead of running them + +*--no-minimal*:: +Don't build the ue4-minimal image (deprecated, use --target instead) + +*--no-full*:: +Don't build the ue4-full image (deprecated, use --target instead) + +*--no-cache*:: +Disable Docker build cache + +*--exclude {ddc,debug,templates}*:: +Exclude the specified component (can be specified multiple times to exclude multiple components) + +*--opt OPT*:: +Set an advanced configuration option (can be specified multiple times to specify multiple options) + +*-username USERNAME*:: +Specify the username to use when cloning the git repository + +*-password PASSWORD*:: +Specify access token or password to use when cloning the git repository + +*-repo REPO*:: +Set the custom git repository to clone when "custom" is specified as the release value + +*-branch BRANCH*:: +Set the custom branch/tag to clone when "custom" is specified as the release value + +*-basetag BASETAG*:: +Operating system base image tag to use. +For Linux this is the version of Ubuntu (default is ubuntu18.04). +For Windows this is the Windows Server Core base image tag (default is the host OS version) + +*-suffix SUFFIX*:: +Add a suffix to the tags of the built images + +*-m M*:: +Override the default memory limit under Windows (also overrides --random-memory) + +*-ue4cli UE4CLI*:: +Override the default version of ue4cli installed in the ue4-full image + +*-conan-ue4cli CONAN_UE4CLI*:: +Override the default version of conan-ue4cli installed in the ue4-full image + +*-layout LAYOUT*:: +Copy generated Dockerfiles to the specified directory and don't build the images + +*--combine*:: +Combine generated Dockerfiles into a single multi-stage build Dockerfile + +*--monitor*:: +Monitor resource usage during builds (useful for debugging) + +*-interval INTERVAL*:: +Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) + +*-v*:: +Enable verbose output during builds (useful for debugging) + +== Linux-specific options + +*--cuda VERSION*:: +Add CUDA support as well as OpenGL support + +== Windows-specific options + +*--ignore-blacklist*:: +Run builds even on blacklisted versions of Windows (advanced use only) + +*-isolation ISOLATION*:: +Set the isolation mode to use (process or hyperv) + +*--linux*:: +Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support) + +*--random-memory*:: +Use a random memory limit for Windows containers + +*--visual-studio VISUAL_STUDIO*:: +Specify Visual Studio Build Tools version diff --git a/docs/commands/ue4-docker-clean.adoc b/docs/commands/ue4-docker-clean.adoc new file mode 100644 index 00000000..2a7d7625 --- /dev/null +++ b/docs/commands/ue4-docker-clean.adoc @@ -0,0 +1,32 @@ += ue4-docker-clean (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-clean - cleans built container images. + +== Synopsis + +*ue4-docker clean* [*-tag* _TAG_] [*--source*] [*--all*] [*--dry-run*] + +== Description + +By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed. + +== Options + +*-tag* _TAG_:: +Apply a filter for the three flags below, restricting them to removing only images with the specified _TAG_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) + +*--source*:: +Remove ../building-images/available-container-images.adoc#ue4-source[ue4-source] images, applying the tag filter if one was specified + +*--all*:: +Remove all ue4-docker images, applying the tag filter if one was specified + +*--dry-run*:: +If you're unsure as to exactly what images will be removed by a given invocation of the command, append the `--dry-run` flag to have ue4-docker print the generated `docker rmi` commands instead of running them. diff --git a/docs/commands/ue4-docker-diagnostics.adoc b/docs/commands/ue4-docker-diagnostics.adoc new file mode 100644 index 00000000..8791fdd7 --- /dev/null +++ b/docs/commands/ue4-docker-diagnostics.adoc @@ -0,0 +1,52 @@ += ue4-docker-diagnostics (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker diagnostics - run diagnostics to detect issues with the host system configuration. +{{ page.blurb | escape }} + +== Synopsis + +*ue4-docker diagnostics* _DIAGNOSTIC_ + +== Description + +This command can be used to run the following diagnostics: + +=== Checking for the Docker 8GiB filesystem layer bug + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. +This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, link:../building-images/troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. + +This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: + +[source,shell] +---- +ue4-docker diagnostics 8gig +---- + +=== Checking for the Windows Host Compute Service (HCS) `storage-opt` bug + +Windows Server versions 1903 and 1909 and Windows 10 versions 1903 and 1909 contain https://github.com/docker/for-win/issues/4100[a bug in the Host Compute Service (HCS)] that prevents users from increasing the maximum allowed image size using Docker's `storage-opt` configuration key. +Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the link:ue4-docker-build.adoc[ue4-docker build] command from functioning correctly on affected systems. + +This diagnostic tests whether the host system is affected this bug, by attempting to run a container with a non-default `storage-opt` value: + +[source,shell] +---- +ue4-docker diagnostics maxsize +---- + +=== Checking for container network connectivity issues + +This diagnostic tests whether running containers are able to access the internet, resolve DNS entries, and download remote files: + +[source,shell] +---- +ue4-docker diagnostics network +---- diff --git a/docs/commands/ue4-docker-export.adoc b/docs/commands/ue4-docker-export.adoc new file mode 100644 index 00000000..42932538 --- /dev/null +++ b/docs/commands/ue4-docker-export.adoc @@ -0,0 +1,62 @@ += ue4-docker-export (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-export - export components from built container image to the host system + +== Synopsis + +*ue4-docker export* _COMPONENT_ _TAG_ _DESTINATION_ + +This command can be used to export the following components: + +== Description + +=== Exporting Installed Builds under Linux + +Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or link:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: + +[source,shell] +---- +# Example: specify a version without a full image tag (assumes `adamrehn/ue4-full`) +# Exports the Installed Build from `adamrehn/ue4-full:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "4.27.0" ~/UnrealInstalled +---- + +[source,shell] +---- +# Example: specify a full image tag +# Exports the Installed Build from `adamrehn/ue4-minimal:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "adamrehn/ue4-minimal:4.27.0" ~/UnrealInstalled +---- + +[source,shell] +---- +# Example: use a container image based on ue4-minimal with a completely different tag +# Exports the Installed Build from `ghcr.io/epicgames/unreal-engine:dev-4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/UnrealInstalled +---- + +=== Exporting Conan packages + +The Conan wrapper packages generated by `conan-ue4cli` can be exported from the link:../building-images/available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: + +[source,shell] +---- +ue4-docker export packages 4.27.0 cache +---- + +https://conan.io/[Conan] will need to be installed on the host system for this to work. +To use the exported packages for development on the host system, you will also need to generate the accompanying profile-wide packages by running the command: + +[source,shell] +---- +ue4 conan generate --profile-only +---- + +This will require both `ue4cli` and `conan-ue4cli` to be installed on the host system. diff --git a/docs/commands/ue4-docker-info.adoc b/docs/commands/ue4-docker-info.adoc new file mode 100644 index 00000000..4820b368 --- /dev/null +++ b/docs/commands/ue4-docker-info.adoc @@ -0,0 +1,26 @@ += ue4-docker-info (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-info - display information about the host system and Docker daemon + +== Synopsis + +*ue4-docker info* + +== Description + +The command will output the following information: + +- The ue4-docker version +- The host OS version +- The Docker daemon version +- Whether the NVIDIA Container Toolkit is supported under the current host configuration +- The detected configuration value for the maximum image size for Windows containers +- The total amount of detected system memory +- The number of detected physical and logical CPUs diff --git a/docs/commands/setup.md b/docs/commands/ue4-docker-setup.adoc similarity index 52% rename from docs/commands/setup.md rename to docs/commands/ue4-docker-setup.adoc index 0947bc32..ef867dee 100644 --- a/docs/commands/setup.md +++ b/docs/commands/ue4-docker-setup.adoc @@ -1,27 +1,31 @@ ---- -title: ue4-docker setup -blurb: Automatically configures the host system where possible. -pagenum: 7 ---- +== ue4-docker-setup (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge -{{ page.blurb | escape }} +== Name -**Usage syntax:** +ue4-docker-setup - automatically configure the host system where possible -{% highlight bash %} -ue4-docker setup -{% endhighlight %} +== Synopsis + +*ue4-docker setup* + +== Description This command will automatically configure a Linux or Windows Server host system with the settings required in order to build and run containers produced by ue4-docker. **Under Linux:** -- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image. +- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. **Under Windows Server:** - The Docker daemon will be configured to set the maximum image size for Windows containers to 400GB. -- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image. -- Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the [full Windows base image](https://hub.docker.com/_/microsoft-windows). Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. +- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. +- Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the https://hub.docker.com/_/microsoft-windows[full Windows base image]. +Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. **Under Windows 10 and macOS** this command will print a message informing the user that automatic configuration is not supported under their platform and that they will need to configure the system manually. diff --git a/docs/commands/ue4-docker-test.adoc b/docs/commands/ue4-docker-test.adoc new file mode 100644 index 00000000..00450bc5 --- /dev/null +++ b/docs/commands/ue4-docker-test.adoc @@ -0,0 +1,20 @@ += ue4-docker-test (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-test - run tests to verify the correctness of built container images + +== Synopsis + +*ue4-docker test* _TAG_ + +== Description + +This command runs a suite of tests to verify that built link:../building-images/available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. + +This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/commands/ue4-docker-version.adoc b/docs/commands/ue4-docker-version.adoc new file mode 100644 index 00000000..5116b021 --- /dev/null +++ b/docs/commands/ue4-docker-version.adoc @@ -0,0 +1,18 @@ += ue4-docker-version (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-version - display version information about ue4-docker + +== Synopsis + +*ue4-docker version* + +== Description + +Prints ue4-docker version on the standard output. diff --git a/docs/commands/version.md b/docs/commands/version.md deleted file mode 100644 index 01f3e41e..00000000 --- a/docs/commands/version.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -title: ue4-docker version -blurb: Prints the ue4-docker version number. -pagenum: 9 ---- - -{{ page.blurb | escape }} - -**Usage syntax:** - -{% highlight bash %} -ue4-docker version -{% endhighlight %} diff --git a/docs/configuration/configuring-linux.adoc b/docs/configuration/configuring-linux.adoc new file mode 100644 index 00000000..1b9c16b3 --- /dev/null +++ b/docs/configuration/configuring-linux.adoc @@ -0,0 +1,72 @@ += Configuring Linux +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Requirements + +- 64-bit version of one of Docker's https://docs.docker.com/install/#supported-platforms[supported Linux distributions] (CentOS 7+, Debian 7.7+, Fedora 26+, Ubuntu 14.04+) +- Minimum 8GB of RAM +- Minimum 400GB available disk space for building container images + +== Step 1: Install Docker CE + +Follow the official installation instructions from the Docker Documentation for your distribution: + +- https://docs.docker.com/install/linux/docker-ce/centos/[CentOS] +- https://docs.docker.com/install/linux/docker-ce/debian/[Debian] +- https://docs.docker.com/install/linux/docker-ce/fedora/[Fedora] +- https://docs.docker.com/install/linux/docker-ce/ubuntu/[Ubuntu] + +Once Docker is installed, follow the instructions from the https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user[Post-installation steps for Linux] page of the Docker Documentation to allow Docker commands to be run by a non-root user. +This step is required in order to enable audio support when performing cloud rendering using the NVIDIA Container Toolkit. + +== Step 2: Install Python 3.6 or newer + +WARNING: Note that older versions of these Linux distributions may not have Python 3.6 available in their system repositories by default. +When working with an older distribution it may be necessary to configure community repositories that provide newer versions of Python. + +Under CentOS, run: + +[source,shell] +---- +sudo yum install python3 python3-devel python3-pip +---- + +Under Debian and Ubuntu, run: + +[source,shell] +---- +sudo apt-get install python3 python3-dev python3-pip +---- + +Under Fedora, run: + +[source,shell] +---- +sudo dnf install python3 python3-devel python3-pip +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command: + +[source,shell] +---- +sudo pip3 install ue4-docker +---- + +== Step 4: Use ue4-docker to automatically configure the Linux firewall + +If the host system is running an active firewall that blocks access to port 9876 (required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. +The link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. +Simply run: + +[source,shell] +---- +sudo ue4-docker setup +---- + +Note that the `iptables-persistent` service will need to be installed for the newly-created firewall rule to persist after the host system reboots. diff --git a/docs/configuration/configuring-linux.md b/docs/configuration/configuring-linux.md deleted file mode 100644 index 551e15fd..00000000 --- a/docs/configuration/configuring-linux.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: Configuring Linux -pagenum: 2 ---- - -## Requirements - -- 64-bit version of one of Docker's [supported Linux distributions](https://docs.docker.com/install/#supported-platforms) (CentOS 7+, Debian 7.7+, Fedora 26+, Ubuntu 14.04+) -- Minimum 8GB of RAM -- Minimum {{ site.data.ue4-docker.common.diskspace_linux | escape }} available disk space for building container images - - -## Step 1: Install Docker CE - -Follow the official installation instructions from the Docker Documentation for your distribution: - -- [CentOS](https://docs.docker.com/install/linux/docker-ce/centos/) -- [Debian](https://docs.docker.com/install/linux/docker-ce/debian/) -- [Fedora](https://docs.docker.com/install/linux/docker-ce/fedora/) -- [Ubuntu](https://docs.docker.com/install/linux/docker-ce/ubuntu/) - -Once Docker is installed, follow the instructions from the [Post-installation steps for Linux](https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user) page of the Docker Documentation to allow Docker commands to be run by a non-root user. This step is required in order to enable audio support when performing cloud rendering using the NVIDIA Container Toolkit. - - -## Step 2: Install Python 3.6 or newer - -{% capture _alert_content %} -Note that older versions of these Linux distributions may not have Python 3.6 available in their system repositories by default. When working with an older distribution it may be necessary to configure community repositories that provide newer versions of Python. -{% endcapture %} -{% include alerts/info.html content=_alert_content %} - -Under CentOS, run: - -{% highlight bash %} -sudo yum install python3 python3-devel python3-pip -{% endhighlight %} - -Under Debian and Ubuntu, run: - -{% highlight bash %} -sudo apt-get install python3 python3-dev python3-pip -{% endhighlight %} - -Under Fedora, run: - -{% highlight bash %} -sudo dnf install python3 python3-devel python3-pip -{% endhighlight %} - - -## Step 3: Install ue4-docker - -Install the ue4-docker Python package by running the following command: - -{% highlight console %} -sudo pip3 install ue4-docker -{% endhighlight %} - - -## Step 4: Use ue4-docker to automatically configure the Linux firewall - -If the host system is running an active firewall that blocks access to port 9876 (required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image) then it is necessary to create a firewall rule to permit access to this port. The [ue4-docker setup](../commands/setup) command will detect this scenario and perform the appropriate firewall configuration automatically. Simply run: - -{% highlight console %} -sudo ue4-docker setup -{% endhighlight %} - -Note that the `iptables-persistent` service will need to be installed for the newly-created firewall rule to persist after the host system reboots. diff --git a/docs/configuration/configuring-macos.adoc b/docs/configuration/configuring-macos.adoc new file mode 100644 index 00000000..f513e595 --- /dev/null +++ b/docs/configuration/configuring-macos.adoc @@ -0,0 +1,51 @@ += Configuring macOS +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +WARNING: macOS provides a suboptimal experience when running Linux containers, due to the following factors: +Linux containers are unable to use GPU acceleration via the link:../read-these-first/nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. + +== Requirements + +- 2010 or newer model Mac hardware +- macOS 10.10.3 Yosemite or newer +- Minimum 8GB of RAM +- Minimum 400GB available disk space for building container images + +== Step 1: Install Docker CE for Mac + +Download and install https://store.docker.com/editions/community/docker-ce-desktop-mac[Docker CE for Mac from the Docker Store]. + +== Step 2: Install Python 3 via Homebrew + +The simplest way to install Python 3 and pip under macOS is to use the https://brew.sh/[Homebrew package manager]. +To do so, run the following commands from a Terminal prompt: + +[source,shell] +---- +# Install Homebrew +/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install) +---- + +== Install Python + +[source,shell] +---- +brew install python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from a Terminal prompt: + +[source,shell] +---- +sudo pip3 install ue4-docker +---- + +== Step 4: Manually configure Docker daemon settings + +Use https://docs.docker.com/desktop/mac/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 400GB. diff --git a/docs/configuration/configuring-macos.md b/docs/configuration/configuring-macos.md deleted file mode 100644 index a4ff7fcc..00000000 --- a/docs/configuration/configuring-macos.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -title: Configuring macOS -pagenum: 5 ---- - -{% capture _alert_content %} -macOS provides a sub-optimal experience when running Linux containers, due to the following factors: - -- Linux containers are unable to use GPU acceleration via the [NVIDIA Container Toolkit](../read-these-first/nvidia-docker-primer). -{% endcapture %} -{% include alerts/warning.html content=_alert_content %} - - -## Requirements - -- 2010 or newer model Mac hardware -- macOS 10.10.3 Yosemite or newer -- Minimum 8GB of RAM -- Minimum {{ site.data.ue4-docker.common.diskspace_linux | escape }} available disk space for building container images - - -## Step 1: Install Docker CE for Mac - -Download and install [Docker CE for Mac from the Docker Store](https://store.docker.com/editions/community/docker-ce-desktop-mac). - - -## Step 2: Install Python 3 via Homebrew - -The simplest way to install Python 3 and pip under macOS is to use the [Homebrew package manager](https://brew.sh/). To do so, run the following commands from a Terminal prompt: - -{% highlight bash %} -# Install Homebrew -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" - -# Install Python -brew install python -{% endhighlight %} - - -## Step 3: Install ue4-docker - -Install the ue4-docker Python package by running the following command from a Terminal prompt: - -{% highlight console %} -sudo pip3 install ue4-docker -{% endhighlight %} - - -## Step 4: Manually configure Docker daemon settings - -Use [the Advanced section under the Resources tab of the Docker Desktop settings pane](https://docs.docker.com/desktop/mac/#resources) to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to {{ site.data.ue4-docker.common.diskspace_linux | escape }}. diff --git a/docs/configuration/configuring-windows-10.adoc b/docs/configuration/configuring-windows-10.adoc new file mode 100644 index 00000000..f684bb40 --- /dev/null +++ b/docs/configuration/configuring-windows-10.adoc @@ -0,0 +1,75 @@ += Configuring Windows 10 and 11 +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Warning + +Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to link:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +The default isolation mode depends on the specific version of Windows being used: + +- Under Windows 10, Hyper-V isolation mode is the default isolation mode and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-[process isolation mode must be manually enabled] each time a container is built or run. +The link:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** + +- Under Windows 11, process isolation mode is the default isolation mode. + +== Requirements + +- 64-bit Windows 10 Pro, Enterprise, or Education (Version 1607 or newer) +- Hardware-accelerated virtualization enabled in the system BIOS/EFI +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker CE for Windows + +Download and install https://store.docker.com/editions/community/docker-ce-desktop-windows[Docker CE for Windows from the Docker Store]. + +== Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the https://chocolatey.org/[Chocolatey package manager]. +To do so, run the following command from an elevated PowerShell prompt: + +[source,powershell] +---- +Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString( 'https://chocolatey.org/install.ps1')) +---- + +You may need to restart your shell for it to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. + +Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +[source,powershell] +---- +choco install -y python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +[source,powershell] +---- +pip install ue4-docker +---- + +== Step 4: Manually configure Docker daemon settings + +For building and running Windows containers: + +- Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Windows containers] rather than Linux containers. +- Configure the Docker daemon to increase the maximum container disk size from the default 20GB limit by following https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits[the instructions provided by Microsoft]. +The 120GB limit specified in the instructions is not quite enough, so set a 400GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** + +WARNING: **The ue4-docker maintainers do not provide support for building and running Linux containers under Windows**, due to the various technical limitations of the Hyper-V and WSL2 backends for Docker Desktop (see https://github.com/adamrehn/ue4-docker/issues/205[this issue] for details of these limitations). +This functionality is still present in ue4-docker for those who choose to use it, but users are solely responsible for troubleshooting any issues they encounter when doing so. + +For building and running Linux containers: + +- Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Linux containers] rather than Windows containers. + +- **If you are using the Hyper-V backend** then use https://docs.docker.com/desktop/windows/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 400GB. + +- **If you are using the WSL2 backend** then https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk[expand the WSL2 virtual hard disk] to at least 400GB. diff --git a/docs/configuration/configuring-windows-10.md b/docs/configuration/configuring-windows-10.md deleted file mode 100644 index 12d0b8f8..00000000 --- a/docs/configuration/configuring-windows-10.md +++ /dev/null @@ -1,71 +0,0 @@ ---- -title: Configuring Windows 10 and 11 -pagenum: 4 ---- - -{% capture _alert_content %} -Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. Using [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container) will result in a sub-optimal experience due to [several issues that impact performance and stability](../read-these-first/windows-container-primer). The default isolation mode depends on the specific version of Windows being used: - -- Under Windows 10, Hyper-V isolation mode is the default isolation mode and [process isolation mode must manually enabled](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-) each time a container is built or run. The [ue4-docker build](../commands/build) command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** - -- Under Windows 11, process isolation mode is the default isolation mode. -{% endcapture %} -{% include alerts/warning.html content=_alert_content %} - - -## Requirements - -- 64-bit Windows 10 Pro, Enterprise, or Education (Version 1607 or newer) -- Hardware-accelerated virtualisation enabled in the system BIOS/EFI -- Minimum 8GB of RAM -- Minimum {{ site.data.ue4-docker.common.diskspace_windows | escape }} available disk space for building container images - - -## Step 1: Install Docker CE for Windows - -Download and install [Docker CE for Windows from the Docker Store](https://store.docker.com/editions/community/docker-ce-desktop-windows). - - -## Step 2: Install Python 3 via Chocolatey - -The simplest way to install Python and pip under Windows is to use the [Chocolatey package manager](https://chocolatey.org/). To do so, run the following command from an elevated PowerShell prompt: - -{% highlight powershell %} -Set-ExecutionPolicy Bypass -Scope Process -Force; -iex ((New-Object System.Net.WebClient).DownloadString( - 'https://chocolatey.org/install.ps1' -)) -{% endhighlight %} - -You may need to restart your shell for it to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: - -{% highlight powershell %} -choco install -y python -{% endhighlight %} - - -## Step 3: Install ue4-docker - -Install the ue4-docker Python package by running the following command from an elevated Command Prompt: - -{% highlight console %} -pip install ue4-docker -{% endhighlight %} - - -## Step 4: Manually configure Docker daemon settings - -For building and running Windows containers: - -- Configure the Docker daemon to [use Windows containers](https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers) rather than Linux containers. -- Configure the Docker daemon to increase the maximum container disk size from the default 20GB limit by following [the instructions provided by Microsoft](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits). The 120GB limit specified in the instructions is not quite enough, so set a 400GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** - -{% include alerts/warning.html content="**The ue4-docker maintainers do not provide support for building and running Linux containers under Windows**, due to the various technical limitations of the Hyper-V and WSL2 backends for Docker Desktop. (See [this issue](https://github.com/adamrehn/ue4-docker/issues/205) for details of these limitations.) This functionality is still present in ue4-docker for those who choose to use it, but users are solely responsible for troubleshooting any issues they encounter when doing so." %} - -For building and running Linux containers: - -- Configure the Docker daemon to [use Linux containers](https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers) rather than Windows containers. - -- **If you are using the Hyper-V backend** then use [the Advanced section under the Resources tab of the Docker Desktop settings pane](https://docs.docker.com/desktop/windows/#resources) to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to {{ site.data.ue4-docker.common.diskspace_linux | escape }}. - -- **If you are using the WSL2 backend** then [expand the WSL2 virtual hard disk](https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk) to at least {{ site.data.ue4-docker.common.diskspace_linux | escape }}. diff --git a/docs/configuration/configuring-windows-server.adoc b/docs/configuration/configuring-windows-server.adoc new file mode 100644 index 00000000..5f7d0a6e --- /dev/null +++ b/docs/configuration/configuring-windows-server.adoc @@ -0,0 +1,73 @@ += Configuring Windows Server +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +IMPORTANT: Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to link:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +Process isolation mode is the default isolation mode under Windows Server. + +== Requirements + +- Windows Server 2016 or newer +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker EE + +As per the instructions provided by the https://docs.docker.com/install/windows/docker-ee/[Install Docker Engine - Enterprise on Windows Servers] page of the Docker Documentation, run the following commands from an elevated PowerShell prompt: + +[source,powershell] +---- +# Add the Docker provider to the PowerShell package manager +Install-Module DockerMsftProvider -Force + +# Install Docker EE +Install-Package Docker -ProviderName DockerMsftProvider -Force + +# Restart the computer to enable the containers feature +Restart-Computer +---- + +== Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the https://chocolatey.org/[Chocolatey package manager]. +To do so, run the following command from an elevated PowerShell prompt: + +[source,powershell] +---- +Set-ExecutionPolicy Bypass -Scope Process -Force; +iex ((New-Object System.Net.WebClient).DownloadString( + 'https://chocolatey.org/install.ps1' +)) +---- + +You may need to restart the system for your shell to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. +Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +[source,shell] +---- +choco install -y python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +[source,shell] +---- +pip install ue4-docker +---- + +== Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall + +To automatically configure the required system settings, run the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: + +[source,shell] +---- +ue4-docker setup +---- + +This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. diff --git a/docs/configuration/configuring-windows-server.md b/docs/configuration/configuring-windows-server.md deleted file mode 100644 index 9226ade1..00000000 --- a/docs/configuration/configuring-windows-server.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Configuring Windows Server -pagenum: 3 ---- - -{% capture _alert_content %} -Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. Using [Hyper-V isolation mode](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container) will result in a sub-optimal experience due to [several issues that impact performance and stability](../read-these-first/windows-container-primer). Process isolation mode is the default isolation mode under Windows Server. -{% endcapture %} -{% include alerts/warning.html content=_alert_content %} - - -## Requirements - -- Windows Server 2016 or newer -- Minimum 8GB of RAM -- Minimum {{ site.data.ue4-docker.common.diskspace_windows | escape }} available disk space for building container images - - -## Step 1: Install Docker EE - -As per the instructions provided by the [Install Docker Engine - Enterprise on Windows Servers](https://docs.docker.com/install/windows/docker-ee/) page of the Docker Documentation, run the following commands from an elevated PowerShell prompt: - -{% highlight powershell %} -# Add the Docker provider to the PowerShell package manager -Install-Module DockerMsftProvider -Force - -# Install Docker EE -Install-Package Docker -ProviderName DockerMsftProvider -Force - -# Restart the computer to enable the containers feature -Restart-Computer -{% endhighlight %} - - -## Step 2: Install Python 3 via Chocolatey - -The simplest way to install Python and pip under Windows is to use the [Chocolatey package manager](https://chocolatey.org/). To do so, run the following command from an elevated PowerShell prompt: - -{% highlight powershell %} -Set-ExecutionPolicy Bypass -Scope Process -Force; -iex ((New-Object System.Net.WebClient).DownloadString( - 'https://chocolatey.org/install.ps1' -)) -{% endhighlight %} - -You may need to restart the system for your shell to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: - -{% highlight powershell %} -choco install -y python -{% endhighlight %} - - -## Step 3: Install ue4-docker - -Install the ue4-docker Python package by running the following command from an elevated Command Prompt: - -{% highlight console %} -pip install ue4-docker -{% endhighlight %} - - -## Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall - -To automatically configure the required system settings, run the [ue4-docker setup](../commands/setup) command from an elevated Command Prompt: - -{% highlight console %} -ue4-docker setup -{% endhighlight %} - -This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the [ue4-source](../building-images/available-container-images#ue4-source) image), and download any required DLL files under Windows Server version 1809 and newer. - - -## Step 5: Copy required DLL files (Windows Server Core version 1803 and older only) - -{% capture _alert_content %} -**This step is only required under when building the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image locally under older versions of Windows Server.** - -Starting from Windows Server version 1809 (base image tag `ltsc2019`), the [ue4-docker setup](../commands/setup) command will automatically extract the required DLL files from the [full Windows base image](https://hub.docker.com/_/microsoft-windowsfamily-windows). Manual configuration is only required under Windows Server version 1803 and older when you are building the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image locally rather than [pulling the prebuilt version](../building-images/advanced-build-options#pulling-a-prebuilt-version-of-the-ue4-build-prerequisites-image). -{% endcapture %} -{% include alerts/info.html content=_alert_content %} - -Under Windows Server Core, you will need to copy a number of DLL files from a copy of Windows 10 (or Windows Server [with the Desktop Experience feature](https://docs.microsoft.com/en-us/windows-server/get-started/getting-started-with-server-with-desktop-experience)) and place them in the `C:\Windows\System32` directory. Note that these DLL files must be copied from the **same version of Windows** as the Windows Server Core host system (e.g. Windows Server 1709 needs DLLs from Windows 10 1709, Windows Server 1803 needs DLLs from Windows 10 1803, etc.) Although DLLs from an older system version may potentially work, Windows containers will refuse to load these DLL files if they have been copied from a version of Windows that is newer than the container. - -The required DLL files for each version of Windows Server Core are listed in the tables below, along with the minimum and maximum versions of these DLLs that have been tested and are known to work. - -{% for server in site.data.ue4-docker.dlls %} -### {{ server.name | escape }} (base image tag `{{ server.basetag | escape }}`) - -{::nomarkdown} -{% if server.notes != "" %} - {% assign content = server.notes | markdownify %} - {% include alerts/info.html content=content %} -{% endif %} - - - - - - - - - - - {% for dll in server.dlls %} - - - - - - {% endfor %} - -
DLL FileMin VersionMax Version
{{ dll.name | escape }}{{ dll.minVersion | escape }}{{ dll.maxVersion | escape }}
-{:/} -{% endfor %} diff --git a/docs/configuration/supported-host-configurations.adoc b/docs/configuration/supported-host-configurations.adoc new file mode 100644 index 00000000..178474b3 --- /dev/null +++ b/docs/configuration/supported-host-configurations.adoc @@ -0,0 +1,43 @@ += Supported host configurations +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +The table below lists the host operating systems can be used to build and run the container images produced by ue4-docker, as well as which features are supported under each system. + +**Click on an operating system's name to view the configuration instructions for that platform.** + +[%autowidth.stretch] +|=== +| Host OS | Linux containers | Windows containers | NVIDIA Container Toolkit | Optimality + +| link:configuring-linux.adoc[Linux] +| icon:check[] +| icon:times[] +| icon:check[] +| Optimal for Linux containers + +| link:configuring-windows-server.adoc[Windows Server] +| icon:times[] +| icon:check[] +| icon:times[] +| Optimal for Windows containers when using process isolation mode + +| link:configuring-windows-10.adoc[Windows 10 and 11] +| Works but not tested or supported +| icon:check[] +| icon:times[] +| Optimal for Windows containers when using process isolation mode + +| link:configuring-macos.adoc[macOS] +| icon:check[] +| icon:times[] +| icon:times[] +| Suboptimal for Linux containers + +|=== + +The *Optimality* column indicates whether a given host operating system provides the best experience for running the container types that it supports. +The configuration instructions page for each operating system provides further details regarding the factors that make it either optimal or suboptimal. diff --git a/docs/configuration/supported-host-configurations.md b/docs/configuration/supported-host-configurations.md deleted file mode 100644 index bdf22da6..00000000 --- a/docs/configuration/supported-host-configurations.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -title: Supported host configurations -pagenum: 1 ---- - -The table below lists the host operating systems can be used to build and run the container images produced by ue4-docker, as well as which features are supported under each system. **Click on an operating system's name to view the configuration instructions for that platform.** - -{::nomarkdown} - - - - - - - - - - - - {% for host in site.data.ue4-docker.hosts %} - - - - - - - - {% endfor %} - -
Host OSLinux containersWindows containersNVIDIA Container ToolkitOptimality
{{ host.name | escape }}{{ host.linux | escape }}{{ host.windows | escape }}{{ host.gpu | escape }}{{ host.optimality | escape }}
-{:/} - -The ***Optimality*** column indicates whether a given host operating system provides the best experience for running the container types that it supports. The configuration instructions page for each operating system provides further details regarding the factors that make it either optimal or sub-optimal. diff --git a/docs/read-these-first/frequently-asked-questions.adoc b/docs/read-these-first/frequently-asked-questions.adoc new file mode 100644 index 00000000..f0a4e06a --- /dev/null +++ b/docs/read-these-first/frequently-asked-questions.adoc @@ -0,0 +1,30 @@ += Frequently Asked Questions +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Why are the Dockerfiles written in such an inefficient manner? There are a large number of `RUN` directives that could be combined to improve both build efficiency and overall image size. + +With the exception of the ../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] and ../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] images, the Dockerfiles have been deliberately written in an inefficient way because doing so serves two very important purposes. + +The first purpose is self-documentation. +These Docker images are the first publicly-available Windows and Linux images to provide comprehensive build capabilities for Unreal Engine 4. +Along with the supporting documentation and https://adamrehn.com/articles/tag/Unreal%20Engine/[articles on adamrehn.com], the code in this repository represents an important source of information regarding the steps that must be taken to get UE4 working correctly inside a container. +The readability of the Dockerfiles is key, which is why they contain so many individual `RUN` directives with explanatory comments. +Combining `RUN` directives would reduce readability and potentially obfuscate the significance of critical steps. + +The second purpose is debuggability. +Updating the Dockerfiles to ensure compatibility with new Unreal Engine releases is an extremely involved process that typically requires building the Engine many times over. +By breaking the Dockerfiles into many fine-grained `RUN` directives, the Docker build cache can be leveraged to ensure only the failing steps need to be repeated when rebuilding the images during debugging. +Combining `RUN` directives would increase the amount of processing that needs to be redone each time one of the commands in a given directive fails, significantly increasing overall debugging times. + +== Can Windows containers be used to perform cloud rendering in the same manner as Linux containers with the NVIDIA Container Toolkit? + +See the answer here: https://unrealcontainers.com/docs/concepts/nvidia-docker#is-there-any-support-for-other-platforms + +== Is it possible to build Unreal projects for macOS or iOS using the Docker containers? + +Building projects for macOS or iOS requires a copy of macOS and Xcode. +Since macOS cannot run inside a Docker container, there is unfortunately no way to perform macOS or iOS builds using Docker containers. diff --git a/docs/read-these-first/frequently-asked-questions.md b/docs/read-these-first/frequently-asked-questions.md deleted file mode 100644 index 873974d5..00000000 --- a/docs/read-these-first/frequently-asked-questions.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Frequently Asked Questions -pagenum: 5 ---- - -## Why are the Dockerfiles written in such an inefficient manner? There are a large number of `RUN` directives that could be combined to improve both build efficiency and overall image size. - -With the exception of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) and [ue4-minimal](../building-images/available-container-images#ue4-minimal) images, the Dockerfiles have been deliberately written in an inefficient way because doing so serves two very important purposes. - -The first purpose is self-documentation. These Docker images are the first publicly-available Windows and Linux images to provide comprehensive build capabilities for Unreal Engine 4. Along with the supporting documentation and [articles on adamrehn.com](https://adamrehn.com/articles/tag/Unreal%20Engine/), the code in this repository represents an important source of information regarding the steps that must be taken to get UE4 working correctly inside a container. The readability of the Dockerfiles is key, which is why they contain so many individual `RUN` directives with explanatory comments. Combining `RUN` directives would reduce readability and potentially obfuscate the significance of critical steps. - -The second purpose is debuggability. Updating the Dockerfiles to ensure compatibility with new Unreal Engine releases is an extremely involved process that typically requires building the Engine many times over. By breaking the Dockerfiles into many fine-grained `RUN` directives, the Docker build cache can be leveraged to ensure only the failing steps need to be repeated when rebuilding the images during debugging. Combining `RUN` directives would increase the amount of processing that needs to be redone each time one of the commands in a given directive fails, significantly increasing overall debugging times. - - -## Can Windows containers be used to perform cloud rendering in the same manner as Linux containers with the NVIDIA Container Toolkit? - -See the answer here: - - -## Is it possible to build Unreal projects for macOS or iOS using the Docker containers? - -Building projects for macOS or iOS requires a copy of macOS and Xcode. Since macOS cannot run inside a Docker container, there is unfortunately no way to perform macOS or iOS builds using Docker containers. diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/read-these-first/introduction-to-ue4-docker.adoc new file mode 100644 index 00000000..1e9d50e2 --- /dev/null +++ b/docs/read-these-first/introduction-to-ue4-docker.adoc @@ -0,0 +1,51 @@ += Introduction to ue4-docker +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Overview + +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' https://www.unrealengine.com/[Unreal Engine]. +The images also incorporate the infrastructure from `ue4cli`, `conan-ue4cli`, and `ue4-ci-helpers` to facilitate a wide variety of use cases. + +Key features include: + +- Unreal Engine 4.20.0 and newer is supported. +- Both Windows containers and Linux containers are supported. +- Building and packaging UE4 projects is supported. +- Running automation tests is supported. +- Running built UE4 projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. + +== Important legal notice + +**With the exception of the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. +As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games UE4 GitHub repository. +**Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.** It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** + +For more details, see the https://unrealcontainers.com/docs/obtaining-images/eula-restrictions[Unreal Engine EULA Restrictions] page on the https://unrealcontainers.com/[Unreal Containers community hub]. + +== Getting started + +Multipurpose Docker images for large, cross-platform projects such as the Unreal Engine involve a great deal more complexity than most typical Docker images. +Before you start using ue4-docker it may be helpful to familiarise yourself with some of these complexities: + +- If you've never built large (multi-gigabyte) Docker images before, be sure to read the link:large-container-images-primer.adoc[Large container images primer]. +- If you've never used Windows containers before, be sure to read the link:windows-container-primer.adoc[Windows containers primer]. +- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the link:nvidia-docker-primer.adoc[NVIDIA Container Toolkit primer]. + +Once you're familiar with all the relevant background material, you can dive right in: + +1. First up, head to the link:../configuration.adoc[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. +2. Next, check out the link:../use-cases.adoc[Use Cases] section for details on the various scenarios in which the Docker images can be used. +Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. +3. If you run into any issues or want to customise your build with advanced options, the link:../building-images.adoc[Building Images] section provides all the relevant details. +4. For more information, check out the link:frequently-asked-questions.adoc[FAQ] and the link:../commands.adoc[Command Reference] section. + +== Links + +- https://github.com/adamrehn/ue4-docker[ue4-docker GitHub repository] +- https://pypi.org/project/ue4-docker/[ue4-docker package on PyPI] +- https://adamrehn.com/articles/tag/Unreal%20Engine/[Related articles on adamrehn.com] +- https://unrealcontainers.com/[Unreal Containers community hub] diff --git a/docs/read-these-first/introduction-to-ue4-docker.md b/docs/read-these-first/introduction-to-ue4-docker.md deleted file mode 100644 index 52aa6cb6..00000000 --- a/docs/read-these-first/introduction-to-ue4-docker.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -title: Introduction to ue4-docker -pagenum: 1 ---- - -## Overview - -The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli]({{ site.data.common.projects.ue4cli.repo }}), [conan-ue4cli]({{ site.data.common.projects.conan-ue4cli.repo }}), and [ue4-ci-helpers]({{ site.data.common.projects.ue4-ci-helpers.repo }}) to facilitate a wide variety of use cases. - -Key features include: - -- Unreal Engine 4.20.0 and newer is supported. -- Both Windows containers and Linux containers are supported. -- Building and packaging UE4 projects is supported. -- Running automation tests is supported. -- Running built UE4 projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. - - -## Important legal notice - -**With the exception of the [ue4-build-prerequisites](../building-images/available-container-images#ue4-build-prerequisites) image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. As per Section 1A of the [Unreal Engine EULA](https://www.unrealengine.com/eula), Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games UE4 GitHub repository.** {::nomarkdown}Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.{:/} **It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** - -For more details, see the [Unreal Engine EULA Restrictions](https://unrealcontainers.com/docs/obtaining-images/eula-restrictions) page on the [Unreal Containers community hub](https://unrealcontainers.com/). - - -## Getting started - -Multi-purpose Docker images for large, cross-platform projects such as the Unreal Engine involve a great deal more complexity than most typical Docker images. Before you start using ue4-docker it may be helpful to familiarise yourself with some of these complexities: - -- If you've never built large (multi-gigabyte) Docker images before, be sure to read the [Large container images primer](./large-container-images-primer). -- If you've never used Windows containers before, be sure to read the [Windows containers primer](./windows-container-primer). -- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the [NVIDIA Container Toolkit primer](./nvidia-docker-primer). - -Once you're familiar with all of the relevant background material, you can dive right in: - -1. First up, head to the [Configuration](../configuration) section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. -2. Next, check out the [Use Cases](../use-cases) section for details on the various scenarios in which the Docker images can be used. Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. -3. If you run into any issues or want to customise your build with advanced options, the [Building Images](../building-images) section provides all of the relevant details. -4. For more information, check out the [FAQ](./frequently-asked-questions) and the [Command Reference](../commands) section. - - -## Links - -- [ue4-docker GitHub repository]({{ site.data.common.projects.ue4-docker.repo }}) -- [ue4-docker package on PyPI](https://pypi.org/project/ue4-docker/) -- [Related articles on adamrehn.com](https://adamrehn.com/articles/tag/Unreal%20Engine/) -- [Unreal Containers community hub](https://unrealcontainers.com/) diff --git a/docs/read-these-first/large-container-images-primer.adoc b/docs/read-these-first/large-container-images-primer.adoc new file mode 100644 index 00000000..0248691c --- /dev/null +++ b/docs/read-these-first/large-container-images-primer.adoc @@ -0,0 +1,32 @@ += Large container images primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Although large container images are in no way different to smaller container images at a technical level, there are several aspects of the Docker build process that impact large images to a far greater extent than smaller images. +This page provides an overview for users who have never built large (multi-gigabyte) container images before and may therefore be unfamiliar with these impacts. +This information applies equally to both Linux containers and Windows containers. + +== Filesystem layer commit performance + +The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. +However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. +For some larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. +(The Installed Build layer in the multi-stage build of the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) + +Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. +After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. +There is no need for concern, as none of the steps in the ue4-docker Dockerfiles can run indefinitely without failing and emitting an error. +When a build step ceases to produce output, it is merely a matter of waiting for the filesystem layer commit to complete. + +== Disk space consumption during the build process + +Due to overheads associated with temporary layers in multi-stage builds and layer difference computation, the Docker build process for an image will consume more disk space than is required to hold the final built image. +These overheads are relatively modest when building smaller container images. +However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. + +Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least {{ site.data.ue4-docker.common.diskspace_linux | escape }} of available disk space under Linux and at least {{ site.data.ue4-docker.common.diskspace_windows | escape }} of available disk space under Windows. +Once a build is complete, the link:../commands/clean[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all of the disk space not occupied by the final built images. +The https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/read-these-first/large-container-images-primer.md b/docs/read-these-first/large-container-images-primer.md deleted file mode 100644 index e6c49211..00000000 --- a/docs/read-these-first/large-container-images-primer.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -title: Large container images primer -pagenum: 2 ---- - -Although large container images are in no way different to smaller container images at a technical level, there are several aspects of the Docker build process that impact large images to a far greater extent than smaller images. This page provides an overview for users who have never built large (multi-gigabyte) container images before and may therefore be unfamiliar with these impacts. This information applies equally to both Linux containers and Windows containers. - - -## Filesystem layer commit performance - -The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. For some of the larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. (The Installed Build layer in the the multi-stage build of the [ue4-minimal](../building-images/available-container-images#ue4-minimal) image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) - -Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. There is no need for concern, as none of the steps in the ue4-docker Dockerfiles can run indefinitely without failing and emitting an error. When a build step ceases to produce output, it is merely a matter of waiting for the filesystem layer commit to complete. - - -## Disk space consumption during the build process - -Due to overheads associated with temporary layers in multi-stage builds and layer difference computation, the Docker build process for an image will consume more disk space than is required to hold the final built image. These overheads are relatively modest when building smaller container images. However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. - -Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least {{ site.data.ue4-docker.common.diskspace_linux | escape }} of available disk space under Linux and at least {{ site.data.ue4-docker.common.diskspace_windows | escape }} of available disk space under Windows. Once a build is complete, the [ue4-docker clean](../commands/clean) command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all of the disk space not occupied by the final built images. The [docker system prune](https://docs.docker.com/engine/reference/commandline/system_prune/) command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/read-these-first/nvidia-docker-primer.adoc b/docs/read-these-first/nvidia-docker-primer.adoc new file mode 100644 index 00000000..8d023cc1 --- /dev/null +++ b/docs/read-these-first/nvidia-docker-primer.adoc @@ -0,0 +1,9 @@ += NVIDIA Container Toolkit primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +IMPORTANT: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/concepts/nvidia-docker[Key Concepts: NVIDIA Container Toolkit]. diff --git a/docs/read-these-first/nvidia-docker-primer.md b/docs/read-these-first/nvidia-docker-primer.md deleted file mode 100644 index 80f00a08..00000000 --- a/docs/read-these-first/nvidia-docker-primer.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: NVIDIA Container Toolkit primer -pagenum: 4 ---- - -{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Key Concepts: NVIDIA Container Toolkit](https://unrealcontainers.com/docs/concepts/nvidia-docker)." %} diff --git a/docs/read-these-first/windows-container-primer.adoc b/docs/read-these-first/windows-container-primer.adoc new file mode 100644 index 00000000..4def217a --- /dev/null +++ b/docs/read-these-first/windows-container-primer.adoc @@ -0,0 +1,20 @@ += Windows containers primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +NOTE: The implementation-agnostic information from this page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/concepts/windows-containers[Key Concepts: Windows Containers]. + +*Details specific to ue4-docker:* + +Due to the performance and stability issues currently associated with containers running in Hyper-V isolation mode, it is strongly recommended that process isolation mode be used for building and running Windows containers. +This necessitates the use of Windows Server as the host system (https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[or Windows 10 version 1809 or newer for development and testing purposes]) and requires that all container images use the same Windows version as the host system. +A number of ue4-docker commands provide specific functionality to facilitate this: + +* The link:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. +Hyper-V isolation mode will still be used if the user explicitly link:../building-images/advanced-build-options.adoc# specifying-the-windows-server-core-base-image-tag[specifies a different kernel version] than that of the host system or link:../building-images/advanced-build-options.adoc#specifying-the-isolation-mode-under-windows[explicitly requests Hyper-V isolation mode]. + +* The link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. diff --git a/docs/read-these-first/windows-container-primer.md b/docs/read-these-first/windows-container-primer.md deleted file mode 100644 index 58a0977e..00000000 --- a/docs/read-these-first/windows-container-primer.md +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: Windows containers primer -pagenum: 3 ---- - -{% include alerts/info.html content="The implementation-agnostic information from this page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Key Concepts: Windows Containers](https://unrealcontainers.com/docs/concepts/windows-containers)." %} - -**Details specific to ue4-docker:** - -Due to the performance and stability issues currently associated with containers running in Hyper-V isolation mode, it is strongly recommended that process isolation mode be used for building and running Windows containers. This necessitates the use of Windows Server as the host system ([or Windows 10 version 1809 or newer for development and testing purposes](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional)) and requires that all container images use the same Windows version as the host system. A number of ue4-docker commands provide specific functionality to facilitate this: - -- The [ue4-docker build](../commands/build) command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. Hyper-V isolation mode will still be used if the user explicitly [specifies a different kernel version](../building-images/advanced-build-options#specifying-the-windows-server-core-base-image-tag) than that of the host system or [explicitly requests Hyper-V isolation mode](../building-images/advanced-build-options#specifying-the-isolation-mode-under-windows). - -- The [ue4-docker setup](../commands/setup) command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. diff --git a/docs/use-cases/continuous-integration.adoc b/docs/use-cases/continuous-integration.adoc new file mode 100644 index 00000000..5e9127fa --- /dev/null +++ b/docs/use-cases/continuous-integration.adoc @@ -0,0 +1,11 @@ += Continuous Integration (CI) +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Use a controlled and reproducible environment to build, test, and package Unreal projects. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/continuous-integration[Use Cases: Continuous Integration and Deployment (CI/CD)]. diff --git a/docs/use-cases/continuous-integration.md b/docs/use-cases/continuous-integration.md deleted file mode 100644 index c1fbe767..00000000 --- a/docs/use-cases/continuous-integration.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: Continuous Integration (CI) -blurb: > - Use a controlled and reproducible environment to build, - test, and package Unreal projects. -pagenum: 2 ---- - -{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Continuous Integration and Deployment (CI/CD)](https://unrealcontainers.com/docs/use-cases/continuous-integration)." %} diff --git a/docs/use-cases/linux-installed-builds.adoc b/docs/use-cases/linux-installed-builds.adoc new file mode 100644 index 00000000..89cbd92e --- /dev/null +++ b/docs/use-cases/linux-installed-builds.adoc @@ -0,0 +1,12 @@ += Linux Installed Builds +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Under Windows and macOS, Engine licensees can easily download and manage Installed Builds of UE4 through the Epic Games Launcher. +Since version 4.21.0 of the Engine, ue4-docker provides an alternative source of Installed Builds under Linux in lieu of a native version of the launcher. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/linux-installed-builds[Use Cases: Linux Installed Builds]. diff --git a/docs/use-cases/linux-installed-builds.md b/docs/use-cases/linux-installed-builds.md deleted file mode 100644 index 7ace6a59..00000000 --- a/docs/use-cases/linux-installed-builds.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Linux Installed Builds -blurb: > - Under Windows and macOS, Engine licensees can easily download and manage - Installed Builds of UE4 through the Epic Games Launcher. Since version 4.21.0 - of the Engine, ue4-docker provides an alternative source of Installed Builds - under Linux in lieu of a native version of the launcher. -pagenum: 4 ---- - -{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Linux Installed Builds](https://unrealcontainers.com/docs/use-cases/linux-installed-builds)." %} diff --git a/docs/use-cases/microservices.adoc b/docs/use-cases/microservices.adoc new file mode 100644 index 00000000..5671a856 --- /dev/null +++ b/docs/use-cases/microservices.adoc @@ -0,0 +1,11 @@ += Microservices +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full image makes it easy to build UE4-powered microservices with Google's popular gRPC framework. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/microservices[Use Cases: Microservices]. diff --git a/docs/use-cases/microservices.md b/docs/use-cases/microservices.md deleted file mode 100644 index 98ab8eec..00000000 --- a/docs/use-cases/microservices.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: Microservices -blurb: > - Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full - image makes it easy to build UE4-powered microservices with Google's - popular gRPC framework. -pagenum: 3 ---- - -{% include alerts/info.html content="This page has migrated to the [Unreal Containers community hub](https://unrealcontainers.com/). You can find the new version here: [Use Cases: Microservices](https://unrealcontainers.com/docs/use-cases/microservices)." %} diff --git a/docs/use-cases/use-cases-overview.adoc b/docs/use-cases/use-cases-overview.adoc new file mode 100644 index 00000000..7f36832a --- /dev/null +++ b/docs/use-cases/use-cases-overview.adoc @@ -0,0 +1,14 @@ += Use cases overview +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +The container images produced by ue4-docker incorporate infrastructure to facilitate a wide variety of use cases. +A number of key use cases are listed below. +Select a use case to see detailed instructions on how to build and run the appropriate container images. + +* link:continuous-integration.adoc[Continuous Integration] +* link:linux-installed-builds.adoc[Linux installed builds] +* link:microservices.adoc[Microservices] diff --git a/docs/use-cases/use-cases-overview.md b/docs/use-cases/use-cases-overview.md deleted file mode 100644 index d0a950a3..00000000 --- a/docs/use-cases/use-cases-overview.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Use cases overview -pagenum: 1 ---- - -The container images produced by ue4-docker incorporate infrastructure to facilitate a wide variety of use cases. A number of key use cases are listed below. Select a use case to see detailed instructions on how to build and run the appropriate container images. - -{::nomarkdown} -{% assign cases = site.documents | where: "subsite", page.subsite | where: "chapter", page.chapter | where_exp: "page", "page.pagenum != null" | where_exp: "page", "page.pagenum > 1" | sort: "pagenum" %} - -{:/} diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index f0271307..93ad6e52 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -185,7 +185,7 @@ def addArguments(parser): parser.add_argument( "-password", default=None, - help="Specify the password or access token to use when cloning the git repository", + help="Specify access token or password to use when cloning the git repository", ) parser.add_argument( "-repo", From 65feb079b7dd43f1ebf5f4eb2fcb3984442011e1 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 23:29:08 +0300 Subject: [PATCH 298/430] Fix README path in setup.py --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index da5fab4c..ef9365ff 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from os.path import abspath, dirname, join from setuptools import setup -# Read the README markdown data from README.md -with open(abspath(join(dirname(__file__), "README.md")), "rb") as readmeFile: +# Read the README markdown data from README.adoc +with open(abspath(join(dirname(__file__), "README.adoc")), "rb") as readmeFile: __readme__ = readmeFile.read().decode("utf-8") # Read the version number from version.py From 462502d19b6509ee30d51c8acd6b91689e12083b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 23:37:26 +0300 Subject: [PATCH 299/430] Fix markup --- docs/commands/ue4-docker-build.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/commands/ue4-docker-build.adoc b/docs/commands/ue4-docker-build.adoc index aee923c7..8ef11723 100644 --- a/docs/commands/ue4-docker-build.adoc +++ b/docs/commands/ue4-docker-build.adoc @@ -34,7 +34,7 @@ See the link:../building-images/available-container-images.adoc[List of availabl *-h, --help*:: Print help and exit -*--rebuild**:: +*--rebuild*:: Rebuild images even if they already exist *--dry-run*:: From fe298d825ee0d8aa9e2a8f16470e3749d7ab08be Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 11 Dec 2022 23:41:44 +0300 Subject: [PATCH 300/430] Improve ue4-docker-build formatting --- docs/commands/ue4-docker-build.adoc | 30 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/commands/ue4-docker-build.adoc b/docs/commands/ue4-docker-build.adoc index 8ef11723..7a92027d 100644 --- a/docs/commands/ue4-docker-build.adoc +++ b/docs/commands/ue4-docker-build.adoc @@ -52,39 +52,39 @@ Disable Docker build cache *--exclude {ddc,debug,templates}*:: Exclude the specified component (can be specified multiple times to exclude multiple components) -*--opt OPT*:: +*--opt* _OPT_:: Set an advanced configuration option (can be specified multiple times to specify multiple options) -*-username USERNAME*:: +*-username* _USERNAME_:: Specify the username to use when cloning the git repository -*-password PASSWORD*:: +*-password* _PASSWORD_:: Specify access token or password to use when cloning the git repository -*-repo REPO*:: +*-repo* _REPO_:: Set the custom git repository to clone when "custom" is specified as the release value -*-branch BRANCH*:: +*-branch* _BRANCH_:: Set the custom branch/tag to clone when "custom" is specified as the release value -*-basetag BASETAG*:: +*-basetag* _BASETAG_:: Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu18.04). For Windows this is the Windows Server Core base image tag (default is the host OS version) -*-suffix SUFFIX*:: +*-suffix* _SUFFIX_:: Add a suffix to the tags of the built images -*-m M*:: +*-m* _M_:: Override the default memory limit under Windows (also overrides --random-memory) -*-ue4cli UE4CLI*:: +*-ue4cli* _UE4CLI_:: Override the default version of ue4cli installed in the ue4-full image -*-conan-ue4cli CONAN_UE4CLI*:: +*-conan-ue4cli* _CONAN_UE4CLI_:: Override the default version of conan-ue4cli installed in the ue4-full image -*-layout LAYOUT*:: +*-layout* _LAYOUT_:: Copy generated Dockerfiles to the specified directory and don't build the images *--combine*:: @@ -93,7 +93,7 @@ Combine generated Dockerfiles into a single multi-stage build Dockerfile *--monitor*:: Monitor resource usage during builds (useful for debugging) -*-interval INTERVAL*:: +*-interval* _INTERVAL_:: Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) *-v*:: @@ -101,7 +101,7 @@ Enable verbose output during builds (useful for debugging) == Linux-specific options -*--cuda VERSION*:: +*--cuda* _VERSION_:: Add CUDA support as well as OpenGL support == Windows-specific options @@ -109,7 +109,7 @@ Add CUDA support as well as OpenGL support *--ignore-blacklist*:: Run builds even on blacklisted versions of Windows (advanced use only) -*-isolation ISOLATION*:: +*-isolation* _ISOLATION_:: Set the isolation mode to use (process or hyperv) *--linux*:: @@ -118,5 +118,5 @@ Use Linux containers under Windows hosts (useful when testing Docker Desktop or *--random-memory*:: Use a random memory limit for Windows containers -*--visual-studio VISUAL_STUDIO*:: +*--visual-studio* _VISUAL_STUDIO_:: Specify Visual Studio Build Tools version From 9a5685655cdc0b13a5499a1da0fb52304fa096df Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 00:03:26 +0300 Subject: [PATCH 301/430] Fix markup --- docs/building-images/available-container-images.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/building-images/available-container-images.adoc b/docs/building-images/available-container-images.adoc index 55fa3ca3..0c90a6d0 100644 --- a/docs/building-images/available-container-images.adoc +++ b/docs/building-images/available-container-images.adoc @@ -19,7 +19,7 @@ You can prevent unwanted images from being built by appending the relevant image **Tags:** -* "**Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Windows Server Core base image tag] +* **Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Windows Server Core base image tag] * **Linux containers**: `adamrehn/ue4-build-prerequisites:CONFIGURATION` where `CONFIGURATION` is as follows: From 7fb98d9c0a7fc34455399a4f63fc3a72289eb2e0 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 12 Dec 2022 11:14:33 +1000 Subject: [PATCH 302/430] Explicitly specify CPU architecture when building Linux images --- ue4docker/infrastructure/ImageBuilder.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index 5ddd122d..a6da227a 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -117,6 +117,9 @@ def build( ], ) + # When building Linux images, explicitly specify the target CPU architecture + archFlags = ["--platform", "linux/amd64"] if self.platform == "linux" else [] + # Create a temporary directory to hold any files needed for the build with tempfile.TemporaryDirectory() as tempDir: @@ -133,9 +136,11 @@ def build( secretFlags.append("id={},src={}".format(secret, secretFile)) # Generate the `docker buildx` command to use our build secrets - command = DockerUtils.buildx(imageTags, context_dir, args, secretFlags) + command = DockerUtils.buildx( + imageTags, context_dir, archFlags + args, secretFlags + ) else: - command = DockerUtils.build(imageTags, context_dir, args) + command = DockerUtils.build(imageTags, context_dir, archFlags + args) command += ["--file", dockerfile] From 9d2dae37143bfca522d810a9e02cf07457f1af6d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 10:26:41 +0300 Subject: [PATCH 303/430] Tweak xrefs --- .../advanced-build-options.adoc | 42 +++++++++++-------- .../available-container-images.adoc | 16 ++++--- .../troubleshooting-build-issues.adoc | 30 ++++++------- docs/commands/ue4-docker-build.adoc | 2 +- docs/commands/ue4-docker-diagnostics.adoc | 5 +-- docs/commands/ue4-docker-export.adoc | 4 +- docs/commands/ue4-docker-setup.adoc | 4 +- docs/commands/ue4-docker-test.adoc | 2 +- docs/configuration/configuring-linux.adoc | 4 +- docs/configuration/configuring-macos.adoc | 2 +- .../configuration/configuring-windows-10.adoc | 4 +- .../configuring-windows-server.adoc | 6 +-- .../supported-host-configurations.adoc | 8 ++-- .../introduction-to-ue4-docker.adoc | 16 +++---- .../large-container-images-primer.adoc | 6 +-- .../windows-container-primer.adoc | 6 +-- docs/use-cases/use-cases-overview.adoc | 6 +-- 17 files changed, 87 insertions(+), 76 deletions(-) diff --git a/docs/building-images/advanced-build-options.adoc b/docs/building-images/advanced-build-options.adoc index 7096cf09..2ee56c92 100644 --- a/docs/building-images/advanced-build-options.adoc +++ b/docs/building-images/advanced-build-options.adoc @@ -9,7 +9,7 @@ === Performing a dry run -If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the link:../commands/ue4-docker-build.adoc[ue4-docker build] command. +If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command. Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. === Specifying a custom namespace for the image tags @@ -18,7 +18,7 @@ If you would like to override the default `adamrehn/` prefix that is used when g === Specifying Git credentials -The link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: +The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: - **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. @@ -55,9 +55,10 @@ ue4-docker build custom:my-custom-build -repo=https://github.com/MyUser/UnrealEn This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn/ue4-minimal:my-custom-build`, etc. +[[exclude-components]] === Excluding Engine components to reduce the final image size -Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the link:../commands/ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] and link:../building-images/available-container-images.adoc#ue4-full[ue4-full] images. +Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] images. The following components can be excluded: - `ddc`: disables building the DDC for the Engine. @@ -101,9 +102,10 @@ ue4-docker build 4.24.2 --monitor ue4-docker build 4.24.2 --monitor -interval=5 ---- +[[exporting-generated-dockerfiles]] === Exporting generated Dockerfiles -Since ue4-docker version 0.0.78, the link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. +Since ue4-docker version 0.0.78, the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a https://docs.docker.com/develop/develop-images/multistage-build/[multi-stage build]. You can use these flags like so: @@ -134,14 +136,15 @@ This generates Dockerfiles that copy the Unreal Engine source code from the host - When building Linux container images, you must either set the <> `source_mode` to `copy` as detailed above, or else specify the `credential_mode` option and set it to `secrets`. This generates Dockerfiles that use the Linux-only https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] functionality to securely provide git credentials, eliminating the dependency on ue4-docker's credential endpoint whilst still facilitating the use of a git repository to provide the Unreal Engine source code. +[[advanced-options-for-dockerfile-generation]] === Advanced options for Dockerfile generation NOTE: Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers[Jinja identifiers]. -Since ue4-docker version 0.0.78, the link:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. +Since ue4-docker version 0.0.78, the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. Some of these options (such as `source_mode`) can only be used when <>, whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: -- **`source_mode`**: *(string)* controls how the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. +- **`source_mode`**: *(string)* controls how the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. Valid options are: - `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. @@ -150,18 +153,18 @@ This is the only mode that can be used when not <>, since the credential endpoint will not be available during the build process. -- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] container during the build process. +- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] container during the build process. -- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image. +- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image. - **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. -This includes the labels which specify the <> as well as the sentinel labels that the link:../commands/ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. +This includes the labels which specify the <> as well as the sentinel labels that the xref:../commands/ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. - **`disable_all_patches`**: *(boolean)* disables all the patches that ue4-docker ordinarily applies to the Unreal Engine source code. This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. @@ -192,10 +195,11 @@ Some older versions of the Unreal Engine did not copy this correctly, breaking t == Windows-specific options +[[windows-base-tag]] === Specifying the Windows Server Core base image tag -NOTE: The `-basetag` flag controls how the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. -If you are building multiple related images over separate invocations of the build command (e.g. building the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +NOTE: The `-basetag` flag controls how the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. @@ -210,6 +214,7 @@ ue4-docker build 4.20.3 -basetag=ltsc2016 # Uses Windows Server 2016 (Long Term For a list of supported base image tags, see the https://hub.docker.com/r/microsoft/windowsservercore/[Windows Server Core base image on Docker Hub]. +[[windows-isolation-mode]] === Specifying the isolation mode under Windows The isolation mode can be explicitly specified via the `-isolation=MODE` flag when invoking the build command. @@ -224,14 +229,14 @@ To do so, pass `--visual-studio=2019` flag when invoking the build command. === Keeping or excluding Installed Build debug symbols under Windows WARNING: Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. -See link:troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy--frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[this section of the Troubleshooting Build Issues page] for further details on this issue. +See xref:troubleshooting-build-issues.adoc#copy-8gb-20gb[this section of the Troubleshooting Build Issues page] for further details on this issue. -Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. +Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. (This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) -Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section <>. +Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section <>. === Building Linux container images under Windows @@ -240,10 +245,11 @@ To build Linux container images instead, simply specify the `--linux` flag when == Linux-specific options +[[cuda]] === Enabling CUDA support for GPU-enabled Linux images -IMPORTANT: The `--cuda` flag controls how the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. -If you are building multiple related images over separate invocations of the build command (e.g. building the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +IMPORTANT: The `--cuda` flag controls how the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. diff --git a/docs/building-images/available-container-images.adoc b/docs/building-images/available-container-images.adoc index 0c90a6d0..d12cb60d 100644 --- a/docs/building-images/available-container-images.adoc +++ b/docs/building-images/available-container-images.adoc @@ -5,7 +5,7 @@ :source-highlighter: rouge :toc: -You can build the following images using the link:../commands/ue4-docker-build.adoc[ue4-docker build] command: +You can build the following images using the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command: - <> - <> @@ -15,17 +15,18 @@ You can build the following images using the link:../commands/ue4-docker-build.a By default, all available images will be built. You can prevent unwanted images from being built by appending the relevant image-specific flag to the build command (see the *"Flag to disable"* entry for each image below.) +[[ue4-build-prerequisites]] == ue4-build-prerequisites **Tags:** -* **Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Windows Server Core base image tag] +* **Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the xref:advanced-build-options.adoc#windows-base-tag[Windows Server Core base image tag] * **Linux containers**: `adamrehn/ue4-build-prerequisites:CONFIGURATION` where `CONFIGURATION` is as follows: ** `opengl` if CUDA support is not enabled -** `cudaglVERSION` where `VERSION` is the CUDA version if link:advanced-build-options.adoc#enabling-cuda-support-for-gpu-enabled-linux-images[CUDA support is enabled] (e.g. `cudagl9.2`, `cudagl10.0`, etc.) +** `cudaglVERSION` where `VERSION` is the CUDA version if xref:advanced-build-options.adoc#cuda[CUDA support is enabled] (e.g. `cudagl9.2`, `cudagl10.0`, etc.) **Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] @@ -35,6 +36,7 @@ You can prevent unwanted images from being built by appending the relevant image * Keep this image on disk to speed up subsequent container image builds. +[[ue4-source]] == ue4-source **Tags:** @@ -53,6 +55,7 @@ The ue4-minimal image uses this source code as the starting point for its build. * Only needed during the build process. Afterwards, this image can be removed using `ue4-docker clean --source` to save disk space. +[[ue4-minimal]] == ue4-minimal **Tags:** @@ -67,8 +70,9 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save **Uses:** -* Use this image for link:../use-cases/continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. +* Use this image for xref:../use-cases/continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. +[[ue4-full]] == ue4-full **Tags:** @@ -90,6 +94,6 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save **Uses:** -* link:../use-cases/continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers +* xref:../use-cases/continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers * Packaging or running cloud rendering projects that optionally utilise UE4Capture under Linux (**note that the (https://hub.docker.com/r/adamrehn/ue4-runtime[ue4-runtime]) image is the recommended base image for running cloud rendering workloads, since this avoids including the multi-gigabyte UE4 developer tools**) -* Packaging link:../use-cases/microservices.adoc[UE4-powered microservices] +* Packaging xref:../use-cases/microservices.adoc[UE4-powered microservices] diff --git a/docs/building-images/troubleshooting-build-issues.adoc b/docs/building-images/troubleshooting-build-issues.adoc index caf6c301..10b69c1a 100644 --- a/docs/building-images/troubleshooting-build-issues.adoc +++ b/docs/building-images/troubleshooting-build-issues.adoc @@ -10,7 +10,7 @@ === Building the `ue4-build-prerequisites` image fails with a network-related error This indicates an underlying network or proxy server issue outside ue4-docker itself that you will need to troubleshoot. -You can use the link:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. +You can use the xref:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. Here are some steps to try: - If your host system accesses the network through a proxy server, make sure that https://docs.docker.com/network/proxy/[Docker is configured to use the correct proxy server settings]. @@ -28,7 +28,7 @@ Here are some steps to try: This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. -(Under Windows Server, the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) +(Under Windows Server, the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) === Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file @@ -42,14 +42,15 @@ Note that restarting the Docker daemon and/or rebooting the host system may also Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. - For Windows containers, follow https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size[the instructions provided by Microsoft], making sure you restart the Docker daemon after you've modified the config JSON. -(Under Windows Server, the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) +(Under Windows Server, the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) - For Linux containers, use the https://docs.docker.com/docker-for-windows/#advanced[Docker for Windows "Advanced" settings tab] under Windows or the https://docs.docker.com/docker-for-mac/#disk[Docker for Mac "Disk" settings tab] under macOS. +[[copy-8gb-20gb]] === Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image WARNING: Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. -Since ue4-docker version 0.0.47, you can use the link:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. -If it does, you may need to link:advanced-build-options.adoc#excluding-engine-components-to-reduce-the-final-image-size[exclude debug symbols] when building Windows images. +Since ue4-docker version 0.0.47, you can use the xref:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. +If it does, you may need to xref:advanced-build-options.adoc#exclude-components[exclude debug symbols] when building Windows images. Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: @@ -58,7 +59,7 @@ Some versions of Docker contain one or more of a series of separate but related https://github.com/moby/moby/issues/37581[#37581] was https://github.com/moby/moby/pull/37771[fixed] in Docker CE 18.09.0, whilst https://github.com/moby/moby/issues/40444[#40444] was https://github.com/moby/moby/pull/41430[fixed] in Docker CE 20.10.0. -If you are using a version of Docker that contains one of these bugs then you will need to link:advanced-build-options.adoc#excluding-engine-components-to-reduce-the-final-image-size[exclude debug symbols], which reduces the size of the Installed Build below the 8GiB threshold. +If you are using a version of Docker that contains one of these bugs then you will need to xref:advanced-build-options.adoc#exclude-components[exclude debug symbols], which reduces the size of the Installed Build below the 8GiB threshold. If debug symbols are required then it will be necessary to upgrade or downgrade to a version of Docker that does not suffer from the 8GiB size limit issue (although finding such a version under Windows may prove quite difficult.) == Linux-specific issues @@ -70,7 +71,7 @@ This issue can occur both inside containers and when running directly on a host . Untag the available-container-images.adoc#ue4-source[ue4-source] image. . Clear the Docker filesystem layer cache by running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune]. -. Re-run the link:../commands/ue4-docker-build.adoc[ue4-docker build] command. +. Re-run the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command. == Windows-specific issues @@ -82,6 +83,7 @@ The issue is resolved by ensuring that both the host system and the container im - Make sure your host system is up-to-date and all available Windows updates are installed. - Make sure you are using the latest version of ue4-docker, which automatically uses container images that incorporate the fix. +[[hcsshim-timeout]] === Building Windows containers fails with the message `hcsshim: timeout waiting for notification extra info` or the message `This operation ended because the timeout has expired` Recent versions of Docker under Windows may sometimes encounter the error https://github.com/Microsoft/hcsshim/issues/152[hcsshim: timeout waiting for notification extra info] when building or running Windows containers. @@ -106,21 +108,21 @@ This error is shown in two situations: - The host system is running an **older kernel version** than the container image. In this case, you will need to build the images using the same kernel version as the host system or older. -See link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. -- The host system is running a **newer kernel version** than the container image and you are attempting to use process isolation mode instead of Hyper-V isolation mode. +See xref:advanced-build-options.adoc#windows-base-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. +- The host system is running a **newer kernel version** than the container image, and you are attempting to use process isolation mode instead of Hyper-V isolation mode. (Process isolation mode is the default under Windows Server.) In this case, you will need to use Hyper-V isolation mode instead. -See link:advanced-build-options.adoc#specifying-the-isolation-mode-under-windows[Specifying the isolation mode under Windows] for details on how to do this. +See xref:advanced-build-options.adoc#windows-isolation-mode[Specifying the isolation mode under Windows] for details on how to do this. === Pulling the .NET Framework base image fails with the message `ProcessUtilityVMImage \\?\`(long path here)`\UtilityVM: The system cannot find the path specified` This is a known issue when the host system is running an older kernel version than the container image. Just like in the case of *"The operating system of the container does not match the operating system of the host"* error mentioned above, you will need to build the images using the same kernel version as the host system or older. -See link:advanced-build-options.adoc#specifying-the-windows-server-core-base-image-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. +See xref:advanced-build-options.adoc#windows-base-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. === Building the Engine in a Windows container fails with the message `The process cannot access the file because it is being used by another process` This is a known bug in some older versions of UnrealBuildTool when using a memory limit that is not a multiple of 4GB. -To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the <> section. +To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the <> section. === Building the Engine in a Windows container fails with the message `fatal error LNK1318: Unexpected PDB error; OK (0)` @@ -132,13 +134,13 @@ Note that a linker wrapper https://docs.unrealengine.com/en-US/Support/Builds/Re === Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` This error typically occurs when the Windows pagefile size is not large enough. -As stated in the link:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. +As stated in the xref:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. However, containers running in process isolation mode will use the pagefile settings of the host system. When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. (Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) === Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system -As described in the link:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. +As described in the xref:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/commands/ue4-docker-build.adoc b/docs/commands/ue4-docker-build.adoc index 7a92027d..64d17432 100644 --- a/docs/commands/ue4-docker-build.adoc +++ b/docs/commands/ue4-docker-build.adoc @@ -27,7 +27,7 @@ You will be prompted for the Git credentials to be used when cloning the Unreal The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. By default, all available images will be built. -See the link:../building-images/available-container-images.adoc[List of available container images] for details on customising which images are built. +See the xref:../building-images/available-container-images.adoc[List of available container images] for details on customising which images are built. == Options diff --git a/docs/commands/ue4-docker-diagnostics.adoc b/docs/commands/ue4-docker-diagnostics.adoc index 8791fdd7..975b698a 100644 --- a/docs/commands/ue4-docker-diagnostics.adoc +++ b/docs/commands/ue4-docker-diagnostics.adoc @@ -8,7 +8,6 @@ == Name ue4-docker diagnostics - run diagnostics to detect issues with the host system configuration. -{{ page.blurb | escape }} == Synopsis @@ -21,7 +20,7 @@ This command can be used to run the following diagnostics: === Checking for the Docker 8GiB filesystem layer bug Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. -This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, link:../building-images/troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. +This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, xref:../building-images/troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: @@ -33,7 +32,7 @@ ue4-docker diagnostics 8gig === Checking for the Windows Host Compute Service (HCS) `storage-opt` bug Windows Server versions 1903 and 1909 and Windows 10 versions 1903 and 1909 contain https://github.com/docker/for-win/issues/4100[a bug in the Host Compute Service (HCS)] that prevents users from increasing the maximum allowed image size using Docker's `storage-opt` configuration key. -Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the link:ue4-docker-build.adoc[ue4-docker build] command from functioning correctly on affected systems. +Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the xref:ue4-docker-build.adoc[ue4-docker build] command from functioning correctly on affected systems. This diagnostic tests whether the host system is affected this bug, by attempting to run a container with a non-default `storage-opt` value: diff --git a/docs/commands/ue4-docker-export.adoc b/docs/commands/ue4-docker-export.adoc index 42932538..df469745 100644 --- a/docs/commands/ue4-docker-export.adoc +++ b/docs/commands/ue4-docker-export.adoc @@ -19,7 +19,7 @@ This command can be used to export the following components: === Exporting Installed Builds under Linux -Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or link:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: +Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: [source,shell] ---- @@ -44,7 +44,7 @@ ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/Unrea === Exporting Conan packages -The Conan wrapper packages generated by `conan-ue4cli` can be exported from the link:../building-images/available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: +The Conan wrapper packages generated by `conan-ue4cli` can be exported from the xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: [source,shell] ---- diff --git a/docs/commands/ue4-docker-setup.adoc b/docs/commands/ue4-docker-setup.adoc index ef867dee..2dd894fb 100644 --- a/docs/commands/ue4-docker-setup.adoc +++ b/docs/commands/ue4-docker-setup.adoc @@ -19,12 +19,12 @@ This command will automatically configure a Linux or Windows Server host system **Under Linux:** -- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. +- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. **Under Windows Server:** - The Docker daemon will be configured to set the maximum image size for Windows containers to 400GB. -- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. +- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. - Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the https://hub.docker.com/_/microsoft-windows[full Windows base image]. Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. diff --git a/docs/commands/ue4-docker-test.adoc b/docs/commands/ue4-docker-test.adoc index 00450bc5..8457e386 100644 --- a/docs/commands/ue4-docker-test.adoc +++ b/docs/commands/ue4-docker-test.adoc @@ -15,6 +15,6 @@ ue4-docker-test - run tests to verify the correctness of built container images == Description -This command runs a suite of tests to verify that built link:../building-images/available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. +This command runs a suite of tests to verify that built xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/configuration/configuring-linux.adoc b/docs/configuration/configuring-linux.adoc index 1b9c16b3..49f96fa1 100644 --- a/docs/configuration/configuring-linux.adoc +++ b/docs/configuration/configuring-linux.adoc @@ -60,8 +60,8 @@ sudo pip3 install ue4-docker == Step 4: Use ue4-docker to automatically configure the Linux firewall -If the host system is running an active firewall that blocks access to port 9876 (required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. -The link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. +If the host system is running an active firewall that blocks access to port 9876 (required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. +The xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. Simply run: [source,shell] diff --git a/docs/configuration/configuring-macos.adoc b/docs/configuration/configuring-macos.adoc index f513e595..bc3b66f1 100644 --- a/docs/configuration/configuring-macos.adoc +++ b/docs/configuration/configuring-macos.adoc @@ -6,7 +6,7 @@ :toc: WARNING: macOS provides a suboptimal experience when running Linux containers, due to the following factors: -Linux containers are unable to use GPU acceleration via the link:../read-these-first/nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. +Linux containers are unable to use GPU acceleration via the xref:../read-these-first/nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. == Requirements diff --git a/docs/configuration/configuring-windows-10.adoc b/docs/configuration/configuring-windows-10.adoc index f684bb40..23bf1b01 100644 --- a/docs/configuration/configuring-windows-10.adoc +++ b/docs/configuration/configuring-windows-10.adoc @@ -8,11 +8,11 @@ == Warning Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. -Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to link:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. The default isolation mode depends on the specific version of Windows being used: - Under Windows 10, Hyper-V isolation mode is the default isolation mode and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-[process isolation mode must be manually enabled] each time a container is built or run. -The link:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** +The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** - Under Windows 11, process isolation mode is the default isolation mode. diff --git a/docs/configuration/configuring-windows-server.adoc b/docs/configuration/configuring-windows-server.adoc index 5f7d0a6e..117a85c9 100644 --- a/docs/configuration/configuring-windows-server.adoc +++ b/docs/configuration/configuring-windows-server.adoc @@ -6,7 +6,7 @@ :toc: IMPORTANT: Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. -Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to link:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. Process isolation mode is the default isolation mode under Windows Server. == Requirements @@ -63,11 +63,11 @@ pip install ue4-docker == Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall -To automatically configure the required system settings, run the link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: +To automatically configure the required system settings, run the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: [source,shell] ---- ue4-docker setup ---- -This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the link:../building-images/available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. +This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. diff --git a/docs/configuration/supported-host-configurations.adoc b/docs/configuration/supported-host-configurations.adoc index 178474b3..619d7dee 100644 --- a/docs/configuration/supported-host-configurations.adoc +++ b/docs/configuration/supported-host-configurations.adoc @@ -13,25 +13,25 @@ The table below lists the host operating systems can be used to build and run th |=== | Host OS | Linux containers | Windows containers | NVIDIA Container Toolkit | Optimality -| link:configuring-linux.adoc[Linux] +| xref:configuring-linux.adoc[Linux] | icon:check[] | icon:times[] | icon:check[] | Optimal for Linux containers -| link:configuring-windows-server.adoc[Windows Server] +| xref:configuring-windows-server.adoc[Windows Server] | icon:times[] | icon:check[] | icon:times[] | Optimal for Windows containers when using process isolation mode -| link:configuring-windows-10.adoc[Windows 10 and 11] +| xref:configuring-windows-10.adoc[Windows 10 and 11] | Works but not tested or supported | icon:check[] | icon:times[] | Optimal for Windows containers when using process isolation mode -| link:configuring-macos.adoc[macOS] +| xref:configuring-macos.adoc[macOS] | icon:check[] | icon:times[] | icon:times[] diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/read-these-first/introduction-to-ue4-docker.adoc index 1e9d50e2..521bf908 100644 --- a/docs/read-these-first/introduction-to-ue4-docker.adoc +++ b/docs/read-these-first/introduction-to-ue4-docker.adoc @@ -20,7 +20,7 @@ Key features include: == Important legal notice -**With the exception of the link:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. +**Except for the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games UE4 GitHub repository. **Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.** It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** @@ -31,17 +31,17 @@ For more details, see the https://unrealcontainers.com/docs/obtaining-images/eul Multipurpose Docker images for large, cross-platform projects such as the Unreal Engine involve a great deal more complexity than most typical Docker images. Before you start using ue4-docker it may be helpful to familiarise yourself with some of these complexities: -- If you've never built large (multi-gigabyte) Docker images before, be sure to read the link:large-container-images-primer.adoc[Large container images primer]. -- If you've never used Windows containers before, be sure to read the link:windows-container-primer.adoc[Windows containers primer]. -- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the link:nvidia-docker-primer.adoc[NVIDIA Container Toolkit primer]. +- If you've never built large (multi-gigabyte) Docker images before, be sure to read the xref:large-container-images-primer.adoc[Large container images primer]. +- If you've never used Windows containers before, be sure to read the xref:windows-container-primer.adoc[Windows containers primer]. +- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the xref:nvidia-docker-primer.adoc[NVIDIA Container Toolkit primer]. Once you're familiar with all the relevant background material, you can dive right in: -1. First up, head to the link:../configuration.adoc[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. -2. Next, check out the link:../use-cases.adoc[Use Cases] section for details on the various scenarios in which the Docker images can be used. +1. First up, head to the xref:../configuration.adoc[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. +2. Next, check out the xref:../use-cases.adoc[Use Cases] section for details on the various scenarios in which the Docker images can be used. Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. -3. If you run into any issues or want to customise your build with advanced options, the link:../building-images.adoc[Building Images] section provides all the relevant details. -4. For more information, check out the link:frequently-asked-questions.adoc[FAQ] and the link:../commands.adoc[Command Reference] section. +3. If you run into any issues or want to customise your build with advanced options, the xref:../building-images.adoc[Building Images] section provides all the relevant details. +4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the xref:../commands.adoc[Command Reference] section. == Links diff --git a/docs/read-these-first/large-container-images-primer.adoc b/docs/read-these-first/large-container-images-primer.adoc index 0248691c..85e3f758 100644 --- a/docs/read-these-first/large-container-images-primer.adoc +++ b/docs/read-these-first/large-container-images-primer.adoc @@ -14,7 +14,7 @@ This information applies equally to both Linux containers and Windows containers The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. For some larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. -(The Installed Build layer in the multi-stage build of the link:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) +(The Installed Build layer in the multi-stage build of the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. @@ -27,6 +27,6 @@ Due to overheads associated with temporary layers in multi-stage builds and laye These overheads are relatively modest when building smaller container images. However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. -Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least {{ site.data.ue4-docker.common.diskspace_linux | escape }} of available disk space under Linux and at least {{ site.data.ue4-docker.common.diskspace_windows | escape }} of available disk space under Windows. -Once a build is complete, the link:../commands/clean[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all of the disk space not occupied by the final built images. +Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least 400GB of available disk space under Linux and at least 800GB of available disk space under Windows. +Once a build is complete, the xref:../commands/clean[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. The https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/read-these-first/windows-container-primer.adoc b/docs/read-these-first/windows-container-primer.adoc index 4def217a..e8c03c47 100644 --- a/docs/read-these-first/windows-container-primer.adoc +++ b/docs/read-these-first/windows-container-primer.adoc @@ -14,7 +14,7 @@ Due to the performance and stability issues currently associated with containers This necessitates the use of Windows Server as the host system (https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[or Windows 10 version 1809 or newer for development and testing purposes]) and requires that all container images use the same Windows version as the host system. A number of ue4-docker commands provide specific functionality to facilitate this: -* The link:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. -Hyper-V isolation mode will still be used if the user explicitly link:../building-images/advanced-build-options.adoc# specifying-the-windows-server-core-base-image-tag[specifies a different kernel version] than that of the host system or link:../building-images/advanced-build-options.adoc#specifying-the-isolation-mode-under-windows[explicitly requests Hyper-V isolation mode]. +* The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. +Hyper-V isolation mode will still be used if the user explicitly xref:../building-images/advanced-build-options.adoc#windows-base-tag[specifies a different kernel version] than that of the host system or xref:../building-images/advanced-build-options.adoc#windows-isolation-mode[explicitly requests Hyper-V isolation mode]. -* The link:../commands/ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. +* The xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. diff --git a/docs/use-cases/use-cases-overview.adoc b/docs/use-cases/use-cases-overview.adoc index 7f36832a..b9dda084 100644 --- a/docs/use-cases/use-cases-overview.adoc +++ b/docs/use-cases/use-cases-overview.adoc @@ -9,6 +9,6 @@ The container images produced by ue4-docker incorporate infrastructure to facili A number of key use cases are listed below. Select a use case to see detailed instructions on how to build and run the appropriate container images. -* link:continuous-integration.adoc[Continuous Integration] -* link:linux-installed-builds.adoc[Linux installed builds] -* link:microservices.adoc[Microservices] +* xref:continuous-integration.adoc[Continuous Integration] +* xref:linux-installed-builds.adoc[Linux installed builds] +* xref:microservices.adoc[Microservices] From d472d67c2e0806e1c2774cd6b981777f8779a338 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 10:46:15 +0300 Subject: [PATCH 304/430] Add index that combines docs into a single document --- docs/building-images/index.adoc | 10 ++++++++++ docs/commands/index.adoc | 15 +++++++++++++++ docs/configuration/index.adoc | 12 ++++++++++++ docs/index.adoc | 13 +++++++++++++ docs/read-these-first/index.adoc | 11 +++++++++++ docs/use-cases/index.adoc | 11 +++++++++++ 6 files changed, 72 insertions(+) create mode 100644 docs/building-images/index.adoc create mode 100644 docs/commands/index.adoc create mode 100644 docs/configuration/index.adoc create mode 100644 docs/index.adoc create mode 100644 docs/read-these-first/index.adoc create mode 100644 docs/use-cases/index.adoc diff --git a/docs/building-images/index.adoc b/docs/building-images/index.adoc new file mode 100644 index 00000000..6c236ebe --- /dev/null +++ b/docs/building-images/index.adoc @@ -0,0 +1,10 @@ += Building images +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::available-container-images.adoc[leveloffset=+1] +include::advanced-build-options.adoc[leveloffset=+1] +include::troubleshooting-build-issues.adoc[leveloffset=+1] diff --git a/docs/commands/index.adoc b/docs/commands/index.adoc new file mode 100644 index 00000000..6ba48c57 --- /dev/null +++ b/docs/commands/index.adoc @@ -0,0 +1,15 @@ += Command reference +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::ue4-docker-build.adoc[leveloffset=+1] +include::ue4-docker-clean.adoc[leveloffset=+1] +include::ue4-docker-diagnostics.adoc[leveloffset=+1] +include::ue4-docker-export.adoc[leveloffset=+1] +include::ue4-docker-info.adoc[leveloffset=+1] +include::ue4-docker-setup.adoc[leveloffset=+1] +include::ue4-docker-test.adoc[leveloffset=+1] +include::ue4-docker-version.adoc[leveloffset=+1] diff --git a/docs/configuration/index.adoc b/docs/configuration/index.adoc new file mode 100644 index 00000000..0eafb5fc --- /dev/null +++ b/docs/configuration/index.adoc @@ -0,0 +1,12 @@ += Configuration +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::supported-host-configurations.adoc[leveloffset=+1] +include::configuring-linux.adoc[leveloffset=+1] +include::configuring-windows-server.adoc[leveloffset=+1] +include::configuring-windows-10.adoc[leveloffset=+1] +include::configuring-macos.adoc[leveloffset=+1] diff --git a/docs/index.adoc b/docs/index.adoc new file mode 100644 index 00000000..2616c252 --- /dev/null +++ b/docs/index.adoc @@ -0,0 +1,13 @@ += ue4-docker +:doctype: book +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::read-these-first/index.adoc[leveloffset=+1] +include::configuration/index.adoc[leveloffset=+1] +include::use-cases/index.adoc[leveloffset=+1] +include::building-images/index.adoc[leveloffset=+1] +include::commands/index.adoc[leveloffset=+1] diff --git a/docs/read-these-first/index.adoc b/docs/read-these-first/index.adoc new file mode 100644 index 00000000..9fefc4ce --- /dev/null +++ b/docs/read-these-first/index.adoc @@ -0,0 +1,11 @@ += Read these first +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::introduction-to-ue4-docker.adoc[leveloffset=+1] +include::large-container-images-primer.adoc[leveloffset=+1] +include::windows-container-primer.adoc[leveloffset=+1] +include::nvidia-docker-primer.adoc[leveloffset=+1] diff --git a/docs/use-cases/index.adoc b/docs/use-cases/index.adoc new file mode 100644 index 00000000..244364cb --- /dev/null +++ b/docs/use-cases/index.adoc @@ -0,0 +1,11 @@ += Use cases +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: left + +include::use-cases-overview.adoc[leveloffset=+1] +include::continuous-integration.adoc[leveloffset=+1] +include::microservices.adoc[leveloffset=+1] +include::linux-installed-builds.adoc[leveloffset=+1] From df7ff1b41017e86e91daeed8589598ed71ef0ef4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 12:01:37 +0300 Subject: [PATCH 305/430] Fix cross-document xref --- docs/read-these-first/large-container-images-primer.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/read-these-first/large-container-images-primer.adoc b/docs/read-these-first/large-container-images-primer.adoc index 85e3f758..e758c76b 100644 --- a/docs/read-these-first/large-container-images-primer.adoc +++ b/docs/read-these-first/large-container-images-primer.adoc @@ -28,5 +28,5 @@ These overheads are relatively modest when building smaller container images. However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least 400GB of available disk space under Linux and at least 800GB of available disk space under Windows. -Once a build is complete, the xref:../commands/clean[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. +Once a build is complete, the xref:../commands/ue4-docker-clean.adoc[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. The https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. From 6a90510cfb04dbf0d365e4f52d35ffe1df17310d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 12:13:03 +0300 Subject: [PATCH 306/430] Quick and dirty pipeline for docs publishing --- .github/workflows/ci.yml | 13 +++++++++++++ docs/build.sh | 15 +++++++++++++++ docs/index.adoc | 1 + .../introduction-to-ue4-docker.adoc | 2 -- 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100755 docs/build.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97c2aab2..50df2f57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,19 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + docs: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/master' + steps: + - name: Checkout + uses: actions/checkout@v3 + - run: docs/build.sh + - name: Publish + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + force_orphan: true + publish_dir: build/gh-pages/ test: strategy: matrix: diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..2b604d9e --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# See http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +project_dir="$(git rev-parse --show-toplevel)" + +function docker_asciidoctor() { + docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" +} + +docker_asciidoctor asciidoctor /project/docs/index.adoc -o /project/build/gh-pages/index.html +docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.pdf +docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.epub diff --git a/docs/index.adoc b/docs/index.adoc index 2616c252..a11dbfc5 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -3,6 +3,7 @@ :icons: font :idprefix: :idseparator: - +:linkcss: :source-highlighter: rouge :toc: left diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/read-these-first/introduction-to-ue4-docker.adoc index 521bf908..703e1e59 100644 --- a/docs/read-these-first/introduction-to-ue4-docker.adoc +++ b/docs/read-these-first/introduction-to-ue4-docker.adoc @@ -5,8 +5,6 @@ :source-highlighter: rouge :toc: -== Overview - The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' https://www.unrealengine.com/[Unreal Engine]. The images also incorporate the infrastructure from `ue4cli`, `conan-ue4cli`, and `ue4-ci-helpers` to facilitate a wide variety of use cases. From d3c8d0948b0625477a29978c3b5ef4b721a8525f Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 12:21:26 +0300 Subject: [PATCH 307/430] Repair logo in README.adoc --- README.adoc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.adoc b/README.adoc index 55d51231..c789f732 100644 --- a/README.adoc +++ b/README.adoc @@ -1,6 +1,9 @@ -= Unreal Engine 4 Docker Containers - -image:https://raw.githubusercontent.com/adamrehn/ue4-docker/master/resources/images/banner.svg?sanitize=true[Unreal Engine and Docker Logos,align="center"] +++++ +

Unreal Engine and Docker Logos

+

Unreal Engine 4 Docker Containers

+

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

+

 

+++++ *Looking for a place to start? Check out the https://unrealcontainers.com/[Unreal Containers community hub] for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the https://docs.adamrehn.com/ue4-docker/[comprehensive ue4-docker documentation] to view details specific to using the ue4-docker project.* From 65dca0cd2a3490c521603768a2f6adf5bff3f1cb Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 13:06:43 +0300 Subject: [PATCH 308/430] Replace "UE4" with "Unreal Engine" in docs --- docs/building-images/advanced-build-options.adoc | 6 +++--- docs/building-images/available-container-images.adoc | 4 ++-- docs/commands/ue4-docker-export.adoc | 2 +- docs/read-these-first/frequently-asked-questions.adoc | 2 +- docs/read-these-first/introduction-to-ue4-docker.adoc | 8 ++++---- docs/use-cases/linux-installed-builds.adoc | 2 +- docs/use-cases/microservices.adoc | 2 +- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/building-images/advanced-build-options.adoc b/docs/building-images/advanced-build-options.adoc index 2ee56c92..b7504656 100644 --- a/docs/building-images/advanced-build-options.adoc +++ b/docs/building-images/advanced-build-options.adoc @@ -18,7 +18,7 @@ If you would like to override the default `adamrehn/` prefix that is used when g === Specifying Git credentials -The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the UE4 Git repository: +The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the Unreal Engine Git repository: - **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. @@ -33,10 +33,10 @@ Users who have enabled https://help.github.com/en/articles/about-two-factor-auth === Building a custom version of the Unreal Engine -If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +If you would like to build a custom version of Unreal Engine rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. When building a custom Engine version, **both the repository URL and branch/tag must be specified**: -If you would like to build a custom version of UE4 rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +If you would like to build a custom version of Unreal Engine rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. When building a custom Engine version, **both the repository URL and branch/tag must be specified**: [source,shell] diff --git a/docs/building-images/available-container-images.adoc b/docs/building-images/available-container-images.adoc index d12cb60d..45a997f1 100644 --- a/docs/building-images/available-container-images.adoc +++ b/docs/building-images/available-container-images.adoc @@ -47,7 +47,7 @@ You can prevent unwanted images from being built by appending the relevant image **Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/linux/Dockerfile[icon:linux[] Linux] -**Contents:** Contains the cloned source code for UE4, along with its downloaded dependency data. +**Contents:** Contains the cloned source code for Unreal Engine, along with its downloaded dependency data. The ue4-minimal image uses this source code as the starting point for its build. **Uses:** @@ -96,4 +96,4 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save * xref:../use-cases/continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers * Packaging or running cloud rendering projects that optionally utilise UE4Capture under Linux (**note that the (https://hub.docker.com/r/adamrehn/ue4-runtime[ue4-runtime]) image is the recommended base image for running cloud rendering workloads, since this avoids including the multi-gigabyte UE4 developer tools**) -* Packaging xref:../use-cases/microservices.adoc[UE4-powered microservices] +* Packaging xref:../use-cases/microservices.adoc[Unreal Engine-powered microservices] diff --git a/docs/commands/ue4-docker-export.adoc b/docs/commands/ue4-docker-export.adoc index df469745..4805f5d3 100644 --- a/docs/commands/ue4-docker-export.adoc +++ b/docs/commands/ue4-docker-export.adoc @@ -19,7 +19,7 @@ This command can be used to export the following components: === Exporting Installed Builds under Linux -Installed Builds of UE4 can be exported to the host system starting with version 4.21.0. Once you have built either the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: +Installed Builds of Unreal Engine can be exported to the host system starting with version 4.21.0. Once you have built either the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: [source,shell] ---- diff --git a/docs/read-these-first/frequently-asked-questions.adoc b/docs/read-these-first/frequently-asked-questions.adoc index f0a4e06a..ebe121f8 100644 --- a/docs/read-these-first/frequently-asked-questions.adoc +++ b/docs/read-these-first/frequently-asked-questions.adoc @@ -11,7 +11,7 @@ With the exception of the ../building-images/available-container-images.adoc#ue4 The first purpose is self-documentation. These Docker images are the first publicly-available Windows and Linux images to provide comprehensive build capabilities for Unreal Engine 4. -Along with the supporting documentation and https://adamrehn.com/articles/tag/Unreal%20Engine/[articles on adamrehn.com], the code in this repository represents an important source of information regarding the steps that must be taken to get UE4 working correctly inside a container. +Along with the supporting documentation and https://adamrehn.com/articles/tag/Unreal%20Engine/[articles on adamrehn.com], the code in this repository represents an important source of information regarding the steps that must be taken to get Unreal Engine working correctly inside a container. The readability of the Dockerfiles is key, which is why they contain so many individual `RUN` directives with explanatory comments. Combining `RUN` directives would reduce readability and potentially obfuscate the significance of critical steps. diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/read-these-first/introduction-to-ue4-docker.adoc index 703e1e59..d4cbb5cd 100644 --- a/docs/read-these-first/introduction-to-ue4-docker.adoc +++ b/docs/read-these-first/introduction-to-ue4-docker.adoc @@ -12,14 +12,14 @@ Key features include: - Unreal Engine 4.20.0 and newer is supported. - Both Windows containers and Linux containers are supported. -- Building and packaging UE4 projects is supported. +- Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. -- Running built UE4 projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. +- Running built Unreal Engine projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. == Important legal notice -**Except for the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the UE4 Engine Tools in both source code and object code form. -As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games UE4 GitHub repository. +**Except for the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the Unreal Engine Tools in both source code and object code form. +As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games Unreal Engine GitHub repository. **Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.** It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** For more details, see the https://unrealcontainers.com/docs/obtaining-images/eula-restrictions[Unreal Engine EULA Restrictions] page on the https://unrealcontainers.com/[Unreal Containers community hub]. diff --git a/docs/use-cases/linux-installed-builds.adoc b/docs/use-cases/linux-installed-builds.adoc index 89cbd92e..38a3fa59 100644 --- a/docs/use-cases/linux-installed-builds.adoc +++ b/docs/use-cases/linux-installed-builds.adoc @@ -5,7 +5,7 @@ :source-highlighter: rouge :toc: -Under Windows and macOS, Engine licensees can easily download and manage Installed Builds of UE4 through the Epic Games Launcher. +Under Windows and macOS, Engine licensees can easily download and manage Installed Builds of Unreal Engine through the Epic Games Launcher. Since version 4.21.0 of the Engine, ue4-docker provides an alternative source of Installed Builds under Linux in lieu of a native version of the launcher. NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. diff --git a/docs/use-cases/microservices.adoc b/docs/use-cases/microservices.adoc index 5671a856..9591d7e9 100644 --- a/docs/use-cases/microservices.adoc +++ b/docs/use-cases/microservices.adoc @@ -5,7 +5,7 @@ :source-highlighter: rouge :toc: -Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full image makes it easy to build UE4-powered microservices with Google's popular gRPC framework. +Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full image makes it easy to build Unreal Engine-powered microservices with Google's popular gRPC framework. NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. You can find the new version here: https://unrealcontainers.com/docs/use-cases/microservices[Use Cases: Microservices]. From 1919b346b64509ed2d26ad9c69647b0ac4fb8aab Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 13:07:42 +0300 Subject: [PATCH 309/430] Update Chocolatey installation command in docs --- docs/configuration/configuring-windows-10.adoc | 2 +- docs/configuration/configuring-windows-server.adoc | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/configuration/configuring-windows-10.adoc b/docs/configuration/configuring-windows-10.adoc index 23bf1b01..109090b1 100644 --- a/docs/configuration/configuring-windows-10.adoc +++ b/docs/configuration/configuring-windows-10.adoc @@ -34,7 +34,7 @@ To do so, run the following command from an elevated PowerShell prompt: [source,powershell] ---- -Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString( 'https://chocolatey.org/install.ps1')) +Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) ---- You may need to restart your shell for it to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. diff --git a/docs/configuration/configuring-windows-server.adoc b/docs/configuration/configuring-windows-server.adoc index 117a85c9..97868c1a 100644 --- a/docs/configuration/configuring-windows-server.adoc +++ b/docs/configuration/configuring-windows-server.adoc @@ -38,10 +38,7 @@ To do so, run the following command from an elevated PowerShell prompt: [source,powershell] ---- -Set-ExecutionPolicy Bypass -Scope Process -Force; -iex ((New-Object System.Net.WebClient).DownloadString( - 'https://chocolatey.org/install.ps1' -)) +Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) ---- You may need to restart the system for your shell to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. From 25a2b103af8967ba6c28dbe76e99d1461f073e12 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 13:09:11 +0300 Subject: [PATCH 310/430] Update Homebrew installation instructions --- docs/configuration/configuring-macos.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration/configuring-macos.adoc b/docs/configuration/configuring-macos.adoc index bc3b66f1..bc3c9e24 100644 --- a/docs/configuration/configuring-macos.adoc +++ b/docs/configuration/configuring-macos.adoc @@ -27,7 +27,7 @@ To do so, run the following commands from a Terminal prompt: [source,shell] ---- # Install Homebrew -/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install) +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" ---- == Install Python From 2eba289bc0bba887187ab5196b71b59e97e00ac4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 13:16:01 +0300 Subject: [PATCH 311/430] Enable sectlinks/sectanchors in docs See https://docs.asciidoctor.org/asciidoc/latest/sections/title-links/ --- docs/index.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/index.adoc b/docs/index.adoc index a11dbfc5..df266cd7 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -4,6 +4,8 @@ :idprefix: :idseparator: - :linkcss: +:sectanchors: +:sectlinks: :source-highlighter: rouge :toc: left From c9031f3ba0c34cb1d2d61cd7cb8b6c1ec61fb1f3 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 13:48:29 +0300 Subject: [PATCH 312/430] Fix xref in docs --- docs/read-these-first/introduction-to-ue4-docker.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/read-these-first/introduction-to-ue4-docker.adoc index d4cbb5cd..3445a03d 100644 --- a/docs/read-these-first/introduction-to-ue4-docker.adoc +++ b/docs/read-these-first/introduction-to-ue4-docker.adoc @@ -39,7 +39,7 @@ Once you're familiar with all the relevant background material, you can dive rig 2. Next, check out the xref:../use-cases.adoc[Use Cases] section for details on the various scenarios in which the Docker images can be used. Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. 3. If you run into any issues or want to customise your build with advanced options, the xref:../building-images.adoc[Building Images] section provides all the relevant details. -4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the xref:../commands.adoc[Command Reference] section. +4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the xref:../commands/index.adoc[Command Reference] section. == Links From 174bbbd2812fd5ce93f6f1b6cecd3c9053c913bb Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 14:35:52 +0300 Subject: [PATCH 313/430] Fix cross-document xrefs --- .../advanced-build-options.adoc | 32 ++++++------- .../available-container-images.adoc | 8 ++-- docs/build.sh | 6 +-- docs/building-images/index.adoc | 10 ----- docs/commands/index.adoc | 15 ------- docs/configuration/index.adoc | 12 ----- .../configuring-linux.adoc | 4 +- .../configuring-macos.adoc | 2 +- .../configuring-windows-10.adoc | 4 +- .../configuring-windows-server.adoc | 6 +-- .../continuous-integration.adoc | 0 .../frequently-asked-questions.adoc | 0 docs/index.adoc | 45 ++++++++++++++++--- .../introduction-to-ue4-docker.adoc | 10 ++--- .../large-container-images-primer.adoc | 4 +- .../linux-installed-builds.adoc | 0 docs/{use-cases => }/microservices.adoc | 0 .../nvidia-docker-primer.adoc | 0 docs/read-these-first/index.adoc | 11 ----- .../supported-host-configurations.adoc | 0 .../troubleshooting-build-issues.adoc | 16 ++++--- docs/{commands => }/ue4-docker-build.adoc | 2 +- docs/{commands => }/ue4-docker-clean.adoc | 0 .../ue4-docker-diagnostics.adoc | 2 +- docs/{commands => }/ue4-docker-export.adoc | 4 +- docs/{commands => }/ue4-docker-info.adoc | 0 docs/{commands => }/ue4-docker-setup.adoc | 4 +- docs/{commands => }/ue4-docker-test.adoc | 2 +- docs/{commands => }/ue4-docker-version.adoc | 0 docs/{use-cases => }/use-cases-overview.adoc | 0 docs/use-cases/index.adoc | 11 ----- .../windows-container-primer.adoc | 6 +-- 32 files changed, 97 insertions(+), 119 deletions(-) rename docs/{building-images => }/advanced-build-options.adoc (83%) rename docs/{building-images => }/available-container-images.adoc (90%) delete mode 100644 docs/building-images/index.adoc delete mode 100644 docs/commands/index.adoc delete mode 100644 docs/configuration/index.adoc rename docs/{configuration => }/configuring-linux.adoc (85%) rename docs/{configuration => }/configuring-macos.adoc (95%) rename docs/{configuration => }/configuring-windows-10.adoc (92%) rename docs/{configuration => }/configuring-windows-server.adoc (85%) rename docs/{use-cases => }/continuous-integration.adoc (100%) rename docs/{read-these-first => }/frequently-asked-questions.adoc (100%) rename docs/{read-these-first => }/introduction-to-ue4-docker.adoc (79%) rename docs/{read-these-first => }/large-container-images-primer.adoc (85%) rename docs/{use-cases => }/linux-installed-builds.adoc (100%) rename docs/{use-cases => }/microservices.adoc (100%) rename docs/{read-these-first => }/nvidia-docker-primer.adoc (100%) delete mode 100644 docs/read-these-first/index.adoc rename docs/{configuration => }/supported-host-configurations.adoc (100%) rename docs/{building-images => }/troubleshooting-build-issues.adoc (90%) rename docs/{commands => }/ue4-docker-build.adoc (95%) rename docs/{commands => }/ue4-docker-clean.adoc (100%) rename docs/{commands => }/ue4-docker-diagnostics.adoc (84%) rename docs/{commands => }/ue4-docker-export.adoc (82%) rename docs/{commands => }/ue4-docker-info.adoc (100%) rename docs/{commands => }/ue4-docker-setup.adoc (84%) rename docs/{commands => }/ue4-docker-test.adoc (60%) rename docs/{commands => }/ue4-docker-version.adoc (100%) rename docs/{use-cases => }/use-cases-overview.adoc (100%) delete mode 100644 docs/use-cases/index.adoc rename docs/{read-these-first => }/windows-container-primer.adoc (59%) diff --git a/docs/building-images/advanced-build-options.adoc b/docs/advanced-build-options.adoc similarity index 83% rename from docs/building-images/advanced-build-options.adoc rename to docs/advanced-build-options.adoc index b7504656..c07486a4 100644 --- a/docs/building-images/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -9,7 +9,7 @@ === Performing a dry run -If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command. +If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the xref:ue4-docker-build.adoc[ue4-docker build] command. Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. === Specifying a custom namespace for the image tags @@ -18,7 +18,7 @@ If you would like to override the default `adamrehn/` prefix that is used when g === Specifying Git credentials -The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the Unreal Engine Git repository: +The xref:ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the Unreal Engine Git repository: - **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. @@ -58,7 +58,7 @@ This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn [[exclude-components]] === Excluding Engine components to reduce the final image size -Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] images. +Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the xref:ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:available-container-images.adoc#ue4-full[ue4-full] images. The following components can be excluded: - `ddc`: disables building the DDC for the Engine. @@ -105,7 +105,7 @@ ue4-docker build 4.24.2 --monitor -interval=5 [[exporting-generated-dockerfiles]] === Exporting generated Dockerfiles -Since ue4-docker version 0.0.78, the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. +Since ue4-docker version 0.0.78, the xref:ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a https://docs.docker.com/develop/develop-images/multistage-build/[multi-stage build]. You can use these flags like so: @@ -141,10 +141,10 @@ This generates Dockerfiles that use the Linux-only https://docs.docker.com/devel NOTE: Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers[Jinja identifiers]. -Since ue4-docker version 0.0.78, the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. +Since ue4-docker version 0.0.78, the xref:ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. Some of these options (such as `source_mode`) can only be used when <>, whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: -- **`source_mode`**: *(string)* controls how the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. +- **`source_mode`**: *(string)* controls how the xref:available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. Valid options are: - `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. @@ -153,18 +153,18 @@ This is the only mode that can be used when not <>, since the credential endpoint will not be available during the build process. -- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] container during the build process. +- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process. -- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image. +- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image. - **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. -This includes the labels which specify the <> as well as the sentinel labels that the xref:../commands/ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. +This includes the labels which specify the <> as well as the sentinel labels that the xref:ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. - **`disable_all_patches`**: *(boolean)* disables all the patches that ue4-docker ordinarily applies to the Unreal Engine source code. This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. @@ -198,8 +198,8 @@ Some older versions of the Unreal Engine did not copy this correctly, breaking t [[windows-base-tag]] === Specifying the Windows Server Core base image tag -NOTE: The `-basetag` flag controls how the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. -If you are building multiple related images over separate invocations of the build command (e.g. building the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +NOTE: The `-basetag` flag controls how the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. @@ -231,7 +231,7 @@ To do so, pass `--visual-studio=2019` flag when invoking the build command. WARNING: Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. See xref:troubleshooting-build-issues.adoc#copy-8gb-20gb[this section of the Troubleshooting Build Issues page] for further details on this issue. -Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. +Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. (This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) @@ -248,8 +248,8 @@ To build Linux container images instead, simply specify the `--linux` flag when [[cuda]] === Enabling CUDA support for GPU-enabled Linux images -IMPORTANT: The `--cuda` flag controls how the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. -If you are building multiple related images over separate invocations of the build command (e.g. building the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. +IMPORTANT: The `--cuda` flag controls how the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. diff --git a/docs/building-images/available-container-images.adoc b/docs/available-container-images.adoc similarity index 90% rename from docs/building-images/available-container-images.adoc rename to docs/available-container-images.adoc index 45a997f1..4bb53b97 100644 --- a/docs/building-images/available-container-images.adoc +++ b/docs/available-container-images.adoc @@ -5,7 +5,7 @@ :source-highlighter: rouge :toc: -You can build the following images using the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command: +You can build the following images using the xref:ue4-docker-build.adoc[ue4-docker build] command: - <> - <> @@ -70,7 +70,7 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save **Uses:** -* Use this image for xref:../use-cases/continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. +* Use this image for xref:continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. [[ue4-full]] == ue4-full @@ -94,6 +94,6 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save **Uses:** -* xref:../use-cases/continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers +* xref:continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers * Packaging or running cloud rendering projects that optionally utilise UE4Capture under Linux (**note that the (https://hub.docker.com/r/adamrehn/ue4-runtime[ue4-runtime]) image is the recommended base image for running cloud rendering workloads, since this avoids including the multi-gigabyte UE4 developer tools**) -* Packaging xref:../use-cases/microservices.adoc[Unreal Engine-powered microservices] +* Packaging xref:microservices.adoc[Unreal Engine-powered microservices] diff --git a/docs/build.sh b/docs/build.sh index 2b604d9e..81c4250b 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -10,6 +10,6 @@ function docker_asciidoctor() { docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" } -docker_asciidoctor asciidoctor /project/docs/index.adoc -o /project/build/gh-pages/index.html -docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.pdf -docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.epub +docker_asciidoctor asciidoctor /project/docs/index.adoc -D /project/build/gh-pages/ +docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -D /project/build/gh-pages/ +docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -D /project/build/gh-pages/ diff --git a/docs/building-images/index.adoc b/docs/building-images/index.adoc deleted file mode 100644 index 6c236ebe..00000000 --- a/docs/building-images/index.adoc +++ /dev/null @@ -1,10 +0,0 @@ -= Building images -:icons: font -:idprefix: -:idseparator: - -:source-highlighter: rouge -:toc: left - -include::available-container-images.adoc[leveloffset=+1] -include::advanced-build-options.adoc[leveloffset=+1] -include::troubleshooting-build-issues.adoc[leveloffset=+1] diff --git a/docs/commands/index.adoc b/docs/commands/index.adoc deleted file mode 100644 index 6ba48c57..00000000 --- a/docs/commands/index.adoc +++ /dev/null @@ -1,15 +0,0 @@ -= Command reference -:icons: font -:idprefix: -:idseparator: - -:source-highlighter: rouge -:toc: left - -include::ue4-docker-build.adoc[leveloffset=+1] -include::ue4-docker-clean.adoc[leveloffset=+1] -include::ue4-docker-diagnostics.adoc[leveloffset=+1] -include::ue4-docker-export.adoc[leveloffset=+1] -include::ue4-docker-info.adoc[leveloffset=+1] -include::ue4-docker-setup.adoc[leveloffset=+1] -include::ue4-docker-test.adoc[leveloffset=+1] -include::ue4-docker-version.adoc[leveloffset=+1] diff --git a/docs/configuration/index.adoc b/docs/configuration/index.adoc deleted file mode 100644 index 0eafb5fc..00000000 --- a/docs/configuration/index.adoc +++ /dev/null @@ -1,12 +0,0 @@ -= Configuration -:icons: font -:idprefix: -:idseparator: - -:source-highlighter: rouge -:toc: left - -include::supported-host-configurations.adoc[leveloffset=+1] -include::configuring-linux.adoc[leveloffset=+1] -include::configuring-windows-server.adoc[leveloffset=+1] -include::configuring-windows-10.adoc[leveloffset=+1] -include::configuring-macos.adoc[leveloffset=+1] diff --git a/docs/configuration/configuring-linux.adoc b/docs/configuring-linux.adoc similarity index 85% rename from docs/configuration/configuring-linux.adoc rename to docs/configuring-linux.adoc index 49f96fa1..71f75f8e 100644 --- a/docs/configuration/configuring-linux.adoc +++ b/docs/configuring-linux.adoc @@ -60,8 +60,8 @@ sudo pip3 install ue4-docker == Step 4: Use ue4-docker to automatically configure the Linux firewall -If the host system is running an active firewall that blocks access to port 9876 (required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. -The xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. +If the host system is running an active firewall that blocks access to port 9876 (required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. +The xref:ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. Simply run: [source,shell] diff --git a/docs/configuration/configuring-macos.adoc b/docs/configuring-macos.adoc similarity index 95% rename from docs/configuration/configuring-macos.adoc rename to docs/configuring-macos.adoc index bc3c9e24..3f587bdf 100644 --- a/docs/configuration/configuring-macos.adoc +++ b/docs/configuring-macos.adoc @@ -6,7 +6,7 @@ :toc: WARNING: macOS provides a suboptimal experience when running Linux containers, due to the following factors: -Linux containers are unable to use GPU acceleration via the xref:../read-these-first/nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. +Linux containers are unable to use GPU acceleration via the xref:nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. == Requirements diff --git a/docs/configuration/configuring-windows-10.adoc b/docs/configuring-windows-10.adoc similarity index 92% rename from docs/configuration/configuring-windows-10.adoc rename to docs/configuring-windows-10.adoc index 109090b1..36bb38d1 100644 --- a/docs/configuration/configuring-windows-10.adoc +++ b/docs/configuring-windows-10.adoc @@ -8,11 +8,11 @@ == Warning Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. -Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:windows-container-primer.adoc[several issues that impact performance and stability]. The default isolation mode depends on the specific version of Windows being used: - Under Windows 10, Hyper-V isolation mode is the default isolation mode and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-[process isolation mode must be manually enabled] each time a container is built or run. -The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** +The xref:ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** - Under Windows 11, process isolation mode is the default isolation mode. diff --git a/docs/configuration/configuring-windows-server.adoc b/docs/configuring-windows-server.adoc similarity index 85% rename from docs/configuration/configuring-windows-server.adoc rename to docs/configuring-windows-server.adoc index 97868c1a..2e857f85 100644 --- a/docs/configuration/configuring-windows-server.adoc +++ b/docs/configuring-windows-server.adoc @@ -6,7 +6,7 @@ :toc: IMPORTANT: Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. -Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:../read-these-first/windows-container-primer.adoc[several issues that impact performance and stability]. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:windows-container-primer.adoc[several issues that impact performance and stability]. Process isolation mode is the default isolation mode under Windows Server. == Requirements @@ -60,11 +60,11 @@ pip install ue4-docker == Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall -To automatically configure the required system settings, run the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: +To automatically configure the required system settings, run the xref:ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: [source,shell] ---- ue4-docker setup ---- -This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. +This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. diff --git a/docs/use-cases/continuous-integration.adoc b/docs/continuous-integration.adoc similarity index 100% rename from docs/use-cases/continuous-integration.adoc rename to docs/continuous-integration.adoc diff --git a/docs/read-these-first/frequently-asked-questions.adoc b/docs/frequently-asked-questions.adoc similarity index 100% rename from docs/read-these-first/frequently-asked-questions.adoc rename to docs/frequently-asked-questions.adoc diff --git a/docs/index.adoc b/docs/index.adoc index df266cd7..0b99e112 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -9,8 +9,43 @@ :source-highlighter: rouge :toc: left -include::read-these-first/index.adoc[leveloffset=+1] -include::configuration/index.adoc[leveloffset=+1] -include::use-cases/index.adoc[leveloffset=+1] -include::building-images/index.adoc[leveloffset=+1] -include::commands/index.adoc[leveloffset=+1] += Read these first + +include::introduction-to-ue4-docker.adoc[leveloffset=+1] +include::large-container-images-primer.adoc[leveloffset=+1] +include::windows-container-primer.adoc[leveloffset=+1] +include::nvidia-docker-primer.adoc[leveloffset=+1] + +[[configuration]] += Configuration + +include::supported-host-configurations.adoc[leveloffset=+1] +include::configuring-linux.adoc[leveloffset=+1] +include::configuring-windows-server.adoc[leveloffset=+1] +include::configuring-windows-10.adoc[leveloffset=+1] +include::configuring-macos.adoc[leveloffset=+1] + +[[use-cases]] += Use cases + +include::use-cases-overview.adoc[leveloffset=+1] +include::continuous-integration.adoc[leveloffset=+1] +include::microservices.adoc[leveloffset=+1] +include::linux-installed-builds.adoc[leveloffset=+1] + += Building images + +include::available-container-images.adoc[leveloffset=+1] +include::advanced-build-options.adoc[leveloffset=+1] +include::troubleshooting-build-issues.adoc[leveloffset=+1] + += Command reference + +include::ue4-docker-build.adoc[leveloffset=+1] +include::ue4-docker-clean.adoc[leveloffset=+1] +include::ue4-docker-diagnostics.adoc[leveloffset=+1] +include::ue4-docker-export.adoc[leveloffset=+1] +include::ue4-docker-info.adoc[leveloffset=+1] +include::ue4-docker-setup.adoc[leveloffset=+1] +include::ue4-docker-test.adoc[leveloffset=+1] +include::ue4-docker-version.adoc[leveloffset=+1] diff --git a/docs/read-these-first/introduction-to-ue4-docker.adoc b/docs/introduction-to-ue4-docker.adoc similarity index 79% rename from docs/read-these-first/introduction-to-ue4-docker.adoc rename to docs/introduction-to-ue4-docker.adoc index 3445a03d..55a100e4 100644 --- a/docs/read-these-first/introduction-to-ue4-docker.adoc +++ b/docs/introduction-to-ue4-docker.adoc @@ -18,7 +18,7 @@ Key features include: == Important legal notice -**Except for the xref:../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the Unreal Engine Tools in both source code and object code form. +**Except for the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the Unreal Engine Tools in both source code and object code form. As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games Unreal Engine GitHub repository. **Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.** It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** @@ -35,11 +35,11 @@ Before you start using ue4-docker it may be helpful to familiarise yourself with Once you're familiar with all the relevant background material, you can dive right in: -1. First up, head to the xref:../configuration.adoc[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. -2. Next, check out the xref:../use-cases.adoc[Use Cases] section for details on the various scenarios in which the Docker images can be used. +1. First up, head to the xref:#configuration[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. +2. Next, check out the xref:#use-cases[Use Cases] section for details on the various scenarios in which the Docker images can be used. Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. -3. If you run into any issues or want to customise your build with advanced options, the xref:../building-images.adoc[Building Images] section provides all the relevant details. -4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the xref:../commands/index.adoc[Command Reference] section. +3. If you run into any issues or want to customise your build with advanced options, the <<#building-images,Building Images>> section provides all the relevant details. +4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the <<#command-reference,Command Reference>> section. == Links diff --git a/docs/read-these-first/large-container-images-primer.adoc b/docs/large-container-images-primer.adoc similarity index 85% rename from docs/read-these-first/large-container-images-primer.adoc rename to docs/large-container-images-primer.adoc index e758c76b..0d2563b9 100644 --- a/docs/read-these-first/large-container-images-primer.adoc +++ b/docs/large-container-images-primer.adoc @@ -14,7 +14,7 @@ This information applies equally to both Linux containers and Windows containers The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. For some larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. -(The Installed Build layer in the multi-stage build of the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) +(The Installed Build layer in the multi-stage build of the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. @@ -28,5 +28,5 @@ These overheads are relatively modest when building smaller container images. However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least 400GB of available disk space under Linux and at least 800GB of available disk space under Windows. -Once a build is complete, the xref:../commands/ue4-docker-clean.adoc[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. +Once a build is complete, the xref:ue4-docker-clean.adoc[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. The https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/use-cases/linux-installed-builds.adoc b/docs/linux-installed-builds.adoc similarity index 100% rename from docs/use-cases/linux-installed-builds.adoc rename to docs/linux-installed-builds.adoc diff --git a/docs/use-cases/microservices.adoc b/docs/microservices.adoc similarity index 100% rename from docs/use-cases/microservices.adoc rename to docs/microservices.adoc diff --git a/docs/read-these-first/nvidia-docker-primer.adoc b/docs/nvidia-docker-primer.adoc similarity index 100% rename from docs/read-these-first/nvidia-docker-primer.adoc rename to docs/nvidia-docker-primer.adoc diff --git a/docs/read-these-first/index.adoc b/docs/read-these-first/index.adoc deleted file mode 100644 index 9fefc4ce..00000000 --- a/docs/read-these-first/index.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Read these first -:icons: font -:idprefix: -:idseparator: - -:source-highlighter: rouge -:toc: left - -include::introduction-to-ue4-docker.adoc[leveloffset=+1] -include::large-container-images-primer.adoc[leveloffset=+1] -include::windows-container-primer.adoc[leveloffset=+1] -include::nvidia-docker-primer.adoc[leveloffset=+1] diff --git a/docs/configuration/supported-host-configurations.adoc b/docs/supported-host-configurations.adoc similarity index 100% rename from docs/configuration/supported-host-configurations.adoc rename to docs/supported-host-configurations.adoc diff --git a/docs/building-images/troubleshooting-build-issues.adoc b/docs/troubleshooting-build-issues.adoc similarity index 90% rename from docs/building-images/troubleshooting-build-issues.adoc rename to docs/troubleshooting-build-issues.adoc index 10b69c1a..8f74cf1d 100644 --- a/docs/building-images/troubleshooting-build-issues.adoc +++ b/docs/troubleshooting-build-issues.adoc @@ -10,7 +10,7 @@ === Building the `ue4-build-prerequisites` image fails with a network-related error This indicates an underlying network or proxy server issue outside ue4-docker itself that you will need to troubleshoot. -You can use the xref:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. +You can use the xref:ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. Here are some steps to try: - If your host system accesses the network through a proxy server, make sure that https://docs.docker.com/network/proxy/[Docker is configured to use the correct proxy server settings]. @@ -28,7 +28,7 @@ Here are some steps to try: This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. -(Under Windows Server, the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) +(Under Windows Server, the xref:ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) === Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file @@ -42,14 +42,14 @@ Note that restarting the Docker daemon and/or rebooting the host system may also Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. - For Windows containers, follow https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size[the instructions provided by Microsoft], making sure you restart the Docker daemon after you've modified the config JSON. -(Under Windows Server, the xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) +(Under Windows Server, the xref:ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) - For Linux containers, use the https://docs.docker.com/docker-for-windows/#advanced[Docker for Windows "Advanced" settings tab] under Windows or the https://docs.docker.com/docker-for-mac/#disk[Docker for Mac "Disk" settings tab] under macOS. [[copy-8gb-20gb]] === Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image WARNING: Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. -Since ue4-docker version 0.0.47, you can use the xref:../commands/ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. +Since ue4-docker version 0.0.47, you can use the xref:ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. If it does, you may need to xref:advanced-build-options.adoc#exclude-components[exclude debug symbols] when building Windows images. Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: @@ -71,7 +71,7 @@ This issue can occur both inside containers and when running directly on a host . Untag the available-container-images.adoc#ue4-source[ue4-source] image. . Clear the Docker filesystem layer cache by running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune]. -. Re-run the xref:../commands/ue4-docker-build.adoc[ue4-docker build] command. +. Re-run the xref:ue4-docker-build.adoc[ue4-docker build] command. == Windows-specific issues @@ -131,16 +131,18 @@ The simplest fix is to simply reboot the host system and then re-run the build c Insufficient available memory may also contribute to triggering this bug. Note that a linker wrapper https://docs.unrealengine.com/en-US/Support/Builds/ReleaseNotes/4_24/index.html[was added in Unreal Engine 4.24.0] to automatically retry link operations in the event that this bug occurs, so it shouldn't be an issue when building version 4.24.0 or newer. +[[pagefile]] === Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` This error typically occurs when the Windows pagefile size is not large enough. -As stated in the xref:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. +As stated in the xref:troubleshooting-build-issues.adoc#pagefile[Troubleshooting build issues], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. However, containers running in process isolation mode will use the pagefile settings of the host system. When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. (Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) +[[windows-bind-mount]] === Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system -As described in the xref:../read-these-first/windows-container-primer.adoc#hyper-v-isolation-mode-issues[Windows containers primer], the paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. +The paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/commands/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc similarity index 95% rename from docs/commands/ue4-docker-build.adoc rename to docs/ue4-docker-build.adoc index 64d17432..e6f3279b 100644 --- a/docs/commands/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -27,7 +27,7 @@ You will be prompted for the Git credentials to be used when cloning the Unreal The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. By default, all available images will be built. -See the xref:../building-images/available-container-images.adoc[List of available container images] for details on customising which images are built. +See the xref:available-container-images.adoc[List of available container images] for details on customising which images are built. == Options diff --git a/docs/commands/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc similarity index 100% rename from docs/commands/ue4-docker-clean.adoc rename to docs/ue4-docker-clean.adoc diff --git a/docs/commands/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc similarity index 84% rename from docs/commands/ue4-docker-diagnostics.adoc rename to docs/ue4-docker-diagnostics.adoc index 975b698a..dcfb0340 100644 --- a/docs/commands/ue4-docker-diagnostics.adoc +++ b/docs/ue4-docker-diagnostics.adoc @@ -20,7 +20,7 @@ This command can be used to run the following diagnostics: === Checking for the Docker 8GiB filesystem layer bug Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. -This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, xref:../building-images/troubleshooting-build-issues.adoc#building-the-ue4-minimal-image-fails-on-the-copy---frombuilder-directive-that-copies-the-installed-build-from-the-intermediate-image-into-the-final-image[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. +This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, xref:troubleshooting-build-issues.adoc#copy-8gb-20gb[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: diff --git a/docs/commands/ue4-docker-export.adoc b/docs/ue4-docker-export.adoc similarity index 82% rename from docs/commands/ue4-docker-export.adoc rename to docs/ue4-docker-export.adoc index 4805f5d3..b249823c 100644 --- a/docs/commands/ue4-docker-export.adoc +++ b/docs/ue4-docker-export.adoc @@ -19,7 +19,7 @@ This command can be used to export the following components: === Exporting Installed Builds under Linux -Installed Builds of Unreal Engine can be exported to the host system starting with version 4.21.0. Once you have built either the xref:../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: +Installed Builds of Unreal Engine can be exported to the host system starting with version 4.21.0. Once you have built either the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: [source,shell] ---- @@ -44,7 +44,7 @@ ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/Unrea === Exporting Conan packages -The Conan wrapper packages generated by `conan-ue4cli` can be exported from the xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: +The Conan wrapper packages generated by `conan-ue4cli` can be exported from the xref:available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: [source,shell] ---- diff --git a/docs/commands/ue4-docker-info.adoc b/docs/ue4-docker-info.adoc similarity index 100% rename from docs/commands/ue4-docker-info.adoc rename to docs/ue4-docker-info.adoc diff --git a/docs/commands/ue4-docker-setup.adoc b/docs/ue4-docker-setup.adoc similarity index 84% rename from docs/commands/ue4-docker-setup.adoc rename to docs/ue4-docker-setup.adoc index 2dd894fb..7e8244c6 100644 --- a/docs/commands/ue4-docker-setup.adoc +++ b/docs/ue4-docker-setup.adoc @@ -19,12 +19,12 @@ This command will automatically configure a Linux or Windows Server host system **Under Linux:** -- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. +- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image. **Under Windows Server:** - The Docker daemon will be configured to set the maximum image size for Windows containers to 400GB. -- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:../building-images/available-container-images.adoc#ue4-source[ue4-source] image. +- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image. - Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the https://hub.docker.com/_/microsoft-windows[full Windows base image]. Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. diff --git a/docs/commands/ue4-docker-test.adoc b/docs/ue4-docker-test.adoc similarity index 60% rename from docs/commands/ue4-docker-test.adoc rename to docs/ue4-docker-test.adoc index 8457e386..2e79dd3d 100644 --- a/docs/commands/ue4-docker-test.adoc +++ b/docs/ue4-docker-test.adoc @@ -15,6 +15,6 @@ ue4-docker-test - run tests to verify the correctness of built container images == Description -This command runs a suite of tests to verify that built xref:../building-images/available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. +This command runs a suite of tests to verify that built xref:available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/commands/ue4-docker-version.adoc b/docs/ue4-docker-version.adoc similarity index 100% rename from docs/commands/ue4-docker-version.adoc rename to docs/ue4-docker-version.adoc diff --git a/docs/use-cases/use-cases-overview.adoc b/docs/use-cases-overview.adoc similarity index 100% rename from docs/use-cases/use-cases-overview.adoc rename to docs/use-cases-overview.adoc diff --git a/docs/use-cases/index.adoc b/docs/use-cases/index.adoc deleted file mode 100644 index 244364cb..00000000 --- a/docs/use-cases/index.adoc +++ /dev/null @@ -1,11 +0,0 @@ -= Use cases -:icons: font -:idprefix: -:idseparator: - -:source-highlighter: rouge -:toc: left - -include::use-cases-overview.adoc[leveloffset=+1] -include::continuous-integration.adoc[leveloffset=+1] -include::microservices.adoc[leveloffset=+1] -include::linux-installed-builds.adoc[leveloffset=+1] diff --git a/docs/read-these-first/windows-container-primer.adoc b/docs/windows-container-primer.adoc similarity index 59% rename from docs/read-these-first/windows-container-primer.adoc rename to docs/windows-container-primer.adoc index e8c03c47..f1320f77 100644 --- a/docs/read-these-first/windows-container-primer.adoc +++ b/docs/windows-container-primer.adoc @@ -14,7 +14,7 @@ Due to the performance and stability issues currently associated with containers This necessitates the use of Windows Server as the host system (https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[or Windows 10 version 1809 or newer for development and testing purposes]) and requires that all container images use the same Windows version as the host system. A number of ue4-docker commands provide specific functionality to facilitate this: -* The xref:../commands/ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. -Hyper-V isolation mode will still be used if the user explicitly xref:../building-images/advanced-build-options.adoc#windows-base-tag[specifies a different kernel version] than that of the host system or xref:../building-images/advanced-build-options.adoc#windows-isolation-mode[explicitly requests Hyper-V isolation mode]. +* The xref:ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. +Hyper-V isolation mode will still be used if the user explicitly xref:advanced-build-options.adoc#windows-base-tag[specifies a different kernel version] than that of the host system or xref:advanced-build-options.adoc#windows-isolation-mode[explicitly requests Hyper-V isolation mode]. -* The xref:../commands/ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. +* The xref:ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. From 14ebd938df128688e8877b28773cf2f93c73d5a5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:14:16 +0300 Subject: [PATCH 314/430] Further docs improvements --- docs/advanced-build-options.adoc | 13 ----------- docs/build.sh | 2 +- docs/index.adoc | 5 +++-- docs/ue4-docker-build.adoc | 37 ++++++++++++++++++++++++++++---- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc index c07486a4..907e88a3 100644 --- a/docs/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -7,15 +7,6 @@ == General options -=== Performing a dry run - -If you would like to see what `docker build` commands will be run without actually building anything, you can specify the `--dry-run` flag when invoking the xref:ue4-docker-build.adoc[ue4-docker build] command. -Execution will proceed as normal, but no Git credentials will be requested and all `docker build` commands will be printed to standard output instead of being executed as child processes. - -=== Specifying a custom namespace for the image tags - -If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. - === Specifying Git credentials The xref:ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the Unreal Engine Git repository: @@ -222,10 +213,6 @@ Valid values are `process` (supported under Windows Server and https://docs.micr === Specifying Visual Studio Build Tools version under Windows -By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. -Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. -To do so, pass `--visual-studio=2019` flag when invoking the build command. - === Keeping or excluding Installed Build debug symbols under Windows WARNING: Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. diff --git a/docs/build.sh b/docs/build.sh index 81c4250b..f9da4c84 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -10,6 +10,6 @@ function docker_asciidoctor() { docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" } -docker_asciidoctor asciidoctor /project/docs/index.adoc -D /project/build/gh-pages/ +docker_asciidoctor asciidoctor -a linkcss /project/docs/index.adoc -D /project/build/gh-pages/ docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -D /project/build/gh-pages/ docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -D /project/build/gh-pages/ diff --git a/docs/index.adoc b/docs/index.adoc index 0b99e112..9a780cf1 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -1,9 +1,10 @@ -= ue4-docker += ue4-docker user manual +Adam Rehn ; other contributors +{docdate} :doctype: book :icons: font :idprefix: :idseparator: - -:linkcss: :sectanchors: :sectlinks: :source-highlighter: rouge diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index e6f3279b..1004bde6 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -38,7 +38,8 @@ Print help and exit Rebuild images even if they already exist *--dry-run*:: -Print `docker build` commands instead of running them +Use this if you would like to see what Docker commands would be run by `ue4-docker build` without actually building anything. +Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. *--no-minimal*:: Don't build the ue4-minimal image (deprecated, use --target instead) @@ -50,7 +51,25 @@ Don't build the ue4-full image (deprecated, use --target instead) Disable Docker build cache *--exclude {ddc,debug,templates}*:: -Exclude the specified component (can be specified multiple times to exclude multiple components) +Exclude the specified component from the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:available-container-images.adoc#ue4-full[ue4-full] images. ++ +The following components can be excluded: ++ +- `ddc`: disables building the DDC for the Engine. +This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. + +- `debug`: removes all debug symbols from the built images. + +- `templates`: removes the template projects and samples that ship with the Engine. ++ +You can specify the `--exclude` flag multiple times to exclude as many components as you like. +For example: ++ +[source,shell] +---- +# Excludes both debug symbols and template projects +ue4-docker build 4.21.2 --exclude debug --exclude templates +---- *--opt* _OPT_:: Set an advanced configuration option (can be specified multiple times to specify multiple options) @@ -118,5 +137,15 @@ Use Linux containers under Windows hosts (useful when testing Docker Desktop or *--random-memory*:: Use a random memory limit for Windows containers -*--visual-studio* _VISUAL_STUDIO_:: -Specify Visual Studio Build Tools version +*--visual-studio* {2017,2019,2022}:: +Specify Visual Studio Build Tools version. ++ +By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. +Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. ++ +Unreal Engine 5.0 adds support for VS2022 but removes support for VS2017. + +== Environment Variables + +*UE4DOCKER_TAG_NAMESPACE*:: +If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. From d7538e11cc266f20e3294f3fe4d34eb836232105 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:16:30 +0300 Subject: [PATCH 315/430] Fix out paths for PDF/EPUB --- docs/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index f9da4c84..71508773 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -11,5 +11,5 @@ function docker_asciidoctor() { } docker_asciidoctor asciidoctor -a linkcss /project/docs/index.adoc -D /project/build/gh-pages/ -docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -D /project/build/gh-pages/ -docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -D /project/build/gh-pages/ +docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.pdf +docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.epub From ad058965c39bb3830d7352cc51ee6cb5647f1d5f Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:32:48 +0300 Subject: [PATCH 316/430] Minor fixes to manpages --- docs/build.sh | 11 +++++++++++ docs/ue4-docker-diagnostics.adoc | 2 +- docs/ue4-docker-setup.adoc | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 71508773..53028626 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -10,6 +10,17 @@ function docker_asciidoctor() { docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" } +# HTML docker_asciidoctor asciidoctor -a linkcss /project/docs/index.adoc -D /project/build/gh-pages/ + +# PDF docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.pdf + +# EPUB docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.epub + +# Manpages +for f in docs/ue4-docker-*.adoc +do + docker_asciidoctor asciidoctor -b manpage "/project/${f}" -D /project/build/gh-pages/ +done diff --git a/docs/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc index dcfb0340..973f1ce1 100644 --- a/docs/ue4-docker-diagnostics.adoc +++ b/docs/ue4-docker-diagnostics.adoc @@ -7,7 +7,7 @@ == Name -ue4-docker diagnostics - run diagnostics to detect issues with the host system configuration. +ue4-docker-diagnostics - run diagnostics to detect issues with the host system configuration. == Synopsis diff --git a/docs/ue4-docker-setup.adoc b/docs/ue4-docker-setup.adoc index 7e8244c6..ccf2b8e8 100644 --- a/docs/ue4-docker-setup.adoc +++ b/docs/ue4-docker-setup.adoc @@ -1,4 +1,4 @@ -== ue4-docker-setup (1) += ue4-docker-setup (1) :doctype: manpage :icons: font :idprefix: From f7ff732fda4a9729010abe84dbeab7c0eb0ce65d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:38:49 +0300 Subject: [PATCH 317/430] Adjust sections depth --- docs/index.adoc | 58 ++++++++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/docs/index.adoc b/docs/index.adoc index 9a780cf1..4fc65fca 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -10,43 +10,43 @@ Adam Rehn ; other contributors :source-highlighter: rouge :toc: left -= Read these first +== Read these first -include::introduction-to-ue4-docker.adoc[leveloffset=+1] -include::large-container-images-primer.adoc[leveloffset=+1] -include::windows-container-primer.adoc[leveloffset=+1] -include::nvidia-docker-primer.adoc[leveloffset=+1] +include::introduction-to-ue4-docker.adoc[leveloffset=+2] +include::large-container-images-primer.adoc[leveloffset=+2] +include::windows-container-primer.adoc[leveloffset=+2] +include::nvidia-docker-primer.adoc[leveloffset=+2] [[configuration]] -= Configuration +== Configuration -include::supported-host-configurations.adoc[leveloffset=+1] -include::configuring-linux.adoc[leveloffset=+1] -include::configuring-windows-server.adoc[leveloffset=+1] -include::configuring-windows-10.adoc[leveloffset=+1] -include::configuring-macos.adoc[leveloffset=+1] +include::supported-host-configurations.adoc[leveloffset=+2] +include::configuring-linux.adoc[leveloffset=+2] +include::configuring-windows-server.adoc[leveloffset=+2] +include::configuring-windows-10.adoc[leveloffset=+2] +include::configuring-macos.adoc[leveloffset=+2] [[use-cases]] -= Use cases +== Use cases -include::use-cases-overview.adoc[leveloffset=+1] -include::continuous-integration.adoc[leveloffset=+1] -include::microservices.adoc[leveloffset=+1] -include::linux-installed-builds.adoc[leveloffset=+1] +include::use-cases-overview.adoc[leveloffset=+2] +include::continuous-integration.adoc[leveloffset=+2] +include::microservices.adoc[leveloffset=+2] +include::linux-installed-builds.adoc[leveloffset=+2] -= Building images +== Building images -include::available-container-images.adoc[leveloffset=+1] -include::advanced-build-options.adoc[leveloffset=+1] -include::troubleshooting-build-issues.adoc[leveloffset=+1] +include::available-container-images.adoc[leveloffset=+2] +include::advanced-build-options.adoc[leveloffset=+2] +include::troubleshooting-build-issues.adoc[leveloffset=+2] -= Command reference +== Command reference -include::ue4-docker-build.adoc[leveloffset=+1] -include::ue4-docker-clean.adoc[leveloffset=+1] -include::ue4-docker-diagnostics.adoc[leveloffset=+1] -include::ue4-docker-export.adoc[leveloffset=+1] -include::ue4-docker-info.adoc[leveloffset=+1] -include::ue4-docker-setup.adoc[leveloffset=+1] -include::ue4-docker-test.adoc[leveloffset=+1] -include::ue4-docker-version.adoc[leveloffset=+1] +include::ue4-docker-build.adoc[leveloffset=+2] +include::ue4-docker-clean.adoc[leveloffset=+2] +include::ue4-docker-diagnostics.adoc[leveloffset=+2] +include::ue4-docker-export.adoc[leveloffset=+2] +include::ue4-docker-info.adoc[leveloffset=+2] +include::ue4-docker-setup.adoc[leveloffset=+2] +include::ue4-docker-test.adoc[leveloffset=+2] +include::ue4-docker-version.adoc[leveloffset=+2] From 7e455b09d0639fa440d1cb42fb945f50615df511 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:44:16 +0300 Subject: [PATCH 318/430] Put each command on a separate page --- docs/index.adoc | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/index.adoc b/docs/index.adoc index 4fc65fca..af03428c 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -43,10 +43,31 @@ include::troubleshooting-build-issues.adoc[leveloffset=+2] == Command reference include::ue4-docker-build.adoc[leveloffset=+2] + +<<< + include::ue4-docker-clean.adoc[leveloffset=+2] + +<<< + include::ue4-docker-diagnostics.adoc[leveloffset=+2] + +<<< + include::ue4-docker-export.adoc[leveloffset=+2] + +<<< + include::ue4-docker-info.adoc[leveloffset=+2] + +<<< + include::ue4-docker-setup.adoc[leveloffset=+2] + +<<< + include::ue4-docker-test.adoc[leveloffset=+2] + +<<< + include::ue4-docker-version.adoc[leveloffset=+2] From 7e03a22831a194365fa87c2682c2ae4b321ba4d5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 15:58:49 +0300 Subject: [PATCH 319/430] Add "Improve this doc" buttons --- docs/feedback.adoc | 3 +++ docs/index.adoc | 60 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 docs/feedback.adoc diff --git a/docs/feedback.adoc b/docs/feedback.adoc new file mode 100644 index 00000000..37973b61 --- /dev/null +++ b/docs/feedback.adoc @@ -0,0 +1,3 @@ +ifdef::backend-html5[] +image::https://img.shields.io/badge/improve-this%20doc-orange.svg[link=https://github.com/adamrehn/ue4-docker/edit/master/docs/{filename},float=right] +endif::[] diff --git a/docs/index.adoc b/docs/index.adoc index af03428c..8c69755f 100644 --- a/docs/index.adoc +++ b/docs/index.adoc @@ -12,62 +12,122 @@ Adam Rehn ; other contributors == Read these first +:filename: introduction-to-ue4-docker.adoc +include::feedback.adoc[] include::introduction-to-ue4-docker.adoc[leveloffset=+2] + +:filename: large-container-images-primer.adoc +include::feedback.adoc[] include::large-container-images-primer.adoc[leveloffset=+2] + +:filename: windows-container-primer.adoc +include::feedback.adoc[] include::windows-container-primer.adoc[leveloffset=+2] + +:filename: nvidia-docker-primer.adoc +include::feedback.adoc[] include::nvidia-docker-primer.adoc[leveloffset=+2] [[configuration]] == Configuration +:filename: supported-host-configurations.adoc +include::feedback.adoc[] include::supported-host-configurations.adoc[leveloffset=+2] + +:filename: configuring-linux.adoc +include::feedback.adoc[] include::configuring-linux.adoc[leveloffset=+2] + +:filename: configuring-windows-server.adoc +include::feedback.adoc[] include::configuring-windows-server.adoc[leveloffset=+2] + +:filename: configuring-windows-10.adoc +include::feedback.adoc[] include::configuring-windows-10.adoc[leveloffset=+2] + +:filename: configuring-macos.adoc +include::feedback.adoc[] include::configuring-macos.adoc[leveloffset=+2] [[use-cases]] == Use cases +:filename: use-cases-overview.adoc +include::feedback.adoc[] include::use-cases-overview.adoc[leveloffset=+2] + +:filename: continuous-integration.adoc +include::feedback.adoc[] include::continuous-integration.adoc[leveloffset=+2] + +:filename: microservices.adoc +include::feedback.adoc[] include::microservices.adoc[leveloffset=+2] + +:filename: linux-installed-builds +include::feedback.adoc[] include::linux-installed-builds.adoc[leveloffset=+2] == Building images +:filename: available-container-images.adoc +include::feedback.adoc[] include::available-container-images.adoc[leveloffset=+2] + +:filename: advanced-build-options.adoc +include::feedback.adoc[] include::advanced-build-options.adoc[leveloffset=+2] + +:filename: troubleshooting-build-issues.adoc +include::feedback.adoc[] include::troubleshooting-build-issues.adoc[leveloffset=+2] == Command reference +:filename: ue4-docker-build.adoc +include::feedback.adoc[] include::ue4-docker-build.adoc[leveloffset=+2] <<< +:filename: ue4-docker-clean.adoc +include::feedback.adoc[] include::ue4-docker-clean.adoc[leveloffset=+2] <<< +:filename: ue4-docker-diagnostics.adoc +include::feedback.adoc[] include::ue4-docker-diagnostics.adoc[leveloffset=+2] <<< +:filename: ue4-docker-export.adoc +include::feedback.adoc[] include::ue4-docker-export.adoc[leveloffset=+2] <<< +:filename: ue4-docker-info.adoc +include::feedback.adoc[] include::ue4-docker-info.adoc[leveloffset=+2] <<< +:filename: ue4-docker-setup.adoc +include::feedback.adoc[] include::ue4-docker-setup.adoc[leveloffset=+2] <<< +:filename: ue4-docker-test.adoc +include::feedback.adoc[] include::ue4-docker-test.adoc[leveloffset=+2] <<< +:filename: ue4-docker-version.adoc +include::feedback.adoc[] include::ue4-docker-version.adoc[leveloffset=+2] From 460e9ecae37db5d8489ee499ff61486903d1b4bf Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 16:10:57 +0300 Subject: [PATCH 320/430] Drop references to UE4Capture We no longer bundle UE4Capture since c78a3ede76c55f19ca1bf48c66fb69b8a6a86cf5 --- docs/available-container-images.adoc | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/available-container-images.adoc b/docs/available-container-images.adoc index 4bb53b97..505eb2d1 100644 --- a/docs/available-container-images.adoc +++ b/docs/available-container-images.adoc @@ -88,12 +88,10 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save * ue4cli * conan-ue4cli * ue4-ci-helpers -* UE4Capture (Linux image only) * PulseAudio support (Linux image only) * X11 support (Linux image only) **Uses:** * xref:continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers -* Packaging or running cloud rendering projects that optionally utilise UE4Capture under Linux (**note that the (https://hub.docker.com/r/adamrehn/ue4-runtime[ue4-runtime]) image is the recommended base image for running cloud rendering workloads, since this avoids including the multi-gigabyte UE4 developer tools**) * Packaging xref:microservices.adoc[Unreal Engine-powered microservices] From 1da9663b493d75078b4aeff6ce7366af0ad8101a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 17:16:02 +0300 Subject: [PATCH 321/430] Add missing docs for `ue4-docker clean --prune` option --- docs/ue4-docker-clean.adoc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc index 2a7d7625..df8731bd 100644 --- a/docs/ue4-docker-clean.adoc +++ b/docs/ue4-docker-clean.adoc @@ -30,3 +30,6 @@ Remove all ue4-docker images, applying the tag filter if one was specified *--dry-run*:: If you're unsure as to exactly what images will be removed by a given invocation of the command, append the `--dry-run` flag to have ue4-docker print the generated `docker rmi` commands instead of running them. + +*--prune*:: +Run `docker system prune` after cleaning From fd80a56ac624da6f82bc8e6392c58ea951a962f9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 17:19:29 +0300 Subject: [PATCH 322/430] Remove docs about nonexistent `ue4-docker diagnostics maxsize` --- docs/ue4-docker-diagnostics.adoc | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc index 973f1ce1..5280a825 100644 --- a/docs/ue4-docker-diagnostics.adoc +++ b/docs/ue4-docker-diagnostics.adoc @@ -29,17 +29,7 @@ This diagnostic tests whether the host system's Docker daemon suffers from this ue4-docker diagnostics 8gig ---- -=== Checking for the Windows Host Compute Service (HCS) `storage-opt` bug - -Windows Server versions 1903 and 1909 and Windows 10 versions 1903 and 1909 contain https://github.com/docker/for-win/issues/4100[a bug in the Host Compute Service (HCS)] that prevents users from increasing the maximum allowed image size using Docker's `storage-opt` configuration key. -Since Unreal Engine containers require a far higher limit than the default during builds, this bug prevents the xref:ue4-docker-build.adoc[ue4-docker build] command from functioning correctly on affected systems. - -This diagnostic tests whether the host system is affected this bug, by attempting to run a container with a non-default `storage-opt` value: - -[source,shell] ----- -ue4-docker diagnostics maxsize ----- +// TODO: Add docs on ue4-docker diagnostics 20gig === Checking for container network connectivity issues From d58f59aac0ccc6db5ff8a40115244f43b1710439 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 17:24:00 +0300 Subject: [PATCH 323/430] Minimal docs on `ue4-docker build --target=TARGET` --- docs/ue4-docker-build.adoc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index 1004bde6..678e3173 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -34,6 +34,13 @@ See the xref:available-container-images.adoc[List of available container images] *-h, --help*:: Print help and exit +*--target*=_TARGET_:: +Tells ue4-docker to build specific image (including its dependencies). ++ +Supported values: `all`, `build-prerequisites`, `full`, `minimal`, `source`. ++ +You can specify the `--target` flag multiple times. + *--rebuild*:: Rebuild images even if they already exist @@ -42,10 +49,10 @@ Use this if you would like to see what Docker commands would be run by `ue4-dock Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. *--no-minimal*:: -Don't build the ue4-minimal image (deprecated, use --target instead) +Don't build the ue4-minimal image (deprecated, use *--target* _TARGET_ instead) *--no-full*:: -Don't build the ue4-full image (deprecated, use --target instead) +Don't build the ue4-full image (deprecated, use *--target* _TARGET_ instead) *--no-cache*:: Disable Docker build cache From 15cf89d1bec7ca8fb460ce1444a75724b826f826 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 12 Dec 2022 19:05:30 +0300 Subject: [PATCH 324/430] Stop using CAPS in manpages Lowercase params are pretty readable too. --- docs/ue4-docker-build.adoc | 46 ++++++++++++++++++-------------- docs/ue4-docker-clean.adoc | 6 ++--- docs/ue4-docker-diagnostics.adoc | 2 +- docs/ue4-docker-export.adoc | 2 +- docs/ue4-docker-test.adoc | 2 +- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index 678e3173..c46a3537 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -11,7 +11,7 @@ ue4-docker-build - build container image for a specific version of Unreal Engine == Synopsis -*ue4-docker build* [_OPTION_]... _VERSION_ +*ue4-docker build* [_option_]... _version_ == Description @@ -34,7 +34,7 @@ See the xref:available-container-images.adoc[List of available container images] *-h, --help*:: Print help and exit -*--target*=_TARGET_:: +*--target* _target_:: Tells ue4-docker to build specific image (including its dependencies). + Supported values: `all`, `build-prerequisites`, `full`, `minimal`, `source`. @@ -78,39 +78,39 @@ For example: ue4-docker build 4.21.2 --exclude debug --exclude templates ---- -*--opt* _OPT_:: +*--opt* _opt_:: Set an advanced configuration option (can be specified multiple times to specify multiple options) -*-username* _USERNAME_:: +*-username* _username_:: Specify the username to use when cloning the git repository -*-password* _PASSWORD_:: +*-password* _password_:: Specify access token or password to use when cloning the git repository -*-repo* _REPO_:: +*-repo* _repo_:: Set the custom git repository to clone when "custom" is specified as the release value -*-branch* _BRANCH_:: +*-branch* _branch_:: Set the custom branch/tag to clone when "custom" is specified as the release value -*-basetag* _BASETAG_:: +*-basetag* _basetag_:: Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu18.04). For Windows this is the Windows Server Core base image tag (default is the host OS version) -*-suffix* _SUFFIX_:: +*-suffix* _suffix_:: Add a suffix to the tags of the built images -*-m* _M_:: +*-m* _memory_:: Override the default memory limit under Windows (also overrides --random-memory) -*-ue4cli* _UE4CLI_:: +*-ue4cli* _ue4cli_:: Override the default version of ue4cli installed in the ue4-full image -*-conan-ue4cli* _CONAN_UE4CLI_:: +*-conan-ue4cli* _conan_ue4cli_:: Override the default version of conan-ue4cli installed in the ue4-full image -*-layout* _LAYOUT_:: +*-layout* _layout_:: Copy generated Dockerfiles to the specified directory and don't build the images *--combine*:: @@ -119,15 +119,15 @@ Combine generated Dockerfiles into a single multi-stage build Dockerfile *--monitor*:: Monitor resource usage during builds (useful for debugging) -*-interval* _INTERVAL_:: +*-interval* _inverval_:: Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) -*-v*:: +*-v*, *--verbose*:: Enable verbose output during builds (useful for debugging) == Linux-specific options -*--cuda* _VERSION_:: +*--cuda* _version_:: Add CUDA support as well as OpenGL support == Windows-specific options @@ -135,8 +135,8 @@ Add CUDA support as well as OpenGL support *--ignore-blacklist*:: Run builds even on blacklisted versions of Windows (advanced use only) -*-isolation* _ISOLATION_:: -Set the isolation mode to use (process or hyperv) +*-isolation {process,hyperv}:: +Set the isolation mode to use *--linux*:: Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support) @@ -144,7 +144,7 @@ Use Linux containers under Windows hosts (useful when testing Docker Desktop or *--random-memory*:: Use a random memory limit for Windows containers -*--visual-studio* {2017,2019,2022}:: +*--visual-studio {2017,2019,2022}*:: Specify Visual Studio Build Tools version. + By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. @@ -152,7 +152,13 @@ Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tool + Unreal Engine 5.0 adds support for VS2022 but removes support for VS2017. -== Environment Variables +== Environment + +This section describes several environment variables that affect how `ue4-docker build` operates. *UE4DOCKER_TAG_NAMESPACE*:: If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. + +== See also + +xref:ue4-docker-clean.adoc[*ue4-docker-clean* (1)] diff --git a/docs/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc index df8731bd..4d631448 100644 --- a/docs/ue4-docker-clean.adoc +++ b/docs/ue4-docker-clean.adoc @@ -11,7 +11,7 @@ ue4-docker-clean - cleans built container images. == Synopsis -*ue4-docker clean* [*-tag* _TAG_] [*--source*] [*--all*] [*--dry-run*] +*ue4-docker clean* [*-tag* _tag_] [*--source*] [*--all*] [*--dry-run*] == Description @@ -19,8 +19,8 @@ By default, only dangling intermediate images leftover from ue4-docker multi-sta == Options -*-tag* _TAG_:: -Apply a filter for the three flags below, restricting them to removing only images with the specified _TAG_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) +*-tag* _tag_:: +Apply a filter for the three flags below, restricting them to removing only images with the specified _tag_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) *--source*:: Remove ../building-images/available-container-images.adoc#ue4-source[ue4-source] images, applying the tag filter if one was specified diff --git a/docs/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc index 5280a825..9dd4f898 100644 --- a/docs/ue4-docker-diagnostics.adoc +++ b/docs/ue4-docker-diagnostics.adoc @@ -11,7 +11,7 @@ ue4-docker-diagnostics - run diagnostics to detect issues with the host system c == Synopsis -*ue4-docker diagnostics* _DIAGNOSTIC_ +*ue4-docker diagnostics* _diagnostic_ == Description diff --git a/docs/ue4-docker-export.adoc b/docs/ue4-docker-export.adoc index b249823c..8d4597dc 100644 --- a/docs/ue4-docker-export.adoc +++ b/docs/ue4-docker-export.adoc @@ -11,7 +11,7 @@ ue4-docker-export - export components from built container image to the host sys == Synopsis -*ue4-docker export* _COMPONENT_ _TAG_ _DESTINATION_ +*ue4-docker export* _component_ _tag_ _destination_ This command can be used to export the following components: diff --git a/docs/ue4-docker-test.adoc b/docs/ue4-docker-test.adoc index 2e79dd3d..68f6eecf 100644 --- a/docs/ue4-docker-test.adoc +++ b/docs/ue4-docker-test.adoc @@ -11,7 +11,7 @@ ue4-docker-test - run tests to verify the correctness of built container images == Synopsis -*ue4-docker test* _TAG_ +*ue4-docker test* _tag_ == Description From 335bfe2f8ca4c77cc5c7c57fa131be882f8716ed Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 13 Dec 2022 10:06:45 +0300 Subject: [PATCH 325/430] Further docs tweaks --- docs/build.sh | 2 +- docs/ue4-docker-build.adoc | 13 ++++++------- docs/ue4-docker-clean.adoc | 1 + docs/ue4-docker-diagnostics.adoc | 1 + docs/ue4-docker-export.adoc | 1 + docs/ue4-docker-info.adoc | 1 + docs/ue4-docker-setup.adoc | 1 + docs/ue4-docker-test.adoc | 1 + docs/ue4-docker-version.adoc | 1 + 9 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/build.sh b/docs/build.sh index 53028626..2647f199 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -7,7 +7,7 @@ IFS=$'\n\t' project_dir="$(git rev-parse --show-toplevel)" function docker_asciidoctor() { - docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" + docker run --rm -it --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" } # HTML diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index c46a3537..ebd40a3d 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-build]] = ue4-docker-build (1) :doctype: manpage :icons: font @@ -15,7 +16,7 @@ ue4-docker-build - build container image for a specific version of Unreal Engine == Description -To build container images for a specific version of the Unreal Engine, simply specify the release that you would like to build using full https://semver.org/[semver] version syntax. +To build container images for a specific version of the Unreal Engine, simply specify the _version_ that you would like to build using full https://semver.org/[semver] syntax. For example, to build Unreal Engine 4.20.3, run: [source,shell] @@ -49,10 +50,10 @@ Use this if you would like to see what Docker commands would be run by `ue4-dock Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. *--no-minimal*:: -Don't build the ue4-minimal image (deprecated, use *--target* _TARGET_ instead) +Don't build the ue4-minimal image (deprecated, use *--target* _target_ instead) *--no-full*:: -Don't build the ue4-full image (deprecated, use *--target* _TARGET_ instead) +Don't build the ue4-full image (deprecated, use *--target* _target_ instead) *--no-cache*:: Disable Docker build cache @@ -64,9 +65,7 @@ The following components can be excluded: + - `ddc`: disables building the DDC for the Engine. This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. - - `debug`: removes all debug symbols from the built images. - - `templates`: removes the template projects and samples that ship with the Engine. + You can specify the `--exclude` flag multiple times to exclude as many components as you like. @@ -135,7 +134,7 @@ Add CUDA support as well as OpenGL support *--ignore-blacklist*:: Run builds even on blacklisted versions of Windows (advanced use only) -*-isolation {process,hyperv}:: +*-isolation {process,hyperv}*:: Set the isolation mode to use *--linux*:: @@ -161,4 +160,4 @@ If you would like to override the default `adamrehn/` prefix that is used when g == See also -xref:ue4-docker-clean.adoc[*ue4-docker-clean* (1)] +xref:ue4-docker-clean.adoc#ue4-docker-clean[*ue4-docker-clean*(1)] diff --git a/docs/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc index 4d631448..51131044 100644 --- a/docs/ue4-docker-clean.adoc +++ b/docs/ue4-docker-clean.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-clean]] = ue4-docker-clean (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc index 9dd4f898..23e87b10 100644 --- a/docs/ue4-docker-diagnostics.adoc +++ b/docs/ue4-docker-diagnostics.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-diagnostics]] = ue4-docker-diagnostics (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-export.adoc b/docs/ue4-docker-export.adoc index 8d4597dc..b78186f4 100644 --- a/docs/ue4-docker-export.adoc +++ b/docs/ue4-docker-export.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-export]] = ue4-docker-export (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-info.adoc b/docs/ue4-docker-info.adoc index 4820b368..9d5e6a76 100644 --- a/docs/ue4-docker-info.adoc +++ b/docs/ue4-docker-info.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-info]] = ue4-docker-info (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-setup.adoc b/docs/ue4-docker-setup.adoc index ccf2b8e8..eea570d4 100644 --- a/docs/ue4-docker-setup.adoc +++ b/docs/ue4-docker-setup.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-setup]] = ue4-docker-setup (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-test.adoc b/docs/ue4-docker-test.adoc index 68f6eecf..d4fe2bba 100644 --- a/docs/ue4-docker-test.adoc +++ b/docs/ue4-docker-test.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-test]] = ue4-docker-test (1) :doctype: manpage :icons: font diff --git a/docs/ue4-docker-version.adoc b/docs/ue4-docker-version.adoc index 5116b021..4831c225 100644 --- a/docs/ue4-docker-version.adoc +++ b/docs/ue4-docker-version.adoc @@ -1,3 +1,4 @@ +[[ue4-docker-version]] = ue4-docker-version (1) :doctype: manpage :icons: font From c21667d359faa62070313cd7bd13e573018b8244 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 13 Dec 2022 10:16:14 +0300 Subject: [PATCH 326/430] Sort CLI option docs by alpha --- docs/ue4-docker-build.adoc | 94 +++++++++++++++++++------------------- docs/ue4-docker-clean.adoc | 12 ++--- 2 files changed, 53 insertions(+), 53 deletions(-) diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index ebd40a3d..e0c5a8b5 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -32,32 +32,24 @@ See the xref:available-container-images.adoc[List of available container images] == Options -*-h, --help*:: -Print help and exit +*-basetag* _basetag_:: +Operating system base image tag to use. +For Linux this is the version of Ubuntu (default is ubuntu18.04). +For Windows this is the Windows Server Core base image tag (default is the host OS version) -*--target* _target_:: -Tells ue4-docker to build specific image (including its dependencies). -+ -Supported values: `all`, `build-prerequisites`, `full`, `minimal`, `source`. -+ -You can specify the `--target` flag multiple times. +*-branch* _branch_:: +Set the custom branch/tag to clone when *custom* is specified as the _version_. -*--rebuild*:: -Rebuild images even if they already exist +*--combine*:: +Combine generated Dockerfiles into a single multi-stage build Dockerfile + +*-conan-ue4cli* _conan_ue4cli_:: +Override the default version of conan-ue4cli installed in the ue4-full image *--dry-run*:: Use this if you would like to see what Docker commands would be run by `ue4-docker build` without actually building anything. Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. -*--no-minimal*:: -Don't build the ue4-minimal image (deprecated, use *--target* _target_ instead) - -*--no-full*:: -Don't build the ue4-full image (deprecated, use *--target* _target_ instead) - -*--no-cache*:: -Disable Docker build cache - *--exclude {ddc,debug,templates}*:: Exclude the specified component from the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:available-container-images.adoc#ue4-full[ue4-full] images. + @@ -77,49 +69,57 @@ For example: ue4-docker build 4.21.2 --exclude debug --exclude templates ---- +*-h, --help*:: +Print help and exit + +*-interval* _inverval_:: +Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) + +*-layout* _layout_:: +Copy generated Dockerfiles to the specified directory and don't build the images + +*-m* _memory_:: +Override the default memory limit under Windows (also overrides --random-memory) + +*--monitor*:: +Monitor resource usage during builds (useful for debugging) + +*--no-cache*:: +Disable Docker build cache + +*--no-full*:: +Don't build the ue4-full image (deprecated, use *--target* _target_ instead) + +*--no-minimal*:: +Don't build the ue4-minimal image (deprecated, use *--target* _target_ instead) + *--opt* _opt_:: Set an advanced configuration option (can be specified multiple times to specify multiple options) -*-username* _username_:: -Specify the username to use when cloning the git repository - *-password* _password_:: Specify access token or password to use when cloning the git repository -*-repo* _repo_:: -Set the custom git repository to clone when "custom" is specified as the release value - -*-branch* _branch_:: -Set the custom branch/tag to clone when "custom" is specified as the release value +*--rebuild*:: +Rebuild images even if they already exist -*-basetag* _basetag_:: -Operating system base image tag to use. -For Linux this is the version of Ubuntu (default is ubuntu18.04). -For Windows this is the Windows Server Core base image tag (default is the host OS version) +*-repo* _repo_:: +Set the URL of custom git repository to clone when *custom* is specified as the _version_ *-suffix* _suffix_:: Add a suffix to the tags of the built images -*-m* _memory_:: -Override the default memory limit under Windows (also overrides --random-memory) +*--target* _target_:: +Tells ue4-docker to build specific image (including its dependencies). ++ +Supported values: `all`, `build-prerequisites`, `full`, `minimal`, `source`. ++ +You can specify the `--target` option multiple times. *-ue4cli* _ue4cli_:: Override the default version of ue4cli installed in the ue4-full image -*-conan-ue4cli* _conan_ue4cli_:: -Override the default version of conan-ue4cli installed in the ue4-full image - -*-layout* _layout_:: -Copy generated Dockerfiles to the specified directory and don't build the images - -*--combine*:: -Combine generated Dockerfiles into a single multi-stage build Dockerfile - -*--monitor*:: -Monitor resource usage during builds (useful for debugging) - -*-interval* _inverval_:: -Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) +*-username* _username_:: +Specify the username to use when cloning the git repository *-v*, *--verbose*:: Enable verbose output during builds (useful for debugging) diff --git a/docs/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc index 51131044..8ab13319 100644 --- a/docs/ue4-docker-clean.adoc +++ b/docs/ue4-docker-clean.adoc @@ -20,12 +20,6 @@ By default, only dangling intermediate images leftover from ue4-docker multi-sta == Options -*-tag* _tag_:: -Apply a filter for the three flags below, restricting them to removing only images with the specified _tag_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) - -*--source*:: -Remove ../building-images/available-container-images.adoc#ue4-source[ue4-source] images, applying the tag filter if one was specified - *--all*:: Remove all ue4-docker images, applying the tag filter if one was specified @@ -34,3 +28,9 @@ If you're unsure as to exactly what images will be removed by a given invocation *--prune*:: Run `docker system prune` after cleaning + +*--source*:: +Remove ../building-images/available-container-images.adoc#ue4-source[ue4-source] images, applying the tag filter if one was specified + +*-tag* _tag_:: +Apply a filter for the three flags below, restricting them to removing only images with the specified _tag_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) From 119e8955f898d36c499bfd615727f05f2893b9ac Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 13 Dec 2022 10:18:29 +0300 Subject: [PATCH 327/430] Fix docs publishing on CI --- docs/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/build.sh b/docs/build.sh index 2647f199..53028626 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -7,7 +7,7 @@ IFS=$'\n\t' project_dir="$(git rev-parse --show-toplevel)" function docker_asciidoctor() { - docker run --rm -it --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" + docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" } # HTML From fa2effd5f1466bd73e3b915a48e1109a00293006 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 15 Dec 2022 14:02:33 +1000 Subject: [PATCH 328/430] Add separate GA workflow for publishing docs --- .github/workflows/ci.yml | 13 ----------- .github/workflows/docs.yml | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50df2f57..97c2aab2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,19 +25,6 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} - docs: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' - steps: - - name: Checkout - uses: actions/checkout@v3 - - run: docs/build.sh - - name: Publish - uses: peaceiris/actions-gh-pages@v3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - force_orphan: true - publish_dir: build/gh-pages/ test: strategy: matrix: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..d84b3225 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: Deploy docs to GitHub Pages + +# Only run this workflow when the YAML file itself changes or files change in the "docs" folder +on: + push: + branches: [master] + paths: + - ".github/workflows/docs.yml" + - "docs/**" + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Build docs + run: docs/build.sh + + - name: Upload artifact + uses: actions/upload-pages-artifact@v1 + with: + path: "build/gh-pages" + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v1 From 2204fcc42628635aef5bea90d4ffe90a6261a011 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 19 Jan 2023 15:47:47 +0300 Subject: [PATCH 329/430] Add `gitdependencies_args` option to configure how we invoke GitDependencies The motivation behind such feature is that we're excluding Android by default, but not everyone might want it --- ue4docker/dockerfiles/ue4-source/linux/Dockerfile | 12 ++---------- ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 +- ue4docker/infrastructure/BuildConfiguration.py | 7 +++++++ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index 38e70fdc..b952f903 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -101,11 +101,7 @@ RUN mkdir "$UE_GITDEPS" # When running with BuildKit, we use a cache mount to cache the dependency data across multiple build invocations WORKDIR /home/ue4/UnrealEngine RUN --mount=type=cache,target=/home/ue4/gitdeps,uid=1000,gid=1000 sudo apt-get update && \ - ./Setup.sh \ - --exclude=Android \ - --exclude=Mac \ - --exclude=Win32 \ - --exclude=Win64 && \ + ./Setup.sh {{ gitdependencies_args }} && \ sudo rm -rf /var/lib/apt/lists/* {% else %} @@ -113,11 +109,7 @@ RUN --mount=type=cache,target=/home/ue4/gitdeps,uid=1000,gid=1000 sudo apt-get u # When running without BuildKit, we use the `-no-cache` flag to disable caching of dependency data in `.git/ue4-gitdeps`, saving disk space WORKDIR /home/ue4/UnrealEngine RUN sudo apt-get update && \ - ./Setup.sh -no-cache \ - --exclude=Android \ - --exclude=Mac \ - --exclude=Win32 \ - --exclude=Win64 && \ + ./Setup.sh -no-cache {{ gitdependencies_args }} && \ sudo rm -rf /var/lib/apt/lists/* {% endif %} diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index 9a8f2ce4..e89b1fda 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -71,7 +71,7 @@ RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% # Run post-clone setup steps # (Note that the `-no-cache` flag disables caching of dependency data in `.git/ue4-gitdeps`, saving disk space) WORKDIR C:\UnrealEngine -RUN Setup.bat -no-cache +RUN Setup.bat -no-cache {{ gitdependencies_args }} {% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} # Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 93ad6e52..17580a28 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -526,6 +526,13 @@ def __init__(self, parser, argv, logger): "templates": ExcludedComponent.Templates in self.excludedComponents, } + if "gitdependencies_args" not in self.opts: + self.opts["gitdependencies_args"] = ( + "--exclude=Android --exclude=Mac --exclude=Linux" + if self.containerPlatform == "windows" + else "--exclude=Android --exclude=Mac --exclude=Win32 --exclude=Win64" + ) + # Warn user that they are in danger of Docker 20GB COPY bug # Unfortunately, we don't have a cheap way to check whether user environment is affected # See https://github.com/adamrehn/ue4-docker/issues/99 From c31f014ec860e242df371c9bde63beec94f761a5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 21 Feb 2023 10:26:37 +0300 Subject: [PATCH 330/430] Reformat code with black-23.1.0 --- ue4docker/__main__.py | 1 - ue4docker/build.py | 14 -------------- ue4docker/clean.py | 1 - ue4docker/diagnostics/diagnostic_20gig.py | 2 -- ue4docker/diagnostics/diagnostic_8gig.py | 2 -- ue4docker/diagnostics/diagnostic_all.py | 1 - ue4docker/diagnostics/diagnostic_network.py | 2 -- ue4docker/diagnostics_cmd.py | 4 ---- .../ue4-minimal/linux/copy-toolchain.py | 1 - .../ue4-minimal/linux/set-changelist.py | 2 -- .../ue4-minimal/linux/split-components.py | 3 +-- .../ue4-minimal/windows/set-changelist.py | 2 -- .../ue4-minimal/windows/split-components.py | 3 +-- .../ue4-source/linux/patch-broken-releases.py | 2 -- .../ue4-source/windows/patch-broken-releases.py | 4 ---- ue4docker/export.py | 4 ---- ue4docker/exports/export_packages.py | 4 ---- ue4docker/info.py | 1 - ue4docker/infrastructure/BuildConfiguration.py | 8 -------- ue4docker/infrastructure/DockerUtils.py | 1 - ue4docker/infrastructure/ImageBuilder.py | 5 ----- ue4docker/infrastructure/ResourceMonitor.py | 1 - ue4docker/infrastructure/WindowsUtils.py | 1 - ue4docker/main.py | 4 ---- ue4docker/setup_cmd.py | 9 +-------- ue4docker/test.py | 4 ---- ue4docker/tests/build-and-package.py | 1 - ue4docker/tests/consume-external-deps.py | 1 - 28 files changed, 3 insertions(+), 85 deletions(-) diff --git a/ue4docker/__main__.py b/ue4docker/__main__.py index ebae417b..91bbc937 100644 --- a/ue4docker/__main__.py +++ b/ue4docker/__main__.py @@ -2,7 +2,6 @@ import os, sys if __name__ == "__main__": - # Rewrite sys.argv[0] so our help prompts display the correct base command interpreter = sys.executable if sys.executable not in [None, ""] else "python3" sys.argv[0] = "{} -m ue4docker".format(os.path.basename(interpreter)) diff --git a/ue4docker/build.py b/ue4docker/build.py index af034845..975ce27c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -5,7 +5,6 @@ def _getCredential(args, name, envVar, promptFunc): - # Check if the credential was specified via the command-line if getattr(args, name, None) is not None: print("Using {} specified via `-{}` command-line argument.".format(name, name)) @@ -36,7 +35,6 @@ def _getPassword(args): def build(): - # Create our logger to generate coloured output on stderr logger = Logger(prefix="[{} build] ".format(sys.argv[0])) @@ -90,7 +88,6 @@ def build(): # Create an auto-deleting temporary directory to hold our build context with tempfile.TemporaryDirectory() as tempDir: - contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles") # Create the builder instance to build the Docker images @@ -164,7 +161,6 @@ def build(): # Determine if we are building Windows or Linux containers if config.containerPlatform == "windows": - # Provide the user with feedback so they are aware of the Windows-specific values being used logger.info("WINDOWS CONTAINER SETTINGS", False) logger.info( @@ -303,7 +299,6 @@ def build(): # Determine if we need to prompt for credentials if config.dryRun == True: - # Don't bother prompting the user for any credentials during a dry run logger.info( "Performing a dry run, `docker build` commands will be printed and not executed.", @@ -313,7 +308,6 @@ def build(): password = "" elif config.layoutDir is not None: - # Don't bother prompting the user for any credentials when we're just copying the Dockerfiles to a directory logger.info( "Copying generated Dockerfiles to: {}".format(config.layoutDir), False @@ -325,7 +319,6 @@ def build(): not config.buildTargets["source"] or builder.willBuild("ue4-source", mainTags) == False ): - # Don't bother prompting the user for any credentials if we're not building the ue4-source image logger.info( "Not building the ue4-source image, no Git credentials required.", False @@ -334,7 +327,6 @@ def build(): password = "" else: - # Retrieve the Git username and password from the user when building the ue4-source image print( "\nRetrieving the Git credentials that will be used to clone the UE4 repo" @@ -352,7 +344,6 @@ def build(): endpoint = None try: - # Keep track of our starting time startTime = time.time() @@ -473,7 +464,6 @@ def build(): # Build the full UE4 CI image, unless requested otherwise by the user if config.buildTargets["full"]: - # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them infrastructureFlags = [] if config.ue4cliVersion is not None: @@ -506,10 +496,8 @@ def build(): # If we are generating Dockerfiles then include information about the options used to generate them if config.layoutDir is not None: - # Determine whether we generated a single combined Dockerfile or a set of Dockerfiles if config.combine == True: - # Generate a comment to place at the top of the single combined Dockerfile lines = [ "This file was generated by ue4-docker version {} with the following options:".format( @@ -550,7 +538,6 @@ def build(): FilesystemUtils.writeFile(dockerfile, dockerfileContents) else: - # Create a JSON file to accompany the set of generated Dockerfiles FilesystemUtils.writeFile( join(config.layoutDir, "generated.json"), @@ -581,7 +568,6 @@ def build(): endpoint.stop() except (Exception, KeyboardInterrupt) as e: - # One of the images failed to build logger.error("Error: {}".format(e)) resourceMonitor.stop() diff --git a/ue4docker/clean.py b/ue4docker/clean.py index b8ad1869..d01edfe8 100644 --- a/ue4docker/clean.py +++ b/ue4docker/clean.py @@ -15,7 +15,6 @@ def _cleanMatching(cleaner, filter, tag, dryRun): def clean(): - # Create our logger to generate coloured output on stderr logger = Logger(prefix="[{} clean] ".format(sys.argv[0])) diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/ue4docker/diagnostics/diagnostic_20gig.py index 42539a01..2947d0ef 100644 --- a/ue4docker/diagnostics/diagnostic_20gig.py +++ b/ue4docker/diagnostics/diagnostic_20gig.py @@ -6,12 +6,10 @@ class diagnostic20Gig(DiagnosticBase): - # The tag we use for built images IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:20gig" def __init__(self): - # Setup our argument parser so we can use its help message output in our description text self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 20gig") self._parser.add_argument( diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/ue4docker/diagnostics/diagnostic_8gig.py index f4e50e4a..0662034d 100644 --- a/ue4docker/diagnostics/diagnostic_8gig.py +++ b/ue4docker/diagnostics/diagnostic_8gig.py @@ -6,12 +6,10 @@ class diagnostic8Gig(DiagnosticBase): - # The tag we use for built images IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:8gig" def __init__(self): - # Setup our argument parser so we can use its help message output in our description text self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 8gig") self._parser.add_argument( diff --git a/ue4docker/diagnostics/diagnostic_all.py b/ue4docker/diagnostics/diagnostic_all.py index cada4ba1..e94cf5ff 100644 --- a/ue4docker/diagnostics/diagnostic_all.py +++ b/ue4docker/diagnostics/diagnostic_all.py @@ -30,7 +30,6 @@ def run(self, logger, args=[]): diagnosticNetwork(), ] for index, diagnostic in enumerate(diagnostics): - # Run the diagnostic and report its result logger.info( '[all] Running individual diagnostic: "{}"'.format( diff --git a/ue4docker/diagnostics/diagnostic_network.py b/ue4docker/diagnostics/diagnostic_network.py index b4e00d1a..c2da8dea 100644 --- a/ue4docker/diagnostics/diagnostic_network.py +++ b/ue4docker/diagnostics/diagnostic_network.py @@ -4,12 +4,10 @@ class diagnosticNetwork(DiagnosticBase): - # The tag we use for built images IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:network" def __init__(self): - # Setup our argument parser so we can use its help message output in our description text self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics network") self._parser.add_argument( diff --git a/ue4docker/diagnostics_cmd.py b/ue4docker/diagnostics_cmd.py index 8d40e5d1..bd492d24 100644 --- a/ue4docker/diagnostics_cmd.py +++ b/ue4docker/diagnostics_cmd.py @@ -4,7 +4,6 @@ def diagnostics(): - # The diagnostics that can be run DIAGNOSTICS = { "all": allDiagnostics(), @@ -32,7 +31,6 @@ def diagnostics(): # Determine if we are running a diagnostic if args["help"] == False and args["diagnostic"] is not None: - # Run the diagnostic diagnostic = DIAGNOSTICS[args["diagnostic"]] logger.action('Running diagnostic: "{}"'.format(diagnostic.getName()), False) @@ -46,7 +44,6 @@ def diagnostics(): # Determine if we are displaying the help for a specific diagnostic elif args["help"] == True and args["diagnostic"] is not None: - # Display the help for the diagnostic diagnostic = DIAGNOSTICS[args["diagnostic"]] print("{} diagnostics {}".format(sys.argv[0], args["diagnostic"])) @@ -54,7 +51,6 @@ def diagnostics(): print(diagnostic.getDescription()) else: - # Print usage syntax print("Usage: {} diagnostics DIAGNOSTIC\n".format(sys.argv[0])) print("Runs diagnostics to detect issues with the host system configuration\n") diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py b/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py index c8a797dd..df70e6ec 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py @@ -19,7 +19,6 @@ "x86_64-unknown-linux-gnu", ) for bundled in glob.glob(sdkGlob): - # Extract the root path for the toolchain toolchain = dirname(bundled) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py index c902526b..ea54442c 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py @@ -17,10 +17,8 @@ def writeFile(filename, data): # Determine whether a changelist override value was specified changelistOverride = None if len(sys.argv) > 2: - # If the override was "auto" then attempt to retrieve the CL number from the git commit message if sys.argv[2] == "auto": - # Retrieve the commit message from git engineRoot = dirname(dirname(dirname(sys.argv[1]))) commitMessage = run( diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py index 5ec6243b..742ce5aa 100644 --- a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py @@ -2,6 +2,7 @@ import glob, os, shutil, sys from os.path import basename, dirname, exists, join + # Logs a message to stderr def log(message): print(message, file=sys.stderr) @@ -10,7 +11,6 @@ def log(message): # Extracts the files and directories for the specified component and moves them to a separate output directory def extractComponent(inputDir, outputDir, component, description, items): - # Print progress output log("\nExtracting {}...".format(description)) @@ -20,7 +20,6 @@ def extractComponent(inputDir, outputDir, component, description, items): # Move each file and directory for the component to the output directory for item in items: - # Verify that the item exists if not exists(item): log("Skipping non-existent item: {}".format(item)) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py index 7c772393..1a879e09 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py @@ -17,10 +17,8 @@ def writeFile(filename, data): # Determine whether a changelist override value was specified changelistOverride = None if len(sys.argv) > 2 and sys.argv[2] != "%CHANGELIST%": - # If the override was "auto" then attempt to retrieve the CL number from the git commit message if sys.argv[2] == "auto": - # Retrieve the commit message from git engineRoot = dirname(dirname(dirname(sys.argv[1]))) commitMessage = run( diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py index b3bcf03e..c6412b87 100644 --- a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py +++ b/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -2,6 +2,7 @@ import glob, os, shutil, sys from os.path import basename, dirname, exists, join + # Logs a message to stderr def log(message): print(message, file=sys.stderr) @@ -10,7 +11,6 @@ def log(message): # Extracts the files and directories for the specified component and moves them to a separate output directory def extractComponent(inputDir, outputDir, component, description, items): - # Print progress output log("\nExtracting {}...".format(description)) @@ -20,7 +20,6 @@ def extractComponent(inputDir, outputDir, component, description, items): # Move each file and directory for the component to the output directory for item in items: - # Verify that the item exists if not exists(item): log("Skipping non-existent item: {}".format(item)) diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py index 177f8247..3251a1a9 100644 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -24,13 +24,11 @@ def writeFile(filename, data): and versionDetails["MinorVersion"] == 25 and versionDetails["PatchVersion"] == 4 ): - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them # (See: ) gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") gitdepsXml = readFile(gitdepsFile) if '', '\n ' diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index 02fe54a2..4b655928 100644 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -24,13 +24,11 @@ def writeFile(filename, data): and versionDetails["MinorVersion"] == 25 and versionDetails["PatchVersion"] == 4 ): - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them # (See: ) gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") gitdepsXml = readFile(gitdepsFile) if '', '\n ' @@ -61,13 +59,11 @@ def writeFile(filename, data): and versionDetails["MinorVersion"] == 1 and versionDetails["PatchVersion"] == 0 ): - # Hack InstalledEngineFilters.xml with the changes from CL 23300641 # (See: ) buildFile = join(engineRoot, "Engine", "Build", "InstalledEngineFilters.xml") buildXml = readFile(buildFile) if "HoloLens.Automation.json" not in buildXml: - buildXml = buildXml.replace( '', '\n' diff --git a/ue4docker/export.py b/ue4docker/export.py index 3e98fcda..4f46be8f 100644 --- a/ue4docker/export.py +++ b/ue4docker/export.py @@ -20,7 +20,6 @@ def _stripHelpFlags(args): def export(): - # The components that can be exported COMPONENTS = { "installed": { @@ -59,7 +58,6 @@ def export(): if args["help"] == False and _notNone( [args["component"], args["tag"], args["destination"]] ): - # Determine if the user specified an image and a tag or just a tag tag = args["tag"] details = COMPONENTS[args["component"]] @@ -81,7 +79,6 @@ def export(): # Determine if we are displaying the help for a specific component elif args["help"] == True and args["component"] is not None: - # Display the help for the component component = sys.argv[1] details = COMPONENTS[component] @@ -91,7 +88,6 @@ def export(): print(details["help"]) else: - # Print usage syntax print("Usage: {} export COMPONENT TAG DESTINATION\n".format(sys.argv[0])) print("Exports components from built container images to the host system\n") diff --git a/ue4docker/exports/export_packages.py b/ue4docker/exports/export_packages.py index c78de730..b54a3df0 100644 --- a/ue4docker/exports/export_packages.py +++ b/ue4docker/exports/export_packages.py @@ -30,7 +30,6 @@ def exportPackages(image, destination, extraArgs): - # Create our logger to generate coloured output on stderr logger = Logger() @@ -73,7 +72,6 @@ def exportPackages(image, destination, extraArgs): # Create an auto-deleting temporary directory to hold our server config file with tempfile.TemporaryDirectory() as tempDir: - # Progress output print("Starting conan_server in a container...") @@ -109,7 +107,6 @@ def exportPackages(image, destination, extraArgs): serverOutput = None try: - # Copy the server config file to the expected location inside the container DockerUtils.execMultiple( container, @@ -176,7 +173,6 @@ def exportPackages(image, destination, extraArgs): serverOutput = None finally: - # Stop the container, irrespective of whether or not the export succeeded print("Stopping conan_server...") container.stop() diff --git a/ue4docker/info.py b/ue4docker/info.py index 18a1763e..3420f457 100644 --- a/ue4docker/info.py +++ b/ue4docker/info.py @@ -19,7 +19,6 @@ def _formatSize(size): def info(): - # Verify that Docker is installed if DockerUtils.installed() == False: print( diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 17580a28..5cf231ab 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -386,7 +386,6 @@ def __init__(self, parser, argv, logger): # Determine if we are building a custom version of UE4 rather than an official release self.args.release = self.args.release.lower() if self.args.release == "custom" or self.args.release.startswith("custom:"): - # Both a custom repository and a custom branch/tag must be specified if self.args.repo is None or self.args.branch is None: raise RuntimeError( @@ -405,7 +404,6 @@ def __init__(self, parser, argv, logger): self.custom = True else: - # Validate the specified version string try: ue4Version = semver.parse(self.args.release) @@ -642,7 +640,6 @@ def _generateWindowsConfig(self): if self.args.isolation is not None: self.isolation = self.args.isolation else: - # If we are able to use process isolation mode then use it, otherwise use Hyper-V isolation mode differentKernels = self.basetag != self.hostBasetag dockerSupportsProcess = parse_version( @@ -666,7 +663,6 @@ def _generateWindowsConfig(self): except: raise RuntimeError('invalid memory limit "{}"'.format(self.args.m)) else: - # Only specify a memory limit when using Hyper-V isolation mode, in order to override the 1GB default limit # (Process isolation mode does not impose any memory limits by default) if self.isolation == "hyperv": @@ -683,7 +679,6 @@ def _generateWindowsConfig(self): self.platformArgs.extend(["-m", "{:.2f}GB".format(self.memLimit)]) def _generateLinuxConfig(self): - # Verify that any user-specified tag suffix does not collide with our base tags if self.suffix.startswith("opengl") or self.suffix.startswith("cudagl"): raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') @@ -691,7 +686,6 @@ def _generateLinuxConfig(self): # Determine if we are building CUDA-enabled container images self.cuda = None if self.args.cuda is not None: - # Verify that the specified CUDA version is valid self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION # Use the appropriate base image for the specified CUDA version @@ -709,7 +703,6 @@ def _generateLinuxConfig(self): ) def _processPackageVersion(self, package, version): - # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified # (e.g. package==X.X.X, package>=X.X.X, git+https://url/for/package/repo.git, etc.) if version is None or "/" in version or version.lower().startswith(package): @@ -723,7 +716,6 @@ def _processPackageVersion(self, package, version): return "{}=={}".format(package, version) def _processTemplateValue(self, value): - # If the value is a boolean (either raw or represented by zero or one) then parse it if value.lower() in ["true", "1"]: return True diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py index 5570a28e..87f4fa02 100644 --- a/ue4docker/infrastructure/DockerUtils.py +++ b/ue4docker/infrastructure/DockerUtils.py @@ -218,7 +218,6 @@ def injectPostRunMessage(dockerfile, platform, messageLines): ) if runMatches is not None: for match in runMatches: - # Append the `echo` commands to the directive contents = contents.replace( match[0], "RUN{}{}\n".format(match[1], echoCommands) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py index a6da227a..374ede0a 100644 --- a/ue4docker/infrastructure/ImageBuilder.py +++ b/ue4docker/infrastructure/ImageBuilder.py @@ -122,12 +122,10 @@ def build( # Create a temporary directory to hold any files needed for the build with tempfile.TemporaryDirectory() as tempDir: - # Determine whether we are building using `docker buildx` with build secrets imageTags = self._formatTags(name, tags) if self.platform == "linux" and secrets is not None and len(secrets) > 0: - # Create temporary files to store the contents of each of our secrets secretFlags = [] for secret, contents in secrets.items(): @@ -220,10 +218,8 @@ def _processImage( # Determine if we're just copying the Dockerfile to an output directory if self.layoutDir is not None: - # Determine whether we're performing a simple copy or combining generated Dockerfiles if self.combine: - # Ensure the destination directory exists dest = join(self.layoutDir, "combined") self.logger.action( @@ -261,7 +257,6 @@ def _processImage( ) else: - # Copy the source directory to the destination dest = join(self.layoutDir, basename(name)) self.logger.action( diff --git a/ue4docker/infrastructure/ResourceMonitor.py b/ue4docker/infrastructure/ResourceMonitor.py index 48e5fc24..aa9bf270 100644 --- a/ue4docker/infrastructure/ResourceMonitor.py +++ b/ue4docker/infrastructure/ResourceMonitor.py @@ -44,7 +44,6 @@ def run(self): # Loop until asked to stop while True: - # Check that the thread has not been asked to stop with self._lock: if self._shouldStop == True: diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py index 22e387b6..8961dae4 100644 --- a/ue4docker/infrastructure/WindowsUtils.py +++ b/ue4docker/infrastructure/WindowsUtils.py @@ -8,7 +8,6 @@ class WindowsUtils(object): - # The oldest Windows build we support _minimumRequiredBuild = 17763 diff --git a/ue4docker/main.py b/ue4docker/main.py index 8432b883..430a7779 100644 --- a/ue4docker/main.py +++ b/ue4docker/main.py @@ -22,11 +22,9 @@ def _exitWithError(err): def main(): - # Configure verbose logging if the user requested it # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) if "-v" in sys.argv or "--verbose" in sys.argv: - # Enable verbose logging logging.getLogger().setLevel(logging.DEBUG) @@ -100,7 +98,6 @@ def main(): # Determine if a command has been specified if len(sys.argv) > 1: - # Verify that the specified command is valid command = sys.argv[1] if command not in COMMANDS: @@ -112,7 +109,6 @@ def main(): COMMANDS[command]["function"]() else: - # Print usage syntax print("Usage: {} COMMAND [OPTIONS]\n".format(sys.argv[0])) print("Windows and Linux containers for Unreal Engine 4\n") diff --git a/ue4docker/setup_cmd.py b/ue4docker/setup_cmd.py index 13a730a2..37e8d5a1 100644 --- a/ue4docker/setup_cmd.py +++ b/ue4docker/setup_cmd.py @@ -1,6 +1,7 @@ import docker, os, platform, requests, shutil, subprocess, sys from .infrastructure import * + # Runs a command without displaying its output and returns the exit code def _runSilent(command): result = SubprocessUtils.capture(command, check=False) @@ -9,7 +10,6 @@ def _runSilent(command): # Performs setup for Linux hosts def _setupLinux(): - # Pull the latest version of the Alpine container image alpineImage = "alpine:latest" SubprocessUtils.capture(["docker", "pull", alpineImage]) @@ -19,7 +19,6 @@ def _setupLinux(): endpoint.start() try: - # Run an Alpine container to see if we can access the host port for the credential endpoint SubprocessUtils.capture( [ @@ -39,7 +38,6 @@ def _setupLinux(): print("No firewall configuration required.") except: - # The host port is blocked, so we need to perform firewall configuration print("Creating firewall rule for credential endpoint...") @@ -61,18 +59,15 @@ def _setupLinux(): print("be installed for the rule to persist after the host system reboots.") finally: - # Stop the credential endpoint endpoint.stop() # Performs setup for Windows Server hosts def _setupWindowsServer(): - # Check if we need to configure the maximum image size requiredLimit = WindowsUtils.requiredSizeLimit() if DockerUtils.maxsize() < requiredLimit: - # Attempt to stop the Docker daemon print("Stopping the Docker daemon...") subprocess.run(["sc.exe", "stop", "docker"], check=True) @@ -117,7 +112,6 @@ def _setupWindowsServer(): == 0 ) if ruleExists == False: - # Add a rule to ensure Windows firewall allows access to the credential helper from our containers print("Creating firewall rule for credential endpoint...") subprocess.run( @@ -141,7 +135,6 @@ def _setupWindowsServer(): def setup(): - # We don't currently support auto-config for VM-based containers if platform.system() == "Darwin" or ( platform.system() == "Windows" and WindowsUtils.isWindowsServer() == False diff --git a/ue4docker/test.py b/ue4docker/test.py index 100bf0ff..8e410e39 100644 --- a/ue4docker/test.py +++ b/ue4docker/test.py @@ -4,7 +4,6 @@ def test(): - # Create our logger to generate coloured output on stderr logger = Logger(prefix="[{} test] ".format(sys.argv[0])) @@ -13,7 +12,6 @@ def test(): # Check that an image tag has been specified if len(sys.argv) > 1 and sys.argv[1].strip("-") not in ["h", "help"]: - # Verify that the specified container image exists tag = sys.argv[1] image = GlobalConfiguration.resolveTag( @@ -37,7 +35,6 @@ def test(): ) container = ContainerUtils.start_for_exec(client, image, isolation=isolation) with ContainerUtils.automatically_stop(container): - # Create the workspace directory in the container workspaceDir = ContainerUtils.workspace_dir(container) ContainerUtils.exec( @@ -78,7 +75,6 @@ def runTest(script): logger.action("All tests passed.", False) else: - # Print usage syntax print("Usage: {} test TAG".format(sys.argv[0])) print("Runs tests to verify the correctness of built container images\n") diff --git a/ue4docker/tests/build-and-package.py b/ue4docker/tests/build-and-package.py index 3871ee15..82046684 100644 --- a/ue4docker/tests/build-and-package.py +++ b/ue4docker/tests/build-and-package.py @@ -14,7 +14,6 @@ def run(command, **kwargs): # Create an auto-deleting temporary directory to work in with tempfile.TemporaryDirectory() as tempDir: - # Clone a simple C++ project and verify that we can build and package it repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) projectDir = os.path.join(tempDir, "BasicCxx") diff --git a/ue4docker/tests/consume-external-deps.py b/ue4docker/tests/consume-external-deps.py index 2d093956..791c8d26 100644 --- a/ue4docker/tests/consume-external-deps.py +++ b/ue4docker/tests/consume-external-deps.py @@ -26,7 +26,6 @@ def write(filename, data): # Create an auto-deleting temporary directory to work in with tempfile.TemporaryDirectory() as tempDir: - # Clone a simple C++ project repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) projectDir = os.path.join(tempDir, "BasicCxx") From 80e68eb4590b0dc68e09f8ce1195414ff840c914 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 21 Feb 2023 10:46:13 +0300 Subject: [PATCH 331/430] resolves #290 use BuildKit secrets by default for Linux containers --- docs/advanced-build-options.adoc | 7 ++----- ue4docker/build.py | 4 ++-- ue4docker/infrastructure/BuildConfiguration.py | 8 +++++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc index 907e88a3..30c50dfc 100644 --- a/docs/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -124,9 +124,6 @@ You can also use the generated Dockerfiles to build container images independent - When building Windows container images, you must specify the <> `source_mode` and set it to `copy`. This generates Dockerfiles that copy the Unreal Engine source code from the host filesystem rather than cloning it from a git repository, thus eliminating the dependency on ue4-docker's credential endpoint to securely provide git credentials and allowing container images to be built without the need for ue4-docker itself. -- When building Linux container images, you must either set the <> `source_mode` to `copy` as detailed above, or else specify the `credential_mode` option and set it to `secrets`. -This generates Dockerfiles that use the Linux-only https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] functionality to securely provide git credentials, eliminating the dependency on ue4-docker's credential endpoint whilst still facilitating the use of a git repository to provide the Unreal Engine source code. - [[advanced-options-for-dockerfile-generation]] === Advanced options for Dockerfile generation @@ -147,10 +144,10 @@ The filesystem path can be specified using the `SOURCE_LOCATION` Docker build ar - **`credential_mode`**: *(string)* controls how the xref:available-container-images.adoc#ue4-source[ue4-source] Dockerfile securely obtains credentials for authenticating with remote git repositories when `source_mode` is set to `git`. Valid options are: -- `endpoint`: the default mode, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process by way of a Docker build argument. +- `endpoint`: the default mode for Windows Containers, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process by way of a Docker build argument. This mode will not work when <>, since the credential endpoint will not be available during the build process. -- `secrets`: **(Linux containers only)** uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process. +- `secrets`: **(Linux containers only)** default mode for Linux Containers, uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process. - **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image. diff --git a/ue4docker/build.py b/ue4docker/build.py index 975ce27c..19a9bf4c 100644 --- a/ue4docker/build.py +++ b/ue4docker/build.py @@ -411,13 +411,13 @@ def build(): # Build the UE4 source image if config.buildTargets["source"]: # Start the HTTP credential endpoint as a child process and wait for it to start - if config.opts.get("credential_mode", "endpoint") == "endpoint": + if config.opts["credential_mode"] == "endpoint": endpoint = CredentialEndpoint(username, password) endpoint.start() # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets secrets = {} - if config.opts.get("credential_mode", "endpoint") == "secrets": + if config.opts["credential_mode"] == "secrets": secrets = {"username": username, "password": password} credentialArgs = [] if len(secrets) > 0 else endpoint.args() diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py index 5cf231ab..e961c692 100644 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ b/ue4docker/infrastructure/BuildConfiguration.py @@ -504,13 +504,19 @@ def __init__(self, parser, argv, logger): ) ) + if "credential_mode" not in self.opts: + # On Linux, default to secrets mode that causes fewer issues with firewalls + self.opts = ( + "secrets" if self.containerPlatform == "linux" else "endpoint" + ) + # Verify that the value for `credential_mode` is valid if specified validCredentialModes = ( ["endpoint", "secrets"] if self.containerPlatform == "linux" else ["endpoint"] ) - if self.opts.get("credential_mode", "endpoint") not in validCredentialModes: + if self.opts["credential_mode"] not in validCredentialModes: raise RuntimeError( "invalid value specified for the `credential_mode` option, valid values are {} when building {} containers".format( validCredentialModes, self.containerPlatform.title() From 70596ab7da1468cd2b6e73e8fe3252afbe99ca41 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 1 Mar 2023 23:13:08 +0300 Subject: [PATCH 332/430] Bump version to 0.0.103 --- ue4docker/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ue4docker/version.py b/ue4docker/version.py index 945bce51..90d621e4 100644 --- a/ue4docker/version.py +++ b/ue4docker/version.py @@ -1 +1 @@ -__version__ = "0.0.102" +__version__ = "0.0.103" From f66831702365936878d44e8b33594bb93da8a595 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 04:53:53 -0800 Subject: [PATCH 333/430] Migrate from `setup.py` to `pyproject.toml` (#297) `pyproject.toml` is a mainstream (as of today) way to organize Python packages. There are multiple reasons for that, I'll just leave some links: * https://peps.python.org/pep-0517/ * https://peps.python.org/pep-0518/ * https://peps.python.org/pep-0621/ * https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html I've also took liberty to add myself to authors list in metadata, hopefully 21% of commits justifies this. Unfortunately, this stuff doesn't properly work with default pip/setuptools shipped with Python 3.6, so it has to be dropped. --- .github/workflows/ci.yml | 18 ++--- pyproject.toml | 56 +++++++++++++++ setup.py | 70 ------------------- {ue4docker => src/ue4docker}/__init__.py | 0 {ue4docker => src/ue4docker}/__main__.py | 0 {ue4docker => src/ue4docker}/build.py | 0 {ue4docker => src/ue4docker}/clean.py | 0 .../ue4docker}/diagnostics/__init__.py | 0 .../ue4docker}/diagnostics/base.py | 0 .../diagnostics/diagnostic_20gig.py | 0 .../ue4docker}/diagnostics/diagnostic_8gig.py | 0 .../ue4docker}/diagnostics/diagnostic_all.py | 0 .../diagnostics/diagnostic_network.py | 0 .../ue4docker}/diagnostics_cmd.py | 0 .../diagnostics/20gig/windows/Dockerfile | 0 .../diagnostics/8gig/linux/Dockerfile | 0 .../diagnostics/8gig/windows/Dockerfile | 0 .../diagnostics/8gig/windows/test.ps1 | 0 .../diagnostics/network/linux/Dockerfile | 0 .../diagnostics/network/windows/Dockerfile | 0 .../ue4-build-prerequisites/linux/Dockerfile | 0 .../windows/.dockerignore | 0 .../windows/.gitignore | 0 .../windows/Dockerfile | 0 .../windows/install-prerequisites.ps1 | 0 .../windows/remove-duplicate-dlls.ps1 | 0 .../dockerfiles/ue4-full/linux/Dockerfile | 0 .../ue4-full/linux/pulseaudio-client.conf | 0 .../dockerfiles/ue4-full/windows/Dockerfile | 0 .../ue4-minimal/linux/.dockerignore | 0 .../dockerfiles/ue4-minimal/linux/Dockerfile | 0 .../ue4-minimal/linux/copy-toolchain.py | 0 .../ue4-minimal/linux/enable-opengl.py | 0 .../ue4-minimal/linux/fix-targets.py | 0 .../ue4-minimal/linux/patch-build-graph.py | 0 .../ue4-minimal/linux/patch-filters-xml.py | 0 .../ue4-minimal/linux/set-changelist.py | 0 .../ue4-minimal/linux/split-components.py | 0 .../ue4-minimal/windows/.dockerignore | 0 .../ue4-minimal/windows/Dockerfile | 0 .../ue4-minimal/windows/fix-targets.py | 0 .../ue4-minimal/windows/patch-build-graph.py | 0 .../ue4-minimal/windows/patch-filters-xml.py | 0 .../ue4-minimal/windows/set-changelist.py | 0 .../ue4-minimal/windows/split-components.py | 0 .../ue4-source/linux/.dockerignore | 0 .../dockerfiles/ue4-source/linux/Dockerfile | 0 .../linux/git-credential-helper-endpoint.sh | 0 .../linux/git-credential-helper-secrets.sh | 0 .../ue4-source/linux/linker-fixup.py | 0 .../ue4-source/linux/patch-broken-releases.py | 0 .../dockerfiles/ue4-source/linux/patch-ubt.py | 0 .../ue4-source/windows/.dockerignore | 0 .../dockerfiles/ue4-source/windows/Dockerfile | 0 .../windows/git-credential-helper.bat | 0 .../windows/patch-broken-releases.py | 0 .../ue4-source/windows/patch-setup-win.py | 0 .../ue4-source/windows/patch-ubt.py | 0 {ue4docker => src/ue4docker}/export.py | 0 .../ue4docker}/exports/__init__.py | 0 .../ue4docker}/exports/export_installed.py | 0 .../ue4docker}/exports/export_packages.py | 0 {ue4docker => src/ue4docker}/info.py | 0 .../infrastructure/BuildConfiguration.py | 0 .../infrastructure/CredentialEndpoint.py | 0 .../ue4docker}/infrastructure/DarwinUtils.py | 0 .../ue4docker}/infrastructure/DockerUtils.py | 0 .../infrastructure/FilesystemUtils.py | 0 .../infrastructure/GlobalConfiguration.py | 0 .../ue4docker}/infrastructure/ImageBuilder.py | 0 .../ue4docker}/infrastructure/ImageCleaner.py | 0 .../ue4docker}/infrastructure/Logger.py | 0 .../ue4docker}/infrastructure/NetworkUtils.py | 0 .../ue4docker}/infrastructure/PackageUtils.py | 0 .../infrastructure/PrettyPrinting.py | 0 .../infrastructure/ResourceMonitor.py | 0 .../infrastructure/SubprocessUtils.py | 2 +- .../ue4docker}/infrastructure/WindowsUtils.py | 0 .../ue4docker}/infrastructure/__init__.py | 0 {ue4docker => src/ue4docker}/main.py | 0 {ue4docker => src/ue4docker}/setup_cmd.py | 0 {ue4docker => src/ue4docker}/test.py | 0 .../ue4docker}/tests/build-and-package.py | 0 .../ue4docker}/tests/consume-external-deps.py | 0 src/ue4docker/version.py | 8 +++ {ue4docker => src/ue4docker}/version_cmd.py | 0 ue4docker/version.py | 1 - 87 files changed, 74 insertions(+), 81 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py rename {ue4docker => src/ue4docker}/__init__.py (100%) rename {ue4docker => src/ue4docker}/__main__.py (100%) rename {ue4docker => src/ue4docker}/build.py (100%) rename {ue4docker => src/ue4docker}/clean.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/__init__.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/base.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/diagnostic_20gig.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/diagnostic_8gig.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/diagnostic_all.py (100%) rename {ue4docker => src/ue4docker}/diagnostics/diagnostic_network.py (100%) rename {ue4docker => src/ue4docker}/diagnostics_cmd.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/20gig/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/8gig/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/8gig/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/8gig/windows/test.ps1 (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/network/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/diagnostics/network/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/windows/.dockerignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/windows/.gitignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-full/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-full/linux/pulseaudio-client.conf (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-full/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/.dockerignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/copy-toolchain.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/enable-opengl.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/fix-targets.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/patch-build-graph.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/patch-filters-xml.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/set-changelist.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/linux/split-components.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/.dockerignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/fix-targets.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/patch-build-graph.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/patch-filters-xml.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/set-changelist.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-minimal/windows/split-components.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/.dockerignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/linker-fixup.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/patch-broken-releases.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/linux/patch-ubt.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/.dockerignore (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/Dockerfile (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/git-credential-helper.bat (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/patch-broken-releases.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/patch-setup-win.py (100%) rename {ue4docker => src/ue4docker}/dockerfiles/ue4-source/windows/patch-ubt.py (100%) rename {ue4docker => src/ue4docker}/export.py (100%) rename {ue4docker => src/ue4docker}/exports/__init__.py (100%) rename {ue4docker => src/ue4docker}/exports/export_installed.py (100%) rename {ue4docker => src/ue4docker}/exports/export_packages.py (100%) rename {ue4docker => src/ue4docker}/info.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/BuildConfiguration.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/CredentialEndpoint.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/DarwinUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/DockerUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/FilesystemUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/GlobalConfiguration.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/ImageBuilder.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/ImageCleaner.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/Logger.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/NetworkUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/PackageUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/PrettyPrinting.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/ResourceMonitor.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/SubprocessUtils.py (98%) rename {ue4docker => src/ue4docker}/infrastructure/WindowsUtils.py (100%) rename {ue4docker => src/ue4docker}/infrastructure/__init__.py (100%) rename {ue4docker => src/ue4docker}/main.py (100%) rename {ue4docker => src/ue4docker}/setup_cmd.py (100%) rename {ue4docker => src/ue4docker}/test.py (100%) rename {ue4docker => src/ue4docker}/tests/build-and-package.py (100%) rename {ue4docker => src/ue4docker}/tests/consume-external-deps.py (100%) create mode 100644 src/ue4docker/version.py rename {ue4docker => src/ue4docker}/version_cmd.py (100%) delete mode 100644 ue4docker/version.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 97c2aab2..62387d4e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,9 @@ jobs: with: python-version: "3.10" - name: Install dependencies - run: pip install setuptools wheel + run: pip install --upgrade build - name: Build package - run: python setup.py sdist bdist_wheel + run: python -m build - name: Publish to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 @@ -29,20 +29,20 @@ jobs: strategy: matrix: include: - - { os: ubuntu-20.04, python: "3.6" } + - { os: ubuntu-20.04, python: "3.7" } - { os: ubuntu-20.04, python: "3.10" } - { os: ubuntu-22.04, python: "3.10" } - - { os: windows-2019, python: "3.6", visual-studio: 2017 } + - { os: windows-2019, python: "3.7", visual-studio: 2017 } - { os: windows-2019, python: "3.10", visual-studio: 2017 } - - { os: windows-2019, python: "3.6", visual-studio: 2019 } + - { os: windows-2019, python: "3.7", visual-studio: 2019 } - { os: windows-2019, python: "3.10", visual-studio: 2019 } - - { os: windows-2019, python: "3.6", visual-studio: 2022 } + - { os: windows-2019, python: "3.7", visual-studio: 2022 } - { os: windows-2019, python: "3.10", visual-studio: 2022 } - - { os: windows-latest, python: "3.6", visual-studio: 2017 } + - { os: windows-latest, python: "3.7", visual-studio: 2017 } - { os: windows-latest, python: "3.10", visual-studio: 2017 } - - { os: windows-latest, python: "3.6", visual-studio: 2019 } + - { os: windows-latest, python: "3.7", visual-studio: 2019 } - { os: windows-latest, python: "3.10", visual-studio: 2019 } - - { os: windows-latest, python: "3.6", visual-studio: 2022 } + - { os: windows-latest, python: "3.7", visual-studio: 2022 } - { os: windows-latest, python: "3.10", visual-studio: 2022 } runs-on: ${{ matrix.os }} steps: diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..518e3e91 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[project] +name = "ue4-docker" +version = "0.0.103" +description = "Windows and Linux containers for Unreal Engine" +requires-python = ">= 3.7" +license = { file = "LICENSE" } +authors = [ + { name = "Adam Rehn", email = "adam@adamrehn.com" }, + { name = "Marat Radchenko", email = "marat@slonopotamus.org" }, +] +keywords = [ + "unreal engine", + "docker", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Build Tools", +] +dependencies = [ + "colorama", + "container-utils", + "docker>=3.0.0", + "humanfriendly", + "importlib-metadata>=1.0;python_version<'3.8'", + "Jinja2>=2.11.3", + "packaging>=19.1", + "psutil", + "requests", + "semver>=2.7.9,<3.0.0", + "termcolor", +] + +[project.urls] +homepage = "https://github.com/adamrehn/ue4-docker" + +[project.scripts] +ue4-docker = "ue4docker:main" + +[build-system] +requires = [ + "setuptools>=61", + "wheel", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-data] +ue4docker = [ + "dockerfiles/*/*/.dockerignore", + "dockerfiles/diagnostics/*/*/*", + "dockerfiles/*/*/*", + "tests/*.py", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index ef9365ff..00000000 --- a/setup.py +++ /dev/null @@ -1,70 +0,0 @@ -from os.path import abspath, dirname, join -from setuptools import setup - -# Read the README markdown data from README.adoc -with open(abspath(join(dirname(__file__), "README.adoc")), "rb") as readmeFile: - __readme__ = readmeFile.read().decode("utf-8") - -# Read the version number from version.py -with open(abspath(join(dirname(__file__), "ue4docker", "version.py"))) as versionFile: - __version__ = ( - versionFile.read().strip().replace("__version__ = ", "").replace('"', "") - ) - -setup( - name="ue4-docker", - version=__version__, - description="Windows and Linux containers for Unreal Engine 4", - long_description=__readme__, - long_description_content_type="text/markdown", - classifiers=[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Software Development :: Build Tools", - "Environment :: Console", - ], - keywords="epic unreal engine docker", - url="http://github.com/adamrehn/ue4-docker", - author="Adam Rehn", - author_email="adam@adamrehn.com", - license="MIT", - packages=[ - "ue4docker", - "ue4docker.diagnostics", - "ue4docker.exports", - "ue4docker.infrastructure", - ], - zip_safe=False, - python_requires=">=3.6", - install_requires=[ - "colorama", - "container-utils", - "docker>=3.0.0", - "humanfriendly", - "Jinja2>=2.11.3", - "packaging>=19.1", - "psutil", - "requests", - "semver>=2.7.9,<3.0.0", - # We want newer setuptools to avoid shim path bug on newer Pythons. - # See #231 and https://github.com/pypa/setuptools/issues/3001 - # Unfortunately, that version of setuptools doesn't have Python 3.6 support - "setuptools>=60.4.0;python_version>='3.7.0'", - "termcolor", - "twine>=1.11.0", - "wheel>=0.31.0", - ], - package_data={ - "ue4docker": [ - "dockerfiles/*/*/.dockerignore", - "dockerfiles/diagnostics/*/*/*", - "dockerfiles/*/*/*", - "tests/*.py", - ] - }, - entry_points={"console_scripts": ["ue4-docker=ue4docker:main"]}, -) diff --git a/ue4docker/__init__.py b/src/ue4docker/__init__.py similarity index 100% rename from ue4docker/__init__.py rename to src/ue4docker/__init__.py diff --git a/ue4docker/__main__.py b/src/ue4docker/__main__.py similarity index 100% rename from ue4docker/__main__.py rename to src/ue4docker/__main__.py diff --git a/ue4docker/build.py b/src/ue4docker/build.py similarity index 100% rename from ue4docker/build.py rename to src/ue4docker/build.py diff --git a/ue4docker/clean.py b/src/ue4docker/clean.py similarity index 100% rename from ue4docker/clean.py rename to src/ue4docker/clean.py diff --git a/ue4docker/diagnostics/__init__.py b/src/ue4docker/diagnostics/__init__.py similarity index 100% rename from ue4docker/diagnostics/__init__.py rename to src/ue4docker/diagnostics/__init__.py diff --git a/ue4docker/diagnostics/base.py b/src/ue4docker/diagnostics/base.py similarity index 100% rename from ue4docker/diagnostics/base.py rename to src/ue4docker/diagnostics/base.py diff --git a/ue4docker/diagnostics/diagnostic_20gig.py b/src/ue4docker/diagnostics/diagnostic_20gig.py similarity index 100% rename from ue4docker/diagnostics/diagnostic_20gig.py rename to src/ue4docker/diagnostics/diagnostic_20gig.py diff --git a/ue4docker/diagnostics/diagnostic_8gig.py b/src/ue4docker/diagnostics/diagnostic_8gig.py similarity index 100% rename from ue4docker/diagnostics/diagnostic_8gig.py rename to src/ue4docker/diagnostics/diagnostic_8gig.py diff --git a/ue4docker/diagnostics/diagnostic_all.py b/src/ue4docker/diagnostics/diagnostic_all.py similarity index 100% rename from ue4docker/diagnostics/diagnostic_all.py rename to src/ue4docker/diagnostics/diagnostic_all.py diff --git a/ue4docker/diagnostics/diagnostic_network.py b/src/ue4docker/diagnostics/diagnostic_network.py similarity index 100% rename from ue4docker/diagnostics/diagnostic_network.py rename to src/ue4docker/diagnostics/diagnostic_network.py diff --git a/ue4docker/diagnostics_cmd.py b/src/ue4docker/diagnostics_cmd.py similarity index 100% rename from ue4docker/diagnostics_cmd.py rename to src/ue4docker/diagnostics_cmd.py diff --git a/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile rename to src/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile diff --git a/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile rename to src/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile diff --git a/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile rename to src/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile diff --git a/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 similarity index 100% rename from ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 rename to src/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 diff --git a/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile rename to src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile diff --git a/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile rename to src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-full/linux/Dockerfile rename to src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf b/src/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf similarity index 100% rename from ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf rename to src/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-full/windows/Dockerfile rename to src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore b/src/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore rename to src/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile rename to src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/split-components.py rename to src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile rename to src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py rename to src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py rename to src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py rename to src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py rename to src/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/split-components.py rename to src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py diff --git a/ue4docker/dockerfiles/ue4-source/linux/.dockerignore b/src/ue4docker/dockerfiles/ue4-source/linux/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/.dockerignore rename to src/ue4docker/dockerfiles/ue4-source/linux/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/Dockerfile rename to src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh b/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh rename to src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh diff --git a/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh b/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh rename to src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh diff --git a/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py b/src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py rename to src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py rename to src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py rename to src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py diff --git a/ue4docker/dockerfiles/ue4-source/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-source/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-source/windows/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/Dockerfile rename to src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile diff --git a/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat b/src/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat rename to src/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py rename to src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py rename to src/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py rename to src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py diff --git a/ue4docker/export.py b/src/ue4docker/export.py similarity index 100% rename from ue4docker/export.py rename to src/ue4docker/export.py diff --git a/ue4docker/exports/__init__.py b/src/ue4docker/exports/__init__.py similarity index 100% rename from ue4docker/exports/__init__.py rename to src/ue4docker/exports/__init__.py diff --git a/ue4docker/exports/export_installed.py b/src/ue4docker/exports/export_installed.py similarity index 100% rename from ue4docker/exports/export_installed.py rename to src/ue4docker/exports/export_installed.py diff --git a/ue4docker/exports/export_packages.py b/src/ue4docker/exports/export_packages.py similarity index 100% rename from ue4docker/exports/export_packages.py rename to src/ue4docker/exports/export_packages.py diff --git a/ue4docker/info.py b/src/ue4docker/info.py similarity index 100% rename from ue4docker/info.py rename to src/ue4docker/info.py diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py similarity index 100% rename from ue4docker/infrastructure/BuildConfiguration.py rename to src/ue4docker/infrastructure/BuildConfiguration.py diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/src/ue4docker/infrastructure/CredentialEndpoint.py similarity index 100% rename from ue4docker/infrastructure/CredentialEndpoint.py rename to src/ue4docker/infrastructure/CredentialEndpoint.py diff --git a/ue4docker/infrastructure/DarwinUtils.py b/src/ue4docker/infrastructure/DarwinUtils.py similarity index 100% rename from ue4docker/infrastructure/DarwinUtils.py rename to src/ue4docker/infrastructure/DarwinUtils.py diff --git a/ue4docker/infrastructure/DockerUtils.py b/src/ue4docker/infrastructure/DockerUtils.py similarity index 100% rename from ue4docker/infrastructure/DockerUtils.py rename to src/ue4docker/infrastructure/DockerUtils.py diff --git a/ue4docker/infrastructure/FilesystemUtils.py b/src/ue4docker/infrastructure/FilesystemUtils.py similarity index 100% rename from ue4docker/infrastructure/FilesystemUtils.py rename to src/ue4docker/infrastructure/FilesystemUtils.py diff --git a/ue4docker/infrastructure/GlobalConfiguration.py b/src/ue4docker/infrastructure/GlobalConfiguration.py similarity index 100% rename from ue4docker/infrastructure/GlobalConfiguration.py rename to src/ue4docker/infrastructure/GlobalConfiguration.py diff --git a/ue4docker/infrastructure/ImageBuilder.py b/src/ue4docker/infrastructure/ImageBuilder.py similarity index 100% rename from ue4docker/infrastructure/ImageBuilder.py rename to src/ue4docker/infrastructure/ImageBuilder.py diff --git a/ue4docker/infrastructure/ImageCleaner.py b/src/ue4docker/infrastructure/ImageCleaner.py similarity index 100% rename from ue4docker/infrastructure/ImageCleaner.py rename to src/ue4docker/infrastructure/ImageCleaner.py diff --git a/ue4docker/infrastructure/Logger.py b/src/ue4docker/infrastructure/Logger.py similarity index 100% rename from ue4docker/infrastructure/Logger.py rename to src/ue4docker/infrastructure/Logger.py diff --git a/ue4docker/infrastructure/NetworkUtils.py b/src/ue4docker/infrastructure/NetworkUtils.py similarity index 100% rename from ue4docker/infrastructure/NetworkUtils.py rename to src/ue4docker/infrastructure/NetworkUtils.py diff --git a/ue4docker/infrastructure/PackageUtils.py b/src/ue4docker/infrastructure/PackageUtils.py similarity index 100% rename from ue4docker/infrastructure/PackageUtils.py rename to src/ue4docker/infrastructure/PackageUtils.py diff --git a/ue4docker/infrastructure/PrettyPrinting.py b/src/ue4docker/infrastructure/PrettyPrinting.py similarity index 100% rename from ue4docker/infrastructure/PrettyPrinting.py rename to src/ue4docker/infrastructure/PrettyPrinting.py diff --git a/ue4docker/infrastructure/ResourceMonitor.py b/src/ue4docker/infrastructure/ResourceMonitor.py similarity index 100% rename from ue4docker/infrastructure/ResourceMonitor.py rename to src/ue4docker/infrastructure/ResourceMonitor.py diff --git a/ue4docker/infrastructure/SubprocessUtils.py b/src/ue4docker/infrastructure/SubprocessUtils.py similarity index 98% rename from ue4docker/infrastructure/SubprocessUtils.py rename to src/ue4docker/infrastructure/SubprocessUtils.py index 429b6ea5..6dedf0e8 100644 --- a/ue4docker/infrastructure/SubprocessUtils.py +++ b/src/ue4docker/infrastructure/SubprocessUtils.py @@ -36,7 +36,7 @@ def capture(command, check=True, **kwargs): stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=check, - **kwargs + **kwargs, ) except subprocess.CalledProcessError as e: raise VerboseCalledProcessError(e) from None diff --git a/ue4docker/infrastructure/WindowsUtils.py b/src/ue4docker/infrastructure/WindowsUtils.py similarity index 100% rename from ue4docker/infrastructure/WindowsUtils.py rename to src/ue4docker/infrastructure/WindowsUtils.py diff --git a/ue4docker/infrastructure/__init__.py b/src/ue4docker/infrastructure/__init__.py similarity index 100% rename from ue4docker/infrastructure/__init__.py rename to src/ue4docker/infrastructure/__init__.py diff --git a/ue4docker/main.py b/src/ue4docker/main.py similarity index 100% rename from ue4docker/main.py rename to src/ue4docker/main.py diff --git a/ue4docker/setup_cmd.py b/src/ue4docker/setup_cmd.py similarity index 100% rename from ue4docker/setup_cmd.py rename to src/ue4docker/setup_cmd.py diff --git a/ue4docker/test.py b/src/ue4docker/test.py similarity index 100% rename from ue4docker/test.py rename to src/ue4docker/test.py diff --git a/ue4docker/tests/build-and-package.py b/src/ue4docker/tests/build-and-package.py similarity index 100% rename from ue4docker/tests/build-and-package.py rename to src/ue4docker/tests/build-and-package.py diff --git a/ue4docker/tests/consume-external-deps.py b/src/ue4docker/tests/consume-external-deps.py similarity index 100% rename from ue4docker/tests/consume-external-deps.py rename to src/ue4docker/tests/consume-external-deps.py diff --git a/src/ue4docker/version.py b/src/ue4docker/version.py new file mode 100644 index 00000000..33ec3a65 --- /dev/null +++ b/src/ue4docker/version.py @@ -0,0 +1,8 @@ +import sys + +if sys.version_info >= (3, 8): + from importlib import metadata +else: + import importlib_metadata as metadata + +__version__ = metadata.version("ue4-docker") diff --git a/ue4docker/version_cmd.py b/src/ue4docker/version_cmd.py similarity index 100% rename from ue4docker/version_cmd.py rename to src/ue4docker/version_cmd.py diff --git a/ue4docker/version.py b/ue4docker/version.py deleted file mode 100644 index 90d621e4..00000000 --- a/ue4docker/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "0.0.103" From ce89fcd226e2e43fd8b29df68de35e4573c65df7 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 05:24:44 -0800 Subject: [PATCH 334/430] Drop dependency on `semver` package (#298) 1. We already use `packaging` because it is less strict 2. This allows to drop hacks around `node-semver` --- pyproject.toml | 1 - .../infrastructure/BuildConfiguration.py | 53 ++++++++----------- src/ue4docker/infrastructure/DarwinUtils.py | 17 +++--- .../infrastructure/GlobalConfiguration.py | 7 +-- src/ue4docker/infrastructure/PackageUtils.py | 20 ------- src/ue4docker/infrastructure/WindowsUtils.py | 9 ++-- src/ue4docker/infrastructure/__init__.py | 1 - 7 files changed, 37 insertions(+), 71 deletions(-) delete mode 100644 src/ue4docker/infrastructure/PackageUtils.py diff --git a/pyproject.toml b/pyproject.toml index 518e3e91..0a82ca0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ "packaging>=19.1", "psutil", "requests", - "semver>=2.7.9,<3.0.0", "termcolor", ] diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index e961c692..eb884d6a 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -1,13 +1,12 @@ +import json +import platform +import random + +import humanfriendly +from packaging.version import Version, InvalidVersion + from .DockerUtils import DockerUtils -from .PackageUtils import PackageUtils from .WindowsUtils import WindowsUtils -import humanfriendly, json, os, platform, random -from pkg_resources import parse_version - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile( - "semver", os.path.join(PackageUtils.getPackageLocation("semver"), "semver.py") -) # The default Unreal Engine git repository DEFAULT_GIT_REPO = "https://github.com/EpicGames/UnrealEngine.git" @@ -55,17 +54,17 @@ class VisualStudio(object): SupportedSince = { # We do not support versions older than 4.20 - VS2017: semver.VersionInfo(4.20), + VS2017: Version("4.20"), # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa # # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 - VS2019: semver.VersionInfo(4, 25), - VS2022: semver.VersionInfo(5, 0, 0), + VS2019: Version("4.25"), + VS2022: Version("5.0.0"), } - UnsupportedSince = {VS2017: semver.VersionInfo(5, 0)} + UnsupportedSince = {VS2017: Version("5.0")} class ExcludedComponent(object): @@ -406,16 +405,13 @@ def __init__(self, parser, argv, logger): else: # Validate the specified version string try: - ue4Version = semver.parse(self.args.release) - if ( - ue4Version["major"] not in [4, 5] - or ue4Version["prerelease"] != None - ): + ue4Version = Version(self.args.release) + if ue4Version.major not in [4, 5] or ue4Version.pre != 0: raise Exception() - self.release = semver.format_version( - ue4Version["major"], ue4Version["minor"], ue4Version["patch"] + self.release = ( + f"{ue4Version.major}.{ue4Version.minor}.{ue4Version.micro}" ) - except: + except InvalidVersion: raise RuntimeError( 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.20.0")'.format( self.args.release @@ -545,11 +541,7 @@ def __init__(self, parser, argv, logger): if ExcludedComponent.Debug not in self.excludedComponents: logger.warning("Warning: You didn't pass --exclude debug", False) warn20GiB = True - if ( - self.release - and not self.custom - and semver.VersionInfo.parse(self.release) >= semver.VersionInfo(5, 0) - ): + if self.release and not self.custom and Version(self.release).major >= 5: logger.warning("Warning: You're building Unreal Engine 5", False) warn20GiB = True @@ -592,10 +584,7 @@ def _generateWindowsConfig(self): if self.release is not None and not self.custom: # Check whether specified Unreal Engine release is compatible with specified Visual Studio supportedSince = VisualStudio.SupportedSince.get(self.visualStudio, None) - if ( - supportedSince is not None - and semver.VersionInfo.parse(self.release) < supportedSince - ): + if supportedSince is not None and Version(self.release) < supportedSince: raise RuntimeError( "specified version of Unreal Engine is too old for Visual Studio {}".format( self.visualStudio @@ -607,7 +596,7 @@ def _generateWindowsConfig(self): ) if ( unsupportedSince is not None - and semver.VersionInfo.parse(self.release) >= unsupportedSince + and Version(self.release) >= unsupportedSince ): raise RuntimeError( "Visual Studio {} is too old for specified version of Unreal Engine".format( @@ -648,9 +637,9 @@ def _generateWindowsConfig(self): else: # If we are able to use process isolation mode then use it, otherwise use Hyper-V isolation mode differentKernels = self.basetag != self.hostBasetag - dockerSupportsProcess = parse_version( + dockerSupportsProcess = Version( DockerUtils.version()["Version"] - ) >= parse_version("18.09.0") + ) >= Version("18.09.0") if not differentKernels and dockerSupportsProcess: self.isolation = "process" else: diff --git a/src/ue4docker/infrastructure/DarwinUtils.py b/src/ue4docker/infrastructure/DarwinUtils.py index bd9ec61f..7fdf415a 100644 --- a/src/ue4docker/infrastructure/DarwinUtils.py +++ b/src/ue4docker/infrastructure/DarwinUtils.py @@ -1,17 +1,18 @@ -from packaging import version -import os, platform +import platform + +from packaging.version import Version class DarwinUtils(object): @staticmethod - def minimumRequiredVersion(): + def minimumRequiredVersion() -> Version: """ Returns the minimum required version of macOS, which is 10.10.3 Yosemite (10.10.3 is the minimum required version for Docker for Mac, as per: ) """ - return "10.10.3" + return Version("10.10.3") @staticmethod def systemString(): @@ -23,17 +24,15 @@ def systemString(): ) @staticmethod - def getMacOsVersion(): + def getMacOsVersion() -> Version: """ Returns the version number for the macOS host system """ - return platform.mac_ver()[0] + return Version(platform.mac_ver()[0]) @staticmethod def isSupportedMacOsVersion(): """ Verifies that the macOS host system meets our minimum version requirements """ - return version.parse(DarwinUtils.getMacOsVersion()) >= version.parse( - DarwinUtils.minimumRequiredVersion() - ) + return DarwinUtils.getMacOsVersion() >= DarwinUtils.minimumRequiredVersion() diff --git a/src/ue4docker/infrastructure/GlobalConfiguration.py b/src/ue4docker/infrastructure/GlobalConfiguration.py index 58e9b86f..9b3cda77 100644 --- a/src/ue4docker/infrastructure/GlobalConfiguration.py +++ b/src/ue4docker/infrastructure/GlobalConfiguration.py @@ -1,6 +1,7 @@ -from pkg_resources import parse_version -import os, requests +import os +import requests +from packaging.version import Version # The default namespace for our tagged container images DEFAULT_TAG_NAMESPACE = "adamrehn" @@ -17,7 +18,7 @@ def getLatestVersion(): Queries PyPI to determine the latest available release of ue4-docker """ releases = [ - parse_version(release) + Version(release) for release in requests.get("https://pypi.org/pypi/ue4-docker/json").json()[ "releases" ] diff --git a/src/ue4docker/infrastructure/PackageUtils.py b/src/ue4docker/infrastructure/PackageUtils.py deleted file mode 100644 index a63a9cdd..00000000 --- a/src/ue4docker/infrastructure/PackageUtils.py +++ /dev/null @@ -1,20 +0,0 @@ -import importlib.util, pkg_resources - - -class PackageUtils(object): - @staticmethod - def getPackageLocation(package): - """ - Attempts to retrieve the filesystem location for the specified Python package - """ - return pkg_resources.get_distribution(package).location - - @staticmethod - def importFile(moduleName, filePath): - """ - Directly imports a Python module from a source file - """ - spec = importlib.util.spec_from_file_location(moduleName, filePath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module diff --git a/src/ue4docker/infrastructure/WindowsUtils.py b/src/ue4docker/infrastructure/WindowsUtils.py index 8961dae4..9f0f1d96 100644 --- a/src/ue4docker/infrastructure/WindowsUtils.py +++ b/src/ue4docker/infrastructure/WindowsUtils.py @@ -1,5 +1,5 @@ from .DockerUtils import DockerUtils -from pkg_resources import parse_version +from packaging.version import Version import platform, sys from typing import Optional @@ -94,11 +94,10 @@ def isBlacklistedWindowsHost() -> bool: Determines if host Windows version is one with bugs that make it unsuitable for use (defaults to checking the host OS release if one is not specified) """ - dockerVersion = parse_version(DockerUtils.version()["Version"]) + dockerVersion = Version(DockerUtils.version()["Version"]) build = WindowsUtils.getWindowsBuild() - return ( - build in WindowsUtils._blacklistedHosts - and dockerVersion < parse_version("19.03.6") + return build in WindowsUtils._blacklistedHosts and dockerVersion < Version( + "19.03.6" ) @staticmethod diff --git a/src/ue4docker/infrastructure/__init__.py b/src/ue4docker/infrastructure/__init__.py index 7ffdb9c5..5e45048e 100644 --- a/src/ue4docker/infrastructure/__init__.py +++ b/src/ue4docker/infrastructure/__init__.py @@ -8,7 +8,6 @@ from .ImageCleaner import ImageCleaner from .Logger import Logger from .NetworkUtils import NetworkUtils -from .PackageUtils import PackageUtils from .PrettyPrinting import PrettyPrinting from .ResourceMonitor import ResourceMonitor from .SubprocessUtils import SubprocessUtils From 6a2a2c398df2298ba98759cdc17dc9598f5861c6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 05:37:26 -0800 Subject: [PATCH 335/430] Drop dependency on `requests` package (#299) Python provides enough batteries to parse JSON from URL out of the box --- pyproject.toml | 1 - .../infrastructure/GlobalConfiguration.py | 14 ++++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0a82ca0e..5ae2c877 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ dependencies = [ "Jinja2>=2.11.3", "packaging>=19.1", "psutil", - "requests", "termcolor", ] diff --git a/src/ue4docker/infrastructure/GlobalConfiguration.py b/src/ue4docker/infrastructure/GlobalConfiguration.py index 9b3cda77..fbb31ac1 100644 --- a/src/ue4docker/infrastructure/GlobalConfiguration.py +++ b/src/ue4docker/infrastructure/GlobalConfiguration.py @@ -1,7 +1,8 @@ import os -import requests from packaging.version import Version +from urllib.request import urlopen +import json # The default namespace for our tagged container images DEFAULT_TAG_NAMESPACE = "adamrehn" @@ -17,13 +18,10 @@ def getLatestVersion(): """ Queries PyPI to determine the latest available release of ue4-docker """ - releases = [ - Version(release) - for release in requests.get("https://pypi.org/pypi/ue4-docker/json").json()[ - "releases" - ] - ] - return sorted(releases)[-1] + with urlopen("https://pypi.org/pypi/ue4-docker/json") as url: + data = json.load(url) + releases = [Version(release) for release in data["releases"]] + return sorted(releases)[-1] @staticmethod def getTagNamespace(): From b3795be3227b8af684625af06b0d0aafb7c29164 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 18:50:23 +0300 Subject: [PATCH 336/430] Remove redundant `tool.setuptools.package-data` from `pyproject.toml` --- pyproject.toml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5ae2c877..fe1bbc03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,11 +44,3 @@ requires = [ "wheel", ] build-backend = "setuptools.build_meta" - -[tool.setuptools.package-data] -ue4docker = [ - "dockerfiles/*/*/.dockerignore", - "dockerfiles/diagnostics/*/*/*", - "dockerfiles/*/*/*", - "tests/*.py", -] From 66d003e927cbb4d4d987c2d82073d006ece52a4b Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 19:01:35 +0300 Subject: [PATCH 337/430] Fix homepage metadata in `pyproject.toml` It seems like it needs to start with a capital letter. See https://packaging.python.org/en/latest/tutorials/packaging-projects/#configuring-metadata --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fe1bbc03..20b08272 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ ] [project.urls] -homepage = "https://github.com/adamrehn/ue4-docker" +"Homepage" = "https://github.com/adamrehn/ue4-docker" [project.scripts] ue4-docker = "ue4docker:main" From 6af529412f78c70836fbae7e8a86c11e1f661287 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 5 Mar 2023 19:06:55 +0300 Subject: [PATCH 338/430] Revert "Remove redundant `tool.setuptools.package-data` from `pyproject.toml`" This reverts commit b3795be3227b8af684625af06b0d0aafb7c29164. --- pyproject.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 20b08272..2d0556f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,3 +44,11 @@ requires = [ "wheel", ] build-backend = "setuptools.build_meta" + +[tool.setuptools.package-data] +ue4docker = [ + "dockerfiles/*/*/.dockerignore", + "dockerfiles/diagnostics/*/*/*", + "dockerfiles/*/*/*", + "tests/*.py", +] From a0e9b31258738c1ca742b3d7ec32802a655193a2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 9 Mar 2023 12:06:18 -0800 Subject: [PATCH 339/430] Bundle code from `container-utils` (#300) [container-utils](https://github.com/adamrehn/container-utils) 1. Isn't used by anyone except us 2. Isn't actively maintained 3. Forces `setuptools`/`wheel`/`twine` deps upon users 4. Overall, doesn't look like a valuable asset in itself This commit brings code from `container-utils` into ue4-docker repo and drops dependency on `container-utils`. --- pyproject.toml | 1 - .../infrastructure/ContainerUtils.py | 138 ++++++++++++++++++ src/ue4docker/infrastructure/DockerUtils.py | 4 +- src/ue4docker/infrastructure/__init__.py | 1 + src/ue4docker/test.py | 51 ++++--- 5 files changed, 175 insertions(+), 20 deletions(-) create mode 100644 src/ue4docker/infrastructure/ContainerUtils.py diff --git a/pyproject.toml b/pyproject.toml index 2d0556f5..e056f79e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ classifiers = [ ] dependencies = [ "colorama", - "container-utils", "docker>=3.0.0", "humanfriendly", "importlib-metadata>=1.0;python_version<'3.8'", diff --git a/src/ue4docker/infrastructure/ContainerUtils.py b/src/ue4docker/infrastructure/ContainerUtils.py new file mode 100644 index 00000000..14676c25 --- /dev/null +++ b/src/ue4docker/infrastructure/ContainerUtils.py @@ -0,0 +1,138 @@ +import contextlib +import io +import logging +import os +import shutil +import sys +import tempfile + +import docker +from docker.models.containers import Container + + +class ContainerUtils(object): + """ + Provides functionality related to Docker containers + """ + + @staticmethod + @contextlib.contextmanager + def automatically_stop(container: Container, timeout: int = 1): + """ + Context manager to automatically stop a container returned by `ContainerUtils.start_for_exec()` + """ + try: + yield container + finally: + logging.info("Stopping Docker container {}...".format(container.short_id)) + container.stop(timeout=timeout) + + @staticmethod + def copy_from_host( + container: Container, host_path: str, container_path: str + ) -> None: + """ + Copies a file or directory from the host system to a container returned by `ContainerUtils.start_for_exec()`. + + `host_path` is the absolute path to the file or directory on the host system. + + `container_path` is the absolute path to the directory in the container where the copied file(s) will be placed. + """ + + # If the host path denotes a file rather than a directory, copy it to a temporary directory + # (If the host path is a directory then we create a no-op context manager to use in our `with` statement below) + tempDir = contextlib.suppress() + if os.path.isfile(host_path): + tempDir = tempfile.TemporaryDirectory() + shutil.copy2( + host_path, os.path.join(tempDir.name, os.path.basename(host_path)) + ) + host_path = tempDir.name + + # Automatically delete the temporary directory if we created one + with tempDir: + # Create a temporary file to hold the .tar archive data + with tempfile.NamedTemporaryFile( + suffix=".tar", delete=False + ) as tempArchive: + # Add the data from the host system to the temporary archive + tempArchive.close() + archiveName = os.path.splitext(tempArchive.name)[0] + shutil.make_archive(archiveName, "tar", host_path) + + # Copy the data from the temporary archive to the container + with open(tempArchive.name, "rb") as archive: + container.put_archive(container_path, archive.read()) + + # Remove the temporary archive + os.unlink(tempArchive.name) + + @staticmethod + def exec(container: Container, command: [str], capture: bool = False, **kwargs): + """ + Executes a command in a container returned by `ContainerUtils.start_for_exec()` and streams or captures the output + """ + + # Determine if we are capturing the output or printing it + stdoutDest = io.StringIO() if capture else sys.stdout + stderrDest = io.StringIO() if capture else sys.stderr + + # Attempt to start the command + details = container.client.api.exec_create(container.id, command, **kwargs) + output = container.client.api.exec_start(details["Id"], stream=True, demux=True) + + # Stream the output + for chunk in output: + # Isolate the stdout and stderr chunks + stdout, stderr = chunk + + # Capture/print the stderr data if we have any + if stderr is not None: + print(stderr.decode("utf-8"), end="", flush=True, file=stderrDest) + + # Capture/print the stdout data if we have any + if stdout is not None: + print(stdout.decode("utf-8"), end="", flush=True, file=stdoutDest) + + # Determine if the command succeeded + capturedOutput = ( + (stdoutDest.getvalue(), stderrDest.getvalue()) if capture else None + ) + result = container.client.api.exec_inspect(details["Id"])["ExitCode"] + if result != 0: + container.stop() + raise RuntimeError( + "Failed to run command {} in container. Process returned exit code {} with output {}.".format( + command, + result, + capturedOutput if capture else "printed above", + ) + ) + + # If we captured the output then return it + return capturedOutput + + @staticmethod + def start_for_exec( + client: docker.DockerClient, image: str, platform: str, **kwargs + ) -> Container: + """ + Starts a container in a detached state using a command that will block indefinitely + and returns the container handle. The handle can then be used to execute commands + inside the container. The container will be removed automatically when it is stopped, + but it will need to be stopped manually by calling `ContainerUtils.stop()`. + """ + command = ( + ["timeout", "/t", "99999", "/nobreak"] + if platform == "windows" + else ["bash", "-c", "sleep infinity"] + ) + return client.containers.run( + image, + command, + stdin_open=platform == "windows", + tty=platform == "windows", + detach=True, + remove=True, + **kwargs, + ) diff --git a/src/ue4docker/infrastructure/DockerUtils.py b/src/ue4docker/infrastructure/DockerUtils.py index 87f4fa02..54b447bf 100644 --- a/src/ue4docker/infrastructure/DockerUtils.py +++ b/src/ue4docker/infrastructure/DockerUtils.py @@ -166,7 +166,7 @@ def listImages(tagFilter=None, filters={}, all=False): return images @staticmethod - def exec(container, command, **kwargs): + def exec(container, command: [str], **kwargs): """ Executes a command in a container returned by `DockerUtils.start()` and returns the output """ @@ -182,7 +182,7 @@ def exec(container, command, **kwargs): return output @staticmethod - def execMultiple(container, commands, **kwargs): + def execMultiple(container, commands: [[str]], **kwargs): """ Executes multiple commands in a container returned by `DockerUtils.start()` """ diff --git a/src/ue4docker/infrastructure/__init__.py b/src/ue4docker/infrastructure/__init__.py index 5e45048e..2d1ac7fe 100644 --- a/src/ue4docker/infrastructure/__init__.py +++ b/src/ue4docker/infrastructure/__init__.py @@ -1,4 +1,5 @@ from .BuildConfiguration import BuildConfiguration +from .ContainerUtils import ContainerUtils from .CredentialEndpoint import CredentialEndpoint from .DarwinUtils import DarwinUtils from .DockerUtils import DockerUtils diff --git a/src/ue4docker/test.py b/src/ue4docker/test.py index 8e410e39..1e63ad0d 100644 --- a/src/ue4docker/test.py +++ b/src/ue4docker/test.py @@ -1,6 +1,16 @@ -from .infrastructure import DockerUtils, GlobalConfiguration, Logger -from container_utils import ContainerUtils, ImageUtils -import docker, os, platform, sys +import ntpath +import os +import posixpath +import sys + +import docker +from docker.errors import ImageNotFound + +from .infrastructure import ( + ContainerUtils, + GlobalConfiguration, + Logger, +) def test(): @@ -14,32 +24,43 @@ def test(): if len(sys.argv) > 1 and sys.argv[1].strip("-") not in ["h", "help"]: # Verify that the specified container image exists tag = sys.argv[1] - image = GlobalConfiguration.resolveTag( + image_name = GlobalConfiguration.resolveTag( "ue4-full:{}".format(tag) if ":" not in tag else tag ) - if DockerUtils.exists(image) == False: + + try: + image = client.images.get(image_name) + except ImageNotFound: logger.error( 'Error: the specified container image "{}" does not exist.'.format( - image + image_name ) ) sys.exit(1) # Use process isolation mode when testing Windows containers, since running Hyper-V containers don't currently support manipulating the filesystem - platform = ImageUtils.image_platform(client, image) + platform = image.attrs["Os"] isolation = "process" if platform == "windows" else None # Start a container to run our tests in, automatically stopping and removing the container when we finish logger.action( - 'Starting a container using the "{}" image...'.format(image), False + 'Starting a container using the "{}" image...'.format(image_name), False + ) + container = ContainerUtils.start_for_exec( + client, image_name, platform, isolation=isolation ) - container = ContainerUtils.start_for_exec(client, image, isolation=isolation) with ContainerUtils.automatically_stop(container): # Create the workspace directory in the container - workspaceDir = ContainerUtils.workspace_dir(container) + workspaceDir = ( + "C:\\workspace" if platform == "windows" else "/tmp/workspace" + ) + shell_prefix = ( + ["cmd", "/S", "/C"] if platform == "windows" else ["bash", "-c"] + ) + ContainerUtils.exec( container, - ContainerUtils.shell_prefix(container) + ["mkdir " + workspaceDir], + shell_prefix + ["mkdir " + workspaceDir], ) # Copy our test scripts into the container @@ -47,12 +68,8 @@ def test(): ContainerUtils.copy_from_host(container, testDir, workspaceDir) # Create a harness to invoke individual tests - containerPath = ContainerUtils.path(container) - pythonCommand = ( - "python" - if ContainerUtils.container_platform(container) == "windows" - else "python3" - ) + containerPath = ntpath if platform == "windows" else posixpath + pythonCommand = "python" if platform == "windows" else "python3" def runTest(script): logger.action('Running test "{}"...'.format(script), False) From 3887487b5909111cf6170de33c4f0119a570efb2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 10 Mar 2023 14:44:02 +0300 Subject: [PATCH 340/430] resolves #301 fix running without `--credential_mode` Broken by 80e68eb4590b0dc68e09f8ce1195414ff840c914 --- src/ue4docker/infrastructure/BuildConfiguration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index eb884d6a..a1b8ff4c 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -502,7 +502,7 @@ def __init__(self, parser, argv, logger): if "credential_mode" not in self.opts: # On Linux, default to secrets mode that causes fewer issues with firewalls - self.opts = ( + self.opts["credential_mode"] = ( "secrets" if self.containerPlatform == "linux" else "endpoint" ) From 3571545b3ccd31cbeeae93e2de4b454137a78697 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 10 Mar 2023 15:00:14 +0300 Subject: [PATCH 341/430] Bump version to 0.0.104 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e056f79e..c7c31ab8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.103" +version = "0.0.104" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 3ce9c2cf892ac2f786da2faa26fd0645b2c363f3 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 10 Mar 2023 05:40:48 -0800 Subject: [PATCH 342/430] Fix upload to PyPI (#302) Multiple things failed at once: 1. Link to readme was lost during conversion to `pyproject.toml` in f66831702365936878d44e8b33594bb93da8a595. This commit restores it. 2. PyPI only accepts rst/markdown/txt, so undo conversion of readme to AsciiDoc that was done in 55191a78aa966010604da5ed1dd5b9fb2492f016 3. twine has a bug that prevents upload from working without readme: https://github.com/pypa/twine/issues/908 Also run `twine check` on CI to detect `pyproject.toml` metadata errors earlier --- .github/workflows/ci.yml | 4 +++- README.adoc | 42 ---------------------------------------- README.md | 39 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 1 + 4 files changed, 43 insertions(+), 43 deletions(-) delete mode 100644 README.adoc create mode 100644 README.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62387d4e..3bd6d362 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,9 +16,11 @@ jobs: with: python-version: "3.10" - name: Install dependencies - run: pip install --upgrade build + run: pip install --upgrade build twine - name: Build package run: python -m build + - name: Check package + run: twine check --strict dist/* - name: Publish to PyPI if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/README.adoc b/README.adoc deleted file mode 100644 index c789f732..00000000 --- a/README.adoc +++ /dev/null @@ -1,42 +0,0 @@ -++++ -

Unreal Engine and Docker Logos

-

Unreal Engine 4 Docker Containers

-

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

-

 

-++++ - -*Looking for a place to start? -Check out the https://unrealcontainers.com/[Unreal Containers community hub] for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the https://docs.adamrehn.com/ue4-docker/[comprehensive ue4-docker documentation] to view details specific to using the ue4-docker project.* - -The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' https://www.unrealengine.com/[Unreal Engine 4]. -The images also incorporate the infrastructure from https://github.com/adamrehn/ue4cli[ue4cli], https://github.com/adamrehn/conan-ue4cli[conan-ue4cli], and https://github.com/adamrehn/ue4-ci-helpers[ue4-ci-helpers] to facilitate a wide variety of use cases. - -Key features include: - -* Unreal Engine 4.20.0 and newer is supported. -* Both Windows containers and Linux containers are supported. -* Building and packaging UE4 projects is supported. -* Running automation tests is supported. -* Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. - -Resources: - -* *Documentation:* https://docs.adamrehn.com/ue4-docker/ -* *GitHub repository:* https://github.com/adamrehn/ue4-docker -* *Package on PyPI:* https://pypi.org/project/ue4-docker/ -* *Related articles:* https://adamrehn.com/articles/tag/Unreal%20Engine/ -* *Unreal Containers community hub:* https://unrealcontainers.com/ -* *Development channel on the Unreal Containers Community Discord server*: https://discord.gg/46CJg9fyJ9 - -== Contributing - -See the file https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md[CONTRIBUTING.md] for guidelines on how to contribute to the development of ue4-docker. - -== Legal - -Copyright © 2018 - 2022, Adam Rehn. -Licensed under the MIT License, see the file https://github.com/adamrehn/ue4-docker/blob/master/LICENSE[LICENSE] for details. - -Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. - -Docker and the Docker logo are trademarks or registered trademarks of Docker in the United States and other countries. diff --git a/README.md b/README.md new file mode 100644 index 00000000..d40ec9c5 --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +

Unreal Engine and Docker Logos

+

Unreal Engine 4 Docker Containers

+

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

+

 

+ +**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://docs.adamrehn.com/ue4-docker/) to view details specific to using the ue4-docker project.** + +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. + +Key features include: + +- Unreal Engine 4.20.0 and newer is supported. +- Both Windows containers and Linux containers are supported. +- Building and packaging UE4 projects is supported. +- Running automation tests is supported. +- Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. + +Resources: + +- **Documentation:** +- **GitHub repository:** +- **Package on PyPI:** +- **Related articles:** +- **Unreal Containers community hub:** +- **Development channel on the Unreal Containers Community Discord server**: + + +## Contributing + +See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. + + +## Legal + +Copyright © 2018 - 2021, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. + +Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. + +Docker and the Docker logo are trademarks or registered trademarks of Docker in the United States and other countries. diff --git a/pyproject.toml b/pyproject.toml index c7c31ab8..a3f71c96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.0.104" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } +readme = "README.md" authors = [ { name = "Adam Rehn", email = "adam@adamrehn.com" }, { name = "Marat Radchenko", email = "marat@slonopotamus.org" }, From ecac3a4154de895647bc76130ebb365ab4be9bb2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 10 Mar 2023 16:43:14 +0300 Subject: [PATCH 343/430] Bump version to 0.0.105 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a3f71c96..072fa159 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.104" +version = "0.0.105" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 739d4dd5e1e132ea9c0dfe60bfb7297953577eb3 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 13 Mar 2023 10:16:11 +0300 Subject: [PATCH 344/430] resolves #303 fix `ue4-docker build` with non-custom engine version Broken by ce89fcd226e2e43fd8b29df68de35e4573c65df7 --- src/ue4docker/infrastructure/BuildConfiguration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index a1b8ff4c..c6bd315d 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -406,8 +406,8 @@ def __init__(self, parser, argv, logger): # Validate the specified version string try: ue4Version = Version(self.args.release) - if ue4Version.major not in [4, 5] or ue4Version.pre != 0: - raise Exception() + if ue4Version.major not in [4, 5] or ue4Version.pre is not None: + raise Exception(f"unsupported engine version: {ue4Version}") self.release = ( f"{ue4Version.major}.{ue4Version.minor}.{ue4Version.micro}" ) From 16571e4939b99350cd138e3d0e9f8b3951f4cbbe Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 13 Mar 2023 10:18:01 +0300 Subject: [PATCH 345/430] Bump version to 0.0.106 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 072fa159..9198c1d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.105" +version = "0.0.106" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From def310b7454dc535a9421bb01568be3c6e3f87bc Mon Sep 17 00:00:00 2001 From: Joanna Anthony Date: Fri, 17 Mar 2023 14:31:07 -0700 Subject: [PATCH 346/430] Pins versions of conan to < v2 in the ue4-full images (#306) --- src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 5 ++++- src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 7 +++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index be372c2c..6b2fff97 100644 --- a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -1,5 +1,6 @@ ARG UE4CLI_VERSION="ue4cli>=0.0.45" ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +ARG CONAN_VERSION="conan>=1.59.0,<2" {% if combine %} FROM source as conan {% else %} @@ -10,11 +11,12 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan {% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION # Install ue4cli and conan-ue4cli USER root RUN pip3 install --upgrade pip setuptools wheel -RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" +RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" USER ue4 # Extract the third-party library details from UBT @@ -29,6 +31,7 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} {% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION # Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers USER root diff --git a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index ceab3634..006acfbe 100644 --- a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -1,6 +1,7 @@ # escape=` ARG UE4CLI_VERSION="ue4cli>=0.0.45" ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +ARG CONAN_VERSION="conan>=1.59.0,<2" {% if combine %} FROM source as conan {% else %} @@ -11,10 +12,11 @@ FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan {% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION # Install ue4cli and conan-ue4cli RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" --no-warn-script-location +RUN pip install "%CONAN_VERSION%" "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" --no-warn-script-location # Build UBT, and extract the third-party library details from UBT # (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) @@ -30,10 +32,11 @@ FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} {% endif %} ARG UE4CLI_VERSION ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION # Install ue4cli conan-ue4cli, and ue4-ci-helpers RUN pip install setuptools wheel --no-warn-script-location -RUN pip install "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue4-ci-helpers --no-warn-script-location +RUN pip install "%CONAN_VERSION%" "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue4-ci-helpers --no-warn-script-location # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) From d992e7dcb9be3be8c125d4cedafe2fb8ef99d5c2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Mar 2023 12:12:08 +0300 Subject: [PATCH 347/430] Bump actions/deploy-pages from 1 to 2 (#309) Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 1 to 2. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v1...v2) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d84b3225..03102eb8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,4 +44,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v1 + uses: actions/deploy-pages@v2 From d29ca33635f032890c60d9503fd546a5096fcca1 Mon Sep 17 00:00:00 2001 From: Joanna Anthony Date: Thu, 13 Apr 2023 04:56:25 -0700 Subject: [PATCH 348/430] Increases size set by setup command, and changes the recommend size throughout the documentation (#308) --- docs/configuring-linux.adoc | 2 +- docs/configuring-macos.adoc | 4 ++-- docs/configuring-windows-10.adoc | 6 +++--- docs/configuring-windows-server.adoc | 2 +- docs/ue4-docker-setup.adoc | 2 +- src/ue4docker/infrastructure/WindowsUtils.py | 2 +- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/configuring-linux.adoc b/docs/configuring-linux.adoc index 71f75f8e..ff9d9b08 100644 --- a/docs/configuring-linux.adoc +++ b/docs/configuring-linux.adoc @@ -9,7 +9,7 @@ - 64-bit version of one of Docker's https://docs.docker.com/install/#supported-platforms[supported Linux distributions] (CentOS 7+, Debian 7.7+, Fedora 26+, Ubuntu 14.04+) - Minimum 8GB of RAM -- Minimum 400GB available disk space for building container images +- Minimum 800GB available disk space for building container images == Step 1: Install Docker CE diff --git a/docs/configuring-macos.adoc b/docs/configuring-macos.adoc index 3f587bdf..ca4434a7 100644 --- a/docs/configuring-macos.adoc +++ b/docs/configuring-macos.adoc @@ -13,7 +13,7 @@ Linux containers are unable to use GPU acceleration via the xref:nvidia-docker-p - 2010 or newer model Mac hardware - macOS 10.10.3 Yosemite or newer - Minimum 8GB of RAM -- Minimum 400GB available disk space for building container images +- Minimum 800GB available disk space for building container images == Step 1: Install Docker CE for Mac @@ -48,4 +48,4 @@ sudo pip3 install ue4-docker == Step 4: Manually configure Docker daemon settings -Use https://docs.docker.com/desktop/mac/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 400GB. +Use https://docs.docker.com/desktop/mac/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 800GB. diff --git a/docs/configuring-windows-10.adoc b/docs/configuring-windows-10.adoc index 36bb38d1..03ed51f4 100644 --- a/docs/configuring-windows-10.adoc +++ b/docs/configuring-windows-10.adoc @@ -61,7 +61,7 @@ For building and running Windows containers: - Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Windows containers] rather than Linux containers. - Configure the Docker daemon to increase the maximum container disk size from the default 20GB limit by following https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits[the instructions provided by Microsoft]. -The 120GB limit specified in the instructions is not quite enough, so set a 400GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** +The 120GB limit specified in the instructions is not quite enough, so set an 800GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** WARNING: **The ue4-docker maintainers do not provide support for building and running Linux containers under Windows**, due to the various technical limitations of the Hyper-V and WSL2 backends for Docker Desktop (see https://github.com/adamrehn/ue4-docker/issues/205[this issue] for details of these limitations). This functionality is still present in ue4-docker for those who choose to use it, but users are solely responsible for troubleshooting any issues they encounter when doing so. @@ -70,6 +70,6 @@ For building and running Linux containers: - Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Linux containers] rather than Windows containers. -- **If you are using the Hyper-V backend** then use https://docs.docker.com/desktop/windows/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 400GB. +- **If you are using the Hyper-V backend** then use https://docs.docker.com/desktop/windows/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 800GB. -- **If you are using the WSL2 backend** then https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk[expand the WSL2 virtual hard disk] to at least 400GB. +- **If you are using the WSL2 backend** then https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk[expand the WSL2 virtual hard disk] to at least 800GB. diff --git a/docs/configuring-windows-server.adoc b/docs/configuring-windows-server.adoc index 2e857f85..2d1b2de8 100644 --- a/docs/configuring-windows-server.adoc +++ b/docs/configuring-windows-server.adoc @@ -67,4 +67,4 @@ To automatically configure the required system settings, run the xref:ue4-docker ue4-docker setup ---- -This will configure the Docker daemon to set the maximum image size to 400GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. +This will configure the Docker daemon to set the maximum image size to 800GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. diff --git a/docs/ue4-docker-setup.adoc b/docs/ue4-docker-setup.adoc index eea570d4..e76985e6 100644 --- a/docs/ue4-docker-setup.adoc +++ b/docs/ue4-docker-setup.adoc @@ -24,7 +24,7 @@ This command will automatically configure a Linux or Windows Server host system **Under Windows Server:** -- The Docker daemon will be configured to set the maximum image size for Windows containers to 400GB. +- The Docker daemon will be configured to set the maximum image size for Windows containers to 800GB. - A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image. - Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the https://hub.docker.com/_/microsoft-windows[full Windows base image]. Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. diff --git a/src/ue4docker/infrastructure/WindowsUtils.py b/src/ue4docker/infrastructure/WindowsUtils.py index 9f0f1d96..fef462b6 100644 --- a/src/ue4docker/infrastructure/WindowsUtils.py +++ b/src/ue4docker/infrastructure/WindowsUtils.py @@ -51,7 +51,7 @@ def requiredSizeLimit() -> float: """ Returns the minimum required image size limit (in GB) for Windows containers """ - return 400.0 + return 800.0 @staticmethod def minimumRequiredBuild() -> int: From e51746f8b852f6f61588dca5531366e92f69e0f5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 3 May 2023 21:16:34 +0300 Subject: [PATCH 349/430] Restrict urllib3 dep to <2 This is a temporary workaround for https://github.com/docker/docker-py/issues/3113 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 9198c1d6..41c4a3ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "packaging>=19.1", "psutil", "termcolor", + # Workaround for https://github.com/docker/docker-py/issues/3113 + "urllib3<2", ] [project.urls] From c0e118ed1b15bc970a833da020c86293373bd891 Mon Sep 17 00:00:00 2001 From: Patrick Creighton Date: Thu, 4 May 2023 02:56:52 -0700 Subject: [PATCH 350/430] Update baseimg from nvidia/cudagl to nvidia/cuda (#315) --- docs/advanced-build-options.adoc | 2 +- docs/available-container-images.adoc | 2 +- src/ue4docker/infrastructure/BuildConfiguration.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc index 30c50dfc..b0f03e7a 100644 --- a/docs/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -253,4 +253,4 @@ ue4-docker build RELEASE --cuda=10.0 {% endhighlight %} ---- -For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the https://hub.docker.com/r/nvidia/cudagl/[nvidia/cudagl] base image. +For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the https://hub.docker.com/r/nvidia/cuda/[nvidia/cuda] base image. diff --git a/docs/available-container-images.adoc b/docs/available-container-images.adoc index 505eb2d1..59776693 100644 --- a/docs/available-container-images.adoc +++ b/docs/available-container-images.adoc @@ -26,7 +26,7 @@ You can prevent unwanted images from being built by appending the relevant image ** `opengl` if CUDA support is not enabled -** `cudaglVERSION` where `VERSION` is the CUDA version if xref:advanced-build-options.adoc#cuda[CUDA support is enabled] (e.g. `cudagl9.2`, `cudagl10.0`, etc.) +** `cudaVERSION` where `VERSION` is the CUDA version if xref:advanced-build-options.adoc#cuda[CUDA support is enabled] (e.g. `cuda9.2`, `cuda10.0`, etc.) **Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index c6bd315d..52798168 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -14,7 +14,7 @@ # The base images for Linux containers LINUX_BASE_IMAGES = { "opengl": "nvidia/opengl:1.0-glvnd-devel-{ubuntu}", - "cudagl": "nvidia/cudagl:{cuda}-devel-{ubuntu}", + "cuda": "nvidia/cuda:{cuda}-devel-{ubuntu}", } # The default ubuntu base to use @@ -675,8 +675,8 @@ def _generateWindowsConfig(self): def _generateLinuxConfig(self): # Verify that any user-specified tag suffix does not collide with our base tags - if self.suffix.startswith("opengl") or self.suffix.startswith("cudagl"): - raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') + if self.suffix.startswith("opengl") or self.suffix.startswith("cuda"): + raise RuntimeError('tag suffix cannot begin with "opengl" or "cuda".') # Determine if we are building CUDA-enabled container images self.cuda = None @@ -684,8 +684,8 @@ def _generateLinuxConfig(self): # Verify that the specified CUDA version is valid self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION # Use the appropriate base image for the specified CUDA version - self.baseImage = LINUX_BASE_IMAGES["cudagl"] - self.prereqsTag = "cudagl{cuda}-{ubuntu}" + self.baseImage = LINUX_BASE_IMAGES["cuda"] + self.prereqsTag = "cuda{cuda}-{ubuntu}" else: self.baseImage = LINUX_BASE_IMAGES["opengl"] self.prereqsTag = "opengl-{ubuntu}" From 3ee632cd2fb7dc86abefd88b47a921580133fbc0 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 5 May 2023 20:27:26 +0300 Subject: [PATCH 351/430] Replace workaround for docker-py breakage with a proper fix (#318) This commit pushes docker-py *a lot* versions forward, but we still keep Python 3.7 as oldest supported. See e51746f8b852f6f61588dca5531366e92f69e0f5 See https://github.com/docker/docker-py/issues/3113 --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 41c4a3ef..2a24f11b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,15 +23,13 @@ classifiers = [ ] dependencies = [ "colorama", - "docker>=3.0.0", + "docker>=6.1.0", "humanfriendly", "importlib-metadata>=1.0;python_version<'3.8'", "Jinja2>=2.11.3", "packaging>=19.1", "psutil", "termcolor", - # Workaround for https://github.com/docker/docker-py/issues/3113 - "urllib3<2", ] [project.urls] From 84d73dae83267a4a343b11c5bf36159f5bd0d274 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 7 May 2023 23:29:58 +0300 Subject: [PATCH 352/430] Bump version to 0.0.107 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 2a24f11b..df6141d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.106" +version = "0.0.107" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From a2f4180e223c811441314b632ae96fbdee7de88e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 1 Jun 2023 18:59:27 +0300 Subject: [PATCH 353/430] Lock Chocolatey to 1.4.0 (#323) This is a workaround for https://github.com/chocolatey/choco/issues/3184: Chocolatey 2.0.0 switched to .NET 4.8, but it throws an exception requiring a reboot. --- .../dockerfiles/diagnostics/network/windows/Dockerfile | 2 +- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 +- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 4 ++-- src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile index adff349b..13e68909 100644 --- a/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile @@ -7,4 +7,4 @@ SHELL ["cmd", "/S", "/C"] LABEL com.adamrehn.ue4-docker.sentinel="1" # Attempt to download the Chocolatey installation script -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; (New-Object System.Net.WebClient).DownloadFile('https://chocolatey.org/install.ps1', 'install.ps1')" +RUN wget 'https://chocolatey.org/install.ps1' diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 72bc765b..47a6ec7a 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -54,7 +54,7 @@ LABEL com.adamrehn.ue4-docker.sentinel="1" RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f # Install Chocolatey -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "$env:chocolateyVersion = '1.4.0'; Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Install the rest of our build prerequisites and clean up afterwards to minimise image size COPY install-prerequisites.ps1 C:\ diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index 35eda990..3bee52d6 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -8,10 +8,10 @@ $env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" # Install the chocolatey packages we need -choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" +choco install --no-progress -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" # pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg +choco install --no-progress -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg # Reload our environment variables from the registry so the `git` command works Update-SessionEnvironment diff --git a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index 006acfbe..b30da1b4 100644 --- a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -50,4 +50,4 @@ RUN ue4 setroot C:\UnrealEngine RUN ue4 conan generate --profile-only # Install CMake and add it to the system PATH -RUN choco install -y cmake --installargs "ADD_CMAKE_TO_PATH=System" +RUN choco install --no-progress -y cmake --installargs "ADD_CMAKE_TO_PATH=System" From 8a5e1dd7b2eaac01067a62cf5e0fe238100d2820 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 1 Jun 2023 20:29:05 +0300 Subject: [PATCH 354/430] Run diagnostics on CI (#322) --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bd6d362..f1c6203b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,5 +56,7 @@ jobs: python-version: ${{ matrix.python }} - name: Install run: pip install . --user - - name: Test + - name: Build prerequisites run: ue4-docker build --target=build-prerequisites --visual-studio ${{ matrix.visual-studio || '2017' }} + - name: Run diagnostics + run: ue4-docker diagnostics all From ad1e169bf445370b9938be6f5aa1b71ce33cac0f Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 1 Jun 2023 20:34:37 +0300 Subject: [PATCH 355/430] resolves #310 disable interactive mode of git-credential-manager --- src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index e89b1fda..ddd5c8a3 100644 --- a/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -37,6 +37,8 @@ ENV HOST_TOKEN=${HOST_TOKEN_ARG} # Install our git credential helper that forwards requests to the host COPY git-credential-helper.bat C:\git-credential-helper.bat ENV GIT_ASKPASS=C:\git-credential-helper.bat +# See https://github.com/git-ecosystem/git-credential-manager/blob/main/docs/environment.md#GCM_INTERACTIVE +ENV GCM_INTERACTIVE=false # Clone the UE4 git repository using the host-supplied credentials WORKDIR C:\ From 26824b5441f6a534344ea0f8ea3adee7ca07e797 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 5 Jun 2023 12:15:13 +0300 Subject: [PATCH 356/430] Bump version to 0.0.108 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index df6141d2..3d2af86f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.107" +version = "0.0.108" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 7a8544223e26e53e7d7e29d863b3649ada42e052 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 5 Jun 2023 17:55:57 +0300 Subject: [PATCH 357/430] resolves #311 use `apk update` to test that network works There seems to be an issue in busybox `wget` with SSL proxy: https://github.com/wurstmeister/kafka-docker/issues/487 --- .../dockerfiles/diagnostics/network/linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile index 922a3a33..d99976cf 100644 --- a/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile @@ -3,5 +3,5 @@ FROM alpine:latest # Add a sentinel label so we can easily identify intermediate images LABEL com.adamrehn.ue4-docker.sentinel="1" -# Attempt to download the Chocolatey installation script (not actually useful under Linux of course, but useful as a network test) -RUN wget 'https://chocolatey.org/install.ps1' +# Test that network works +RUN apk update From 094079e02afd3063c74842ea37e5e403d3965322 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 10 Jun 2023 11:08:12 +0300 Subject: [PATCH 358/430] Build minimal image by default (#327) There are multiple long-standing issues with ue4-full: * https://github.com/adamrehn/conan-ue4cli/issues/14 * https://github.com/adamrehn/conan-ue4cli/issues/23 If user wants to build ue4-full, they should pass `--target=full` option. --- src/ue4docker/infrastructure/BuildConfiguration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 52798168..5d6fbaa6 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -285,9 +285,9 @@ def __init__(self, parser, argv, logger): using_target_specifier_old = self.args.no_minimal or self.args.no_full using_target_specifier_new = self.args.target is not None - # If we specified nothing, it's the same as specifying `all` + # If we specified nothing, it's the same as specifying `minimal` if not using_target_specifier_old and not using_target_specifier_new: - self.args.target = ["all"] + self.args.target = ["minimal"] elif using_target_specifier_old and not using_target_specifier_new: # Convert these to the new style logger.warning( From ec997c243bfd59afca832fce437a34bf1e1effcf Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 10 Jun 2023 15:31:40 +0300 Subject: [PATCH 359/430] Stop sending users to outdated documentation --- README.md | 16 ++++++++-------- docs/available-container-images.adoc | 8 ++++---- pyproject.toml | 5 ++++- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index d40ec9c5..bf16da53 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@

Unreal Engine and Docker Logos

-

Unreal Engine 4 Docker Containers

-

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

+

Unreal Engine Docker Containers

+

Continuous Integration • Cloud Rendering • Unreal Engine Powered Microservices

 

-**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://docs.adamrehn.com/ue4-docker/) to view details specific to using the ue4-docker project.** +**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://github.com/adamrehn/ue4-docker) to view details specific to using the ue4-docker project.** The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. @@ -11,15 +11,15 @@ Key features include: - Unreal Engine 4.20.0 and newer is supported. - Both Windows containers and Linux containers are supported. -- Building and packaging UE4 projects is supported. +- Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. -- Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. +- Running built Unreal Engine projects with offscreen rendering is supported via NVIDIA Docker under Linux. Resources: -- **Documentation:** +- **Documentation:** - **GitHub repository:** -- **Package on PyPI:** +- **Package on PyPI:** - **Related articles:** - **Unreal Containers community hub:** - **Development channel on the Unreal Containers Community Discord server**: @@ -27,7 +27,7 @@ Resources: ## Contributing -See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. +See the file [CONTRIBUTING.adoc](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.adoc) for guidelines on how to contribute to the development of ue4-docker. ## Legal diff --git a/docs/available-container-images.adoc b/docs/available-container-images.adoc index 59776693..4efe2c62 100644 --- a/docs/available-container-images.adoc +++ b/docs/available-container-images.adoc @@ -28,7 +28,7 @@ You can prevent unwanted images from being built by appending the relevant image ** `cudaVERSION` where `VERSION` is the CUDA version if xref:advanced-build-options.adoc#cuda[CUDA support is enabled] (e.g. `cuda9.2`, `cuda10.0`, etc.) -**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] **Contents:** Contains the build prerequisites common to all Engine versions. @@ -45,7 +45,7 @@ You can prevent unwanted images from being built by appending the relevant image * `adamrehn/ue4-source:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag -**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/linux/Dockerfile[icon:linux[] Linux] +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/linux/Dockerfile[icon:linux[] Linux] **Contents:** Contains the cloned source code for Unreal Engine, along with its downloaded dependency data. The ue4-minimal image uses this source code as the starting point for its build. @@ -64,7 +64,7 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save * `adamrehn/ue4-minimal:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag -**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile[icon:linux[] Linux] +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile[icon:linux[] Linux] **Contents:** Contains the absolute minimum set of components required for use in a Continuous Integration (CI) pipeline, consisting of only the build prerequisites and an Installed Build of the Engine. @@ -81,7 +81,7 @@ Afterwards, this image can be removed using `ue4-docker clean --source` to save * `adamrehn/ue4-full:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag -**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/linux/Dockerfile[icon:linux[] Linux] +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/src/master/ue4docker/dockerfiles/ue4-full/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/linux/Dockerfile[icon:linux[] Linux] **Contents:** Contains everything from the `ue4-minimal` image, and adds the following: diff --git a/pyproject.toml b/pyproject.toml index 3d2af86f..893cc91c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,8 +32,11 @@ dependencies = [ "termcolor", ] +# See https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#urls [project.urls] -"Homepage" = "https://github.com/adamrehn/ue4-docker" +Homepage = "https://github.com/adamrehn/ue4-docker" +Documentation = "https://adamrehn.github.io/ue4-docker" +Repository = "https://github.com/adamrehn/ue4-docker.git" [project.scripts] ue4-docker = "ue4docker:main" From 316f9f9892be9f387bf0e9d4fb8ef97faf5549a2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sat, 10 Jun 2023 15:36:53 +0300 Subject: [PATCH 360/430] Fix several more references to outdated documentation --- CONTRIBUTING.adoc | 2 +- README.md | 2 +- src/ue4docker/diagnostics/diagnostic_network.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc index 5667ba89..a63a2448 100644 --- a/CONTRIBUTING.adoc +++ b/CONTRIBUTING.adoc @@ -11,7 +11,7 @@ When creating an issue to report a bug, please follow the provided issue templat Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. If you are creating an issue for a feature request, you can disregard the default issue template. -However, please make sure you have thoroughly read https://docs.adamrehn.com/ue4-docker/[the documentation] to ensure the requested feature does not already exist before creating an issue. +However, please make sure you have thoroughly read https://adamrehn.github.io/ue4-docker[the documentation] to ensure the requested feature does not already exist before creating an issue. == Creating pull requests diff --git a/README.md b/README.md index bf16da53..b14b2385 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

Unreal Engine and Docker Logos

Unreal Engine Docker Containers

-

Continuous Integration • Cloud Rendering • Unreal Engine Powered Microservices

+

Continuous Integration • Pixel Streaming • Unreal Engine Powered Microservices

 

**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://github.com/adamrehn/ue4-docker) to view details specific to using the ue4-docker project.** diff --git a/src/ue4docker/diagnostics/diagnostic_network.py b/src/ue4docker/diagnostics/diagnostic_network.py index c2da8dea..aa56759e 100644 --- a/src/ue4docker/diagnostics/diagnostic_network.py +++ b/src/ue4docker/diagnostics/diagnostic_network.py @@ -101,7 +101,7 @@ def run(self, logger, args=[]): True, ) logger.error( - "[network] https://docs.adamrehn.com/ue4-docker/building-images/troubleshooting-build-issues#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n", + "[network] https://adamrehn.github.io/ue4-docker/#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n", False, ) From dd1fca4a6affcd0c5a9498ad75b90c5cd3c7a016 Mon Sep 17 00:00:00 2001 From: Kalin Norman <41397278+kalinnorman@users.noreply.github.com> Date: Sun, 11 Jun 2023 01:14:01 -0600 Subject: [PATCH 361/430] Fix conan not being installed in second image (#328) --- src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 6b2fff97..368bdfea 100644 --- a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -37,7 +37,7 @@ ARG CONAN_VERSION USER root RUN apt-get update && apt-get install -y --no-install-recommends cmake RUN pip3 install --upgrade pip setuptools wheel -RUN pip3 install "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers +RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers USER ue4 # Explicitly set the configuration directory for ue4cli From 34b8e1a3226de572416dc9645597bb49bd0804a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jul 2023 00:10:09 +0300 Subject: [PATCH 362/430] Bump actions/upload-pages-artifact from 1 to 2 (#331) Bumps [actions/upload-pages-artifact](https://github.com/actions/upload-pages-artifact) from 1 to 2. - [Release notes](https://github.com/actions/upload-pages-artifact/releases) - [Commits](https://github.com/actions/upload-pages-artifact/compare/v1...v2) --- updated-dependencies: - dependency-name: actions/upload-pages-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 03102eb8..51f1fd2f 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,7 +31,7 @@ jobs: run: docs/build.sh - name: Upload artifact - uses: actions/upload-pages-artifact@v1 + uses: actions/upload-pages-artifact@v2 with: path: "build/gh-pages" From 58e5c058b72a81c4eef0d2cdea896e253c8da2fc Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 14 Aug 2023 18:03:31 +0300 Subject: [PATCH 363/430] Add CL for 5.2 --- src/ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 5d6fbaa6..a602f86c 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -38,6 +38,7 @@ "4.27.0": 17155196, "5.0.0": 19505902, "5.1.0": 23058290, + "5.2.0": 25360045, } From 5d9f160c2a86caf1d94c64dd16f1cdf378edfd6d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 17 Aug 2023 15:37:16 +0300 Subject: [PATCH 364/430] Bump version to 0.0.109 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 893cc91c..9a4ed517 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.108" +version = "0.0.109" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From a6fd311dbc350a2fc4c9127d9ea8280afcf3de9d Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 25 Aug 2023 10:28:38 +0300 Subject: [PATCH 365/430] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b14b2385..324b86f3 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://github.com/adamrehn/ue4-docker) to view details specific to using the ue4-docker project.** -The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. Key features include: From dbee2563295119f7bcb86cd00cb0a6b851e84103 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 12:12:51 +0300 Subject: [PATCH 366/430] Bump actions/checkout from 3 to 4 (#333) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 6 +++--- .github/workflows/docs.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f1c6203b..eb427a5d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,13 +4,13 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: psf/black@stable build: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 with: @@ -49,7 +49,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 51f1fd2f..7885444a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Build docs run: docs/build.sh From 29a30035261e809528f22f5a89ec37b7fe6adc21 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 7 Sep 2023 14:30:32 +1000 Subject: [PATCH 367/430] 5.3.0 compat: build SCW during setup steps rather than UHT --- src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 2be28863..bda41250 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -33,7 +33,7 @@ RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/Instal # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild -buildubt +RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild -buildubt # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine @@ -97,7 +97,7 @@ LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templ # Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` # (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) # See the comments on this issue for details, including the need to ensure $HOME is set correctly: -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealHeaderTool Linux Development -SkipBuild && \ +RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild && \ mkdir -p ./Engine/Programs/AutomationTool/Saved && \ chmod a+rw ./Engine/Programs/AutomationTool/Saved From 5ccc0e0e24f15022d4f25e804e8287ca207c3771 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 7 Sep 2023 14:52:41 +1000 Subject: [PATCH 368/430] Add CL number for Unreal Engine 5.3.0 --- src/ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index a602f86c..e9a6f928 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -39,6 +39,7 @@ "5.0.0": 19505902, "5.1.0": 23058290, "5.2.0": 25360045, + "5.3.0": 27405482, } From 9687c68d177adde464216325b036c0499c2353d0 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 8 Sep 2023 14:01:55 +1000 Subject: [PATCH 369/430] 5.3.0 compat: build Editor target for first-run setup --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 4 +++- .../ue4-minimal/linux/print-editor-target.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index bda41250..2a752787 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -97,7 +97,9 @@ LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templ # Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` # (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) # See the comments on this issue for details, including the need to ensure $HOME is set correctly: -RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild && \ +COPY print-editor-target.py /tmp/print-editor-target.py +RUN EDITOR_TARGET=$(python3 /tmp/print-editor-target.py /home/ue4/UnrealEngine) && \ + ./Engine/Build/BatchFiles/Linux/Build.sh "$EDITOR_TARGET" Linux Development -SkipBuild && \ mkdir -p ./Engine/Programs/AutomationTool/Saved && \ chmod a+rw ./Engine/Programs/AutomationTool/Saved diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py new file mode 100644 index 00000000..d1619bf3 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +from pathlib import Path +import json, sys + +# Parse the Unreal Engine version information +engineRoot = Path(sys.argv[1]) +versionFile = engineRoot / "Engine" / "Build" / "Build.version" +versionDetails = json.loads(versionFile.read_text("utf-8")) + +# Determine the name of the Editor target based on the version +target = "UE4Editor" if versionDetails["MajorVersion"] == 4 else "UnrealEditor" +print(target) From 4f22719946f1b9f5d050526622826a6eead44073 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 8 Sep 2023 14:04:26 +1000 Subject: [PATCH 370/430] Bump version to 0.0.110 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9a4ed517..cee82284 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.109" +version = "0.0.110" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From edb8be4f25f866b1f740cfcdd71b00409e8770c5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 17 Oct 2023 19:08:27 +0300 Subject: [PATCH 371/430] resolves #296 use Ubuntu 22.04 by default (#334) --- docs/advanced-build-options.adoc | 6 +++--- docs/ue4-docker-build.adoc | 2 +- src/ue4docker/infrastructure/BuildConfiguration.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc index b0f03e7a..26352a0b 100644 --- a/docs/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -248,9 +248,9 @@ ue4-docker build RELEASE --cuda [source,shell] ---- -# Uses the CUDA 10.0 base image -ue4-docker build RELEASE --cuda=10.0 +# Uses the CUDA 12.2.0 base image +ue4-docker build RELEASE --cuda=12.2.0 {% endhighlight %} ---- -For a list of supported CUDA versions, see the list of Ubuntu 18.04 image tags for the https://hub.docker.com/r/nvidia/cuda/[nvidia/cuda] base image. +For a list of supported CUDA versions, see the list of Ubuntu 22.04 image tags for the https://hub.docker.com/r/nvidia/cuda/[nvidia/cuda] base image. diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index e0c5a8b5..59fec735 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -34,7 +34,7 @@ See the xref:available-container-images.adoc[List of available container images] *-basetag* _basetag_:: Operating system base image tag to use. -For Linux this is the version of Ubuntu (default is ubuntu18.04). +For Linux this is the version of Ubuntu (default is ubuntu22.04). For Windows this is the Windows Server Core base image tag (default is the host OS version) *-branch* _branch_:: diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index e9a6f928..62cc1d59 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -18,10 +18,10 @@ } # The default ubuntu base to use -DEFAULT_LINUX_VERSION = "ubuntu18.04" +DEFAULT_LINUX_VERSION = "ubuntu22.04" # The default CUDA version to use when `--cuda` is specified without a value -DEFAULT_CUDA_VERSION = "11.4" +DEFAULT_CUDA_VERSION = "12.2.0" # The default memory limit (in GB) under Windows DEFAULT_MEMORY_LIMIT = 10.0 @@ -206,7 +206,7 @@ def addArguments(parser): parser.add_argument( "-basetag", default=None if platform.system() == "Windows" else DEFAULT_LINUX_VERSION, - help="Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu18.04). " + help="Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu22.04). " "For Windows this is the Windows Server Core base image tag (default is the host OS version)", ) parser.add_argument( From c363409e07fa7801f7146ed034aab96548398d4d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 24 Oct 2023 14:51:42 +1000 Subject: [PATCH 372/430] Document basetag requirements for Unreal Engine 5.0.x --- docs/troubleshooting-build-issues.adoc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/troubleshooting-build-issues.adoc b/docs/troubleshooting-build-issues.adoc index 8f74cf1d..b7f94a01 100644 --- a/docs/troubleshooting-build-issues.adoc +++ b/docs/troubleshooting-build-issues.adoc @@ -73,6 +73,10 @@ This issue can occur both inside containers and when running directly on a host . Clear the Docker filesystem layer cache by running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune]. . Re-run the xref:ue4-docker-build.adoc[ue4-docker build] command. +=== Building Unreal Engine 5.0 in a Linux container fails with the error message `No usable version of libssl was found` + +This is a known compatibility issue that affects Unreal Engine 5.0.0 through to Unreal Engine 5.0.3 when running under Ubuntu 22.04 or newer. In order to build Unreal Engine 5.0 in a Linux container, you will need to use an Ubuntu 20.04 base image by specifying the flags `-basetag ubuntu20.04`. + == Windows-specific issues === Building the `ue4-build-prerequisites` image fails with an unknown error From d1265783d14283e048dbaa5c1f160df537737e00 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 24 Oct 2023 15:43:16 +1000 Subject: [PATCH 373/430] Explicitly install libgbm1 as per #337 --- .../dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index d16d5181..d73a7d7a 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -49,6 +49,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libcairo2 \ libfontconfig1 \ libfreetype6 \ + libgbm1 \ libglu1 \ libnss3 \ libnspr4 \ From 4ab089a8fd208dbbef1f0b5cfa655bc41b4b4752 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 16 Nov 2023 17:50:57 +0300 Subject: [PATCH 374/430] Fix "SyntaxWarning: invalid escape sequence '\S'" --- .../ue4-source/linux/patch-broken-releases.py | 5 +++-- .../ue4-source/windows/patch-broken-releases.py | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py index 3251a1a9..5fb4f0f2 100644 --- a/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py +++ b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -import json, os, subprocess, sys +import json +import sys from os.path import join @@ -47,7 +48,7 @@ def writeFile(filename, data): writeFile(gitdepsFile, gitdepsXml) - if verboseOutput == True: + if verboseOutput: print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) else: print("PATCHED {}".format(gitdepsFile), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index 4b655928..980d8f02 100644 --- a/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -import json, os, subprocess, sys +import json +import sys from os.path import join @@ -47,7 +48,7 @@ def writeFile(filename, data): writeFile(gitdepsFile, gitdepsXml) - if verboseOutput == True: + if verboseOutput: print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) else: print("PATCHED {}".format(gitdepsFile), file=sys.stderr) @@ -67,12 +68,12 @@ def writeFile(filename, data): buildXml = buildXml.replace( '', '\n' - + " Engine\Saved\CsTools\Engine\Intermediate\ScriptModules\HoloLens.Automation.json\n", + + " Engine\\Saved\\CsTools\\Engine\\Intermediate\\ScriptModules\\HoloLens.Automation.json\n", ) writeFile(buildFile, buildXml) - if verboseOutput == True: + if verboseOutput: print("PATCHED {}:\n\n{}".format(buildFile, buildFile), file=sys.stderr) else: print("PATCHED {}".format(buildFile), file=sys.stderr) From 168960a1553a4248d9aa16155d8841ab97f790b9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 22 Nov 2023 19:00:08 +0300 Subject: [PATCH 375/430] Add `--docker-build-args` option to passthrough `docker build` arguments --- docs/ue4-docker-build.adoc | 4 ++++ src/ue4docker/build.py | 2 +- src/ue4docker/infrastructure/BuildConfiguration.py | 6 ++++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index 59fec735..092f249c 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -46,6 +46,10 @@ Combine generated Dockerfiles into a single multi-stage build Dockerfile *-conan-ue4cli* _conan_ue4cli_:: Override the default version of conan-ue4cli installed in the ue4-full image +*--docker-build-args*:: +Allows passing custom args to `docker build` command. +For example, this can be useful for pushing images. + *--dry-run*:: Use this if you would like to see what Docker commands would be run by `ue4-docker build` without actually building anything. Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index 19a9bf4c..bd8c054b 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -359,7 +359,7 @@ def build(): commonArgs = [ "--build-arg", "NAMESPACE={}".format(GlobalConfiguration.getTagNamespace()), - ] + ] + config.args.docker_build_args # Build the UE4 build prerequisites image if config.buildTargets["build-prerequisites"]: diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 62cc1d59..c6bd5630 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -149,6 +149,12 @@ def addArguments(parser): action="store_true", help="Use a random memory limit for Windows containers", ) + parser.add_argument( + "--docker-build-args", + action="append", + default=[], + help="Specify additional options for 'docker build' commands" + ) parser.add_argument( "--exclude", action="append", From 49cb4f32ea0f02454dd5fb871289b13df2456e6a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 22 Nov 2023 19:00:40 +0300 Subject: [PATCH 376/430] Fix manpage syntax --- docs/ue4-docker-build.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc index 092f249c..9ddba2af 100644 --- a/docs/ue4-docker-build.adoc +++ b/docs/ue4-docker-build.adoc @@ -46,7 +46,7 @@ Combine generated Dockerfiles into a single multi-stage build Dockerfile *-conan-ue4cli* _conan_ue4cli_:: Override the default version of conan-ue4cli installed in the ue4-full image -*--docker-build-args*:: +*--docker-build-args* _args_:: Allows passing custom args to `docker build` command. For example, this can be useful for pushing images. From 9f189ae35ad536dc7d297e52cd7090caf57052ac Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 23 Nov 2023 14:25:55 +0300 Subject: [PATCH 377/430] Fix black --- src/ue4docker/infrastructure/BuildConfiguration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index c6bd5630..092d38fa 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -153,7 +153,7 @@ def addArguments(parser): "--docker-build-args", action="append", default=[], - help="Specify additional options for 'docker build' commands" + help="Specify additional options for 'docker build' commands", ) parser.add_argument( "--exclude", From 8fa3c54001c6321c559df0599801465b1089a394 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:57:44 +0300 Subject: [PATCH 378/430] Bump actions/deploy-pages from 2 to 3 (#339) Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 2 to 3. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7885444a..bef55616 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,4 +44,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v2 + uses: actions/deploy-pages@v3 From 0c8f1f5f90c9a2e28ae78fbdae984776be094a4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Dec 2023 12:24:27 +0300 Subject: [PATCH 379/430] Bump actions/setup-python from 4 to 5 (#340) Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb427a5d..315b5bf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install dependencies @@ -51,7 +51,7 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install From a743bee0dcf37cda71204611d687a7229d65b424 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:00:51 +0300 Subject: [PATCH 380/430] Bump actions/deploy-pages from 3 to 4 (#341) Bumps [actions/deploy-pages](https://github.com/actions/deploy-pages) from 3 to 4. - [Release notes](https://github.com/actions/deploy-pages/releases) - [Commits](https://github.com/actions/deploy-pages/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/deploy-pages dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bef55616..448625e9 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -44,4 +44,4 @@ jobs: steps: - name: Deploy to GitHub Pages id: deployment - uses: actions/deploy-pages@v3 + uses: actions/deploy-pages@v4 From f8aeea85e228bd2d8eb9c2b139e3ec6a5b68f2d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Dec 2023 10:01:13 +0300 Subject: [PATCH 381/430] Bump actions/upload-pages-artifact from 2 to 3 (#342) Bumps [actions/upload-pages-artifact](https://github.com/actions/upload-pages-artifact) from 2 to 3. - [Release notes](https://github.com/actions/upload-pages-artifact/releases) - [Commits](https://github.com/actions/upload-pages-artifact/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/upload-pages-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 448625e9..047a94b4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -31,7 +31,7 @@ jobs: run: docs/build.sh - name: Upload artifact - uses: actions/upload-pages-artifact@v2 + uses: actions/upload-pages-artifact@v3 with: path: "build/gh-pages" From 9b55e0e87f46409e854bcb0fe121c5941c05e623 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 10 Jan 2024 07:20:30 -0800 Subject: [PATCH 382/430] resolves #343 properly check process exit codes in install-prerequisites.ps1 (#345) --- .../windows/install-prerequisites.ps1 | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index 3bee52d6..b4479331 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -1,5 +1,18 @@ $ErrorActionPreference = "stop" +function RunProcessChecked +{ + param ([string] $Cmd, [string[]] $Argv) + + Write-Output "Executing comand: $Cmd $Argv" + + $process = Start-Process -NoNewWindow -PassThru -Wait -FilePath $Cmd -ArgumentList $Argv + if ($process.ExitCode -ne 0) + { + throw "Exit code: $($process.ExitCode)" + } +} + # TODO: Why `Update-SessionEnvironment` doesn't Just Work without this? # Taken from https://stackoverflow.com/a/46760714 # Make `Update-SessionEnvironment` available right away, by defining the $env:ChocolateyInstall @@ -8,25 +21,27 @@ $env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" # Install the chocolatey packages we need -choco install --no-progress -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" +RunProcessChecked "choco" @("install", "--no-progress", "-y", "git", "--params", @' +"'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'`" +'@) # pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -choco install --no-progress -y choco-cleaner python vcredist-all windows-sdk-10-version-1809-windbg +RunProcessChecked "choco" @("install", "--no-progress", "-y", "choco-cleaner", "python", "vcredist-all", "windows-sdk-10-version-1809-windbg") # Reload our environment variables from the registry so the `git` command works Update-SessionEnvironment # Forcibly disable the git credential manager -git config --system credential.helper "" +RunProcessChecked "git" @("config", "--system", "--unset", "credential.helper") # Gather the required DirectX runtime files, since Windows Server Core does not include them Invoke-WebRequest -Uri "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" -OutFile "$env:TEMP\directx_redist.exe" -Start-Process -FilePath "$env:TEMP\directx_redist.exe" -ArgumentList "/Q", "/T:$env:TEMP" -Wait -expand "$env:TEMP\APR2007_xinput_x64.cab" -F:xinput1_3.dll C:\Windows\System32\ -expand "$env:TEMP\Jun2010_D3DCompiler_43_x64.cab" -F:D3DCompiler_43.dll C:\Windows\System32\ -expand "$env:TEMP\Feb2010_X3DAudio_x64.cab" -F:X3DAudio1_7.dll C:\Windows\System32\ -expand "$env:TEMP\Jun2010_XAudio_x64.cab" -F:XAPOFX1_5.dll C:\Windows\System32\ -expand "$env:TEMP\Jun2010_XAudio_x64.cab" -F:XAudio2_7.dll C:\Windows\System32\ +RunProcessChecked "$env:TEMP\directx_redist.exe" @("/Q", "/T:$env:TEMP") +RunProcessChecked "expand" @("$env:TEMP\APR2007_xinput_x64.cab", "-F:xinput1_3.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_D3DCompiler_43_x64.cab", "-F:D3DCompiler_43.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Feb2010_X3DAudio_x64.cab", "-F:X3DAudio1_7.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_XAudio_x64.cab", "-F:XAPOFX1_5.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_XAudio_x64.cab", "-F:XAudio2_7.dll", "C:\Windows\System32\") # Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) Invoke-WebRequest -Uri "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" -OutFile "$env:TEMP\dxc.zip" @@ -97,7 +112,7 @@ $vs_args = @( ) # Install the Visual Studio Build Tools workloads and components we need -Start-Process -FilePath "$env:TEMP\vs_buildtools.exe" -ArgumentList $vs_args -Wait +RunProcessChecked "$env:TEMP\vs_buildtools.exe" $vs_args # NOTE: Install the .Net 4.5 Framework Pack via NuGet as it is no longer available via Visual Studio, but still needed # NOTE: some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 @@ -110,7 +125,7 @@ Remove-Item -LiteralPath "$env:TEMP" -Recurse -Force New-Item -Type directory -Path "$env:TEMP" # This shaves off ~300MB as of 2021-08-31 -choco-cleaner +RunProcessChecked "choco-cleaner" @("--dummy") # Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file # Just remove it, so a proper one will be generated on next NuGet run From 3607cb128140baa6c5115675458bf712f5bc7acf Mon Sep 17 00:00:00 2001 From: Bobby Uhlenbrock Date: Thu, 11 Jan 2024 09:22:01 -0500 Subject: [PATCH 383/430] Follow Epic's logic for removing debug symbols (#346) There are `pdb` files, not prefaced with `UnrealEngine`, which are expected to be present when creating a shipping build. --- .../dockerfiles/ue4-minimal/windows/split-components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py index c6412b87..73149ffc 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -48,7 +48,7 @@ def extractComponent(inputDir, outputDir, component, description, items): extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) # Extract debug symbols -symbolFiles = glob.glob(join(rootDir, "**", "*.pdb"), recursive=True) +symbolFiles = glob.glob(join(rootDir, "**", "*UnrealEditor*.pdb"), recursive=True) extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) # Extract template projects and samples From 844f88190d0a64a172847bc8155aa47d9d9abcc2 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 11 Jan 2024 17:24:25 +0300 Subject: [PATCH 384/430] Release 0.0.111 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cee82284..958f738a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.110" +version = "0.0.111" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From ce6cf9bc648348e68e561b4d68b90b75b41ddb69 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 12 Jan 2024 17:40:44 +0300 Subject: [PATCH 385/430] Fix docs link in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 324b86f3..950b7241 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

Continuous Integration • Pixel Streaming • Unreal Engine Powered Microservices

 

-**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://github.com/adamrehn/ue4-docker) to view details specific to using the ue4-docker project.** +**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://adamrehn.github.io/ue4-docker) to view details specific to using the ue4-docker project.** The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. From 92844f806b50729809a5dda8096b92aabf6909d9 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 4 Feb 2024 16:16:41 +0300 Subject: [PATCH 386/430] Fix code format --- src/ue4docker/build.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index bd8c054b..a3146dfe 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -111,9 +111,11 @@ def build(): # Print the command-line invocation that triggered this build, masking any supplied passwords args = [ - "*******" - if config.args.password is not None and arg == config.args.password - else arg + ( + "*******" + if config.args.password is not None and arg == config.args.password + else arg + ) for arg in sys.argv ] logger.info("COMMAND-LINE INVOCATION:", False) From 9248588a3a8885d10c187a1f52bf8d1d660b3df0 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Sun, 4 Feb 2024 16:18:20 +0300 Subject: [PATCH 387/430] Fix code format --- src/ue4docker/info.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/ue4docker/info.py b/src/ue4docker/info.py index 3420f457..eaf7c411 100644 --- a/src/ue4docker/info.py +++ b/src/ue4docker/info.py @@ -61,9 +61,11 @@ def info(): "ue4-docker version", "{}{}".format( __version__, - "" - if latestVersion is None - else " (latest available version is {})".format(latestVersion), + ( + "" + if latestVersion is None + else " (latest available version is {})".format(latestVersion) + ), ), ), ("Operating system", _osName(dockerInfo)), From 4c0491cbecdca958cd7e3e063dde79fa15c3e939 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 17:04:27 +0300 Subject: [PATCH 388/430] Bump release-drafter/release-drafter from 5 to 6 (#349) Bumps [release-drafter/release-drafter](https://github.com/release-drafter/release-drafter) from 5 to 6. - [Release notes](https://github.com/release-drafter/release-drafter/releases) - [Commits](https://github.com/release-drafter/release-drafter/compare/v5...v6) --- updated-dependencies: - dependency-name: release-drafter/release-drafter dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-drafter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 86e477cd..f8bae4b3 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -8,6 +8,6 @@ jobs: update_release_draft: runs-on: ubuntu-latest steps: - - uses: release-drafter/release-drafter@v5 + - uses: release-drafter/release-drafter@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f249d1e8a49f5ba4096dc3778bf23e0f57120090 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 6 Feb 2024 19:51:28 +1000 Subject: [PATCH 389/430] Add test suite to build images for all supported UE releases --- src/ue4docker/clean.py | 6 +- test-suite/credentials/.gitignore | 2 + test-suite/test-ue-releases.py | 188 ++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 test-suite/credentials/.gitignore create mode 100755 test-suite/test-ue-releases.py diff --git a/src/ue4docker/clean.py b/src/ue4docker/clean.py index d01edfe8..016a3338 100644 --- a/src/ue4docker/clean.py +++ b/src/ue4docker/clean.py @@ -1,4 +1,4 @@ -import argparse, subprocess, sys +import argparse, itertools, subprocess, sys from .infrastructure import * @@ -11,7 +11,9 @@ def _isIntermediateImage(image): def _cleanMatching(cleaner, filter, tag, dryRun): tagSuffix = ":{}".format(tag) if tag is not None else "*" matching = DockerUtils.listImages(tagFilter=filter + tagSuffix) - cleaner.cleanMultiple([image.tags[0] for image in matching], dryRun) + cleaner.cleanMultiple( + itertools.chain.from_iterable([image.tags for image in matching]), dryRun + ) def clean(): diff --git a/test-suite/credentials/.gitignore b/test-suite/credentials/.gitignore new file mode 100644 index 00000000..e45e01ad --- /dev/null +++ b/test-suite/credentials/.gitignore @@ -0,0 +1,2 @@ +username.txt +password.txt diff --git a/test-suite/test-ue-releases.py b/test-suite/test-ue-releases.py new file mode 100755 index 00000000..491a2294 --- /dev/null +++ b/test-suite/test-ue-releases.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +import argparse, os, platform, subprocess, sys, traceback +from pathlib import Path + +try: + import colorama + from termcolor import colored +except: + print( + "Error: could not import colorama and termcolor! Make sure you install ue4-docker at least once before running the test suite." + ) + sys.exit(1) + + +class UERelease: + def __init__( + self, name: str, tag: str, repo: str, vsVersion: int, ubuntuVersion: str + ) -> None: + self.name = name + self.tag = tag + self.repo = repo + self.vsVersion = vsVersion + self.ubuntuVersion = ubuntuVersion + + +# Older releases have broken tags in the upstream Unreal Engine repository, so we use a fork with the updated `Commit.gitdeps.xml` files +UPSTREAM_REPO = "https://github.com/EpicGames/UnrealEngine.git" +COMMITDEPS_REPO = "https://github.com/adamrehn/UnrealEngine.git" + +# The list of Unreal Engine releases that are currently supported by ue4-docker +SUPPORTED_RELEASES = [ + UERelease("4.22", "4.22.3-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("4.23", "4.23.1-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("4.24", "4.24.3-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("4.25", "4.25.4-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("4.26", "4.26.2-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, 2019, "20.04"), + UERelease("5.1", "5.1.1-fixed", COMMITDEPS_REPO, 2019, None), + UERelease("5.2", "5.2.1-release", UPSTREAM_REPO, 2022, None), + UERelease("5.3", "5.3.2-release", UPSTREAM_REPO, 2022, None), +] + + +# Logs a message with the specified colour, making it bold to distinguish it from `ue4-docker build` log output +def log(message: str, colour: str): + print(colored(message, color=colour, attrs=["bold"]), file=sys.stderr, flush=True) + + +# Logs a command and runs it +def run(dryRun: bool, command: str, **kwargs: dict) -> subprocess.CompletedProcess: + log(command, colour="green") + if not dryRun: + return subprocess.run(command, **{"check": True, **kwargs}) + else: + return subprocess.CompletedProcess(command, 0) + + +# Runs our tests for the specified Unreal Engine release +def testRelease( + release: UERelease, username: str, token: str, keepImages: bool, dryRun: bool +) -> None: + + # Pass the supplied credentials to the build process via environment variables + environment = { + **os.environ, + "UE4DOCKER_USERNAME": username, + "UE4DOCKER_PASSWORD": token, + } + + # Generate the command to build the ue4-minimal image (and its dependencies) for the specified Unreal Engine release + command = [ + sys.executable, + "-m", + "ue4docker", + "build", + "--ue-version", + f"custom:{release.name}", + "-repo", + release.repo, + "-branch", + release.tag, + "--target", + "minimal", + ] + + # Apply any platform-specific flags + if platform.system() == "Windows": + command += ["--visual-studio", release.vsVersion] + elif release.ubuntuVersion is not None: + command += ["-basetag", f"ubuntu{release.ubuntuVersion}"] + + # Attempt to run the build + run( + dryRun, + command, + env=environment, + ) + + # Unless requested otherwise, remove the built images to free up disk space + if not keepImages: + run( + dryRun, + [ + sys.executable, + "-m", + "ue4docker", + "clean", + "-tag", + release.name, + "--all", + "--prune", + "--dry-run", + ], + ) + + +if __name__ == "__main__": + + try: + # Initialise coloured log output under Windows + colorama.init() + + # Resolve the paths to our input directories + testDir = Path(__file__).parent + credentialsDir = testDir / "credentials" + repoRoot = testDir.parent + + # Parse our command-line arguments + parser = argparse.ArgumentParser() + parser.add_argument( + "--releases", + default=None, + help="Run tests for the specified Unreal Engine releases (comma-delimited, defaults to all supported releases)", + ) + parser.add_argument( + "--keep-images", + action="store_true", + help="Don't remove images after they are built (uses more disk space)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print commands instead of running them", + ) + args = parser.parse_args() + + # Parse and validate the specified list of releases + if args.releases is not None: + testQueue = [] + versions = args.releases.split(",") + for version in versions: + found = [r for r in SUPPORTED_RELEASES if r.name == version] + if len(found) == 1: + testQueue.append(found[0]) + else: + raise RuntimeError(f'unsupported Unreal Engine release "{version}"') + else: + testQueue = SUPPORTED_RELEASES + + # Read the GitHub username from the credentials directory + usernameFile = credentialsDir / "username.txt" + if usernameFile.exists(): + username = usernameFile.read_text("utf-8").strip() + else: + raise RuntimeError(f"place GitHub username in the file {str(usernameFile)}") + + # Read the GitHub Personal Access Token (PAT) from the credentials directory + tokenFile = credentialsDir / "password.txt" + if tokenFile.exists(): + token = tokenFile.read_text("utf-8").strip() + else: + raise RuntimeError( + f"place GitHub Personal Access Token (PAT) in the file {str(tokenFile)}" + ) + + # Ensure any local changes to ue4-docker are installed + run( + args.dry_run, + [sys.executable, "-m", "pip", "install", "--user", str(repoRoot)], + ) + + # Run the tests for each of the selected Unreal Engine releases + for release in testQueue: + testRelease(release, username, token, args.keep_images, args.dry_run) + + except Exception as e: + log(traceback.format_exc(), colour="red") From 84ca65ca9226996b402eb426b0d39199197e2035 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 6 Feb 2024 19:53:35 +1000 Subject: [PATCH 390/430] 5.4 prep: propagate custom Build IDs under Linux --- README.md | 2 +- .../ue4-build-prerequisites/linux/Dockerfile | 1 + .../dockerfiles/ue4-minimal/linux/Dockerfile | 19 ++++++++++++++----- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 950b7241..566bbb25 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The ue4-docker Python package contains a set of Dockerfiles and accompanying bui Key features include: -- Unreal Engine 4.20.0 and newer is supported. +- Unreal Engine 4.22.0 and newer is supported. - Both Windows containers and Linux containers are supported. - Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index d73a7d7a..b41a644c 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -37,6 +37,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tzdata \ unzip \ xdg-user-dirs \ + xdg-utils \ zip && \ rm -rf /var/lib/apt/lists/* diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 2a752787..9f67c5d2 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -38,17 +38,23 @@ RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Developme # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ - -target="Make Installed Build Linux" \ - -script=Engine/Build/InstalledEngineBuild.xml \ - -set:HostPlatformOnly=true \ - -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} \ - {{ buildgraph_args }} && \ + -target="Make Installed Build Linux" \ + -script=Engine/Build/InstalledEngineBuild.xml \ + -set:HostPlatformOnly=true \ + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} \ + {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers COPY split-components.py /tmp/split-components.py RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components +# Copy InstalledBuild.txt from the Installed Build and run UnrealVersionSelector to populate Install.ini with any custom Build ID specified in the BuildGraph flags +# (Note that the `-unattended` flag used below requires Unreal Engine 4.22 or newer, so this will break under older versions) +# (Note also that custom Build IDs are supported by Unreal Engine 5.3.1 and newer, and older versions will just use a GUID as the Build ID) +RUN cp /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Build/InstalledBuild.txt /home/ue4/UnrealEngine/Engine/Build/InstalledBuild.txt && \ + ./Engine/Binaries/Linux/UnrealVersionSelector-Linux-Shipping -register -unattended + {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini COPY fix-targets.py /tmp/fix-targets.py @@ -85,6 +91,9 @@ COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbo {% if excluded_components.templates == false %} COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine {% endif %} + +# Copy Install.ini from the builder image, so it can be used by tools that read the list of engine installations (e.g. ushell) +COPY --from=builder --chown=ue4:ue4 /home/ue4/.config/Epic/UnrealEngine/Install.ini /home/ue4/.config/Epic/UnrealEngine/Install.ini WORKDIR /home/ue4/UnrealEngine {% if not disable_labels %} From bb44193f9942aa7a099d402101cac4af4100a9d1 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 6 Feb 2024 19:54:11 +1000 Subject: [PATCH 391/430] Bump copyright year --- LICENSE | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 40b1f548..49363f78 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 - 2021 Adam Rehn +Copyright (c) 2018 - 2024 Adam Rehn Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 566bbb25..affc156a 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ See the file [CONTRIBUTING.adoc](https://github.com/adamrehn/ue4-docker/blob/mas ## Legal -Copyright © 2018 - 2021, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. +Copyright © 2018 - 2024, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. From 4d1cec3277fc7208ae05b734be2236634b5b23e1 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 12 Feb 2024 11:26:09 +1000 Subject: [PATCH 392/430] Print warning when WSL is detected --- src/ue4docker/build.py | 29 ++++++++++++++++++-- src/ue4docker/info.py | 6 ++-- src/ue4docker/infrastructure/WindowsUtils.py | 9 +++++- 3 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index a3146dfe..2b97c95c 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -1,4 +1,4 @@ -import argparse, getpass, humanfriendly, json, os, shutil, sys, tempfile, time +import argparse, getpass, humanfriendly, json, os, platform, shutil, sys, tempfile, time from .infrastructure import * from .version import __version__ from os.path import join @@ -299,12 +299,35 @@ def build(): else: logger.info("Not excluding any Engine components.", False) + # Print a warning if the user is attempting to build Linux images under Windows + if config.containerPlatform == "linux" and ( + platform.system() == "Windows" or WindowsUtils.isWSL() + ): + logger.warning( + "Warning: attempting to build Linux container images under Windows (e.g. via WSL)." + ) + logger.warning( + "The ue4-docker maintainers do not provide support for building and running Linux", + False, + ) + logger.warning( + "containers under Windows, and this configuration is not tested to verify that it", + False, + ) + logger.warning( + "functions correctly. Users are solely responsible for troubleshooting any issues", + False, + ) + logger.warning( + "they encounter when attempting to build Linux container images under Windows.", + False, + ) + # Determine if we need to prompt for credentials if config.dryRun == True: # Don't bother prompting the user for any credentials during a dry run logger.info( - "Performing a dry run, `docker build` commands will be printed and not executed.", - False, + "Performing a dry run, `docker build` commands will be printed and not executed." ) username = "" password = "" diff --git a/src/ue4docker/info.py b/src/ue4docker/info.py index eaf7c411..9399514d 100644 --- a/src/ue4docker/info.py +++ b/src/ue4docker/info.py @@ -9,8 +9,10 @@ def _osName(dockerInfo): elif platform.system() == "Darwin": return DarwinUtils.systemString() else: - return "Linux ({}, {})".format( - dockerInfo["OperatingSystem"], dockerInfo["KernelVersion"] + return "Linux ({}, {}{})".format( + dockerInfo["OperatingSystem"], + dockerInfo["KernelVersion"], + ", running under WSL" if WindowsUtils.isWSL() else "", ) diff --git a/src/ue4docker/infrastructure/WindowsUtils.py b/src/ue4docker/infrastructure/WindowsUtils.py index fef462b6..7b47130a 100644 --- a/src/ue4docker/infrastructure/WindowsUtils.py +++ b/src/ue4docker/infrastructure/WindowsUtils.py @@ -1,6 +1,6 @@ from .DockerUtils import DockerUtils from packaging.version import Version -import platform, sys +import os, platform, sys from typing import Optional if platform.system() == "Windows": @@ -108,6 +108,13 @@ def isWindowsServer() -> bool: # TODO: Replace this with something more reliable return "Windows Server" in WindowsUtils._getVersionRegKey("ProductName") + @staticmethod + def isWSL() -> bool: + """ + Determines if the host system is Linux running under WSL + """ + return "WSL_DISTRO_NAME" in os.environ or "WSL_INTEROP" in os.environ + @staticmethod def getDllSrcImage(basetag: str) -> str: """ From 6a840017ba037281627ee6e07ca4af168cc414d2 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 12 Feb 2024 15:39:48 +1000 Subject: [PATCH 393/430] Remove UE 4.25 and older from the test matrix --- README.md | 4 ++-- test-suite/test-ue-releases.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index affc156a..a3c0d436 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,11 @@ The ue4-docker Python package contains a set of Dockerfiles and accompanying bui Key features include: -- Unreal Engine 4.22.0 and newer is supported. +- The six most recent versions of the Unreal Engine are supported (currently Unreal Engine 4.26 and newer). - Both Windows containers and Linux containers are supported. - Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. -- Running built Unreal Engine projects with offscreen rendering is supported via NVIDIA Docker under Linux. +- Running built Unreal Engine projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. Resources: diff --git a/test-suite/test-ue-releases.py b/test-suite/test-ue-releases.py index 491a2294..a3931f9a 100755 --- a/test-suite/test-ue-releases.py +++ b/test-suite/test-ue-releases.py @@ -29,10 +29,6 @@ def __init__( # The list of Unreal Engine releases that are currently supported by ue4-docker SUPPORTED_RELEASES = [ - UERelease("4.22", "4.22.3-fixed", COMMITDEPS_REPO, 2017, None), - UERelease("4.23", "4.23.1-fixed", COMMITDEPS_REPO, 2017, None), - UERelease("4.24", "4.24.3-fixed", COMMITDEPS_REPO, 2017, None), - UERelease("4.25", "4.25.4-fixed", COMMITDEPS_REPO, 2017, None), UERelease("4.26", "4.26.2-fixed", COMMITDEPS_REPO, 2017, None), UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, 2017, None), UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, 2019, "20.04"), From 63c84bb4b28c4154712b30b0d7484bb3659cfa6d Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 16 Feb 2024 20:40:55 +1000 Subject: [PATCH 394/430] 5.4 prep: add experimental ushell support under Linux --- .../ue4-build-prerequisites/linux/Dockerfile | 14 ++++++++++++++ .../dockerfiles/ue4-minimal/linux/Dockerfile | 19 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index b41a644c..d4dba0af 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -28,6 +28,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl \ git \ git-lfs \ + gpg-agent \ python3 \ python3-dev \ python3-pip \ @@ -93,5 +94,18 @@ RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ usermod -a -G audio,video,sudo ue4 USER ue4 +{% if enable_ushell %} +# Install Python 3.12, which is required by ushell +USER root +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends python3.12 python3.12-venv && \ + rm -rf /var/lib/apt/lists/* +USER ue4 + +# Install a copy of pip for Python 3.12 +RUN curl -fsSL 'https://bootstrap.pypa.io/get-pip.py' | python3.12 +{% endif %} + # Enable Git Large File Storage (LFS) support RUN git lfs install diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 9f67c5d2..38cd9d6b 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -49,6 +49,9 @@ RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ COPY split-components.py /tmp/split-components.py RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components +# Ensure UnrealVersionSelector is built, since the prebuilt binaries may not be up-to-date +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealVersionSelector Linux Shipping + # Copy InstalledBuild.txt from the Installed Build and run UnrealVersionSelector to populate Install.ini with any custom Build ID specified in the BuildGraph flags # (Note that the `-unattended` flag used below requires Unreal Engine 4.22 or newer, so this will break under older versions) # (Note also that custom Build IDs are supported by Unreal Engine 5.3.1 and newer, and older versions will just use a GUID as the Build ID) @@ -72,6 +75,13 @@ COPY --chown=ue4:ue4 copy-toolchain.py /tmp/copy-toolchain.py RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine {% endif %} +{% if enable_ushell %} +# Ensure ushell is copied to the Installed Build +RUN rm -rf ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ + cp -r ./Engine/Extras/ushell ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ + bash -c 'set -e; shopt -s globstar; cd /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Extras/ushell && chmod +x ./**/*.sh' +{% endif %} + # Copy the Installed Build into a clean image, discarding the source build {% if combine %} FROM prerequisites as minimal @@ -103,6 +113,15 @@ LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %} {% endif %} +{% if enable_ushell %} +# Add ushell to the system PATH and alias `ushell` to `ushell.sh` +ENV PATH="$PATH:/home/ue4/UnrealEngine/Engine/Extras/ushell" +RUN echo 'alias ushell="ushell.sh"' >> /home/ue4/.bashrc + +# Perform first-run setup for ushell +RUN bash -c 'set -e; source /home/ue4/UnrealEngine/Engine/Extras/ushell/ushell.sh && exit 0' +{% endif %} + # Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` # (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) # See the comments on this issue for details, including the need to ensure $HOME is set correctly: From d07faec9dde322643c4736c59fbd1b9adbd64767 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 16 Feb 2024 20:41:50 +1000 Subject: [PATCH 395/430] Bump version to 0.0.112 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 958f738a..a502e241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.111" +version = "0.0.112" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 5280cef8eebe777dde1eb8fd0d2375ae0978f4bb Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 18 Mar 2024 17:11:23 +1000 Subject: [PATCH 396/430] 5.4 compat: add new Windows dependencies --- .../dockerfiles/ue4-build-prerequisites/windows/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 47a6ec7a..280c044a 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -22,6 +22,7 @@ COPY --from=dlls ` C:\Windows\System32\dsound.dll ` C:\Windows\System32\dxva2.dll ` C:\Windows\System32\glu32.dll ` + C:\Windows\System32\InputHost.dll ` C:\Windows\System32\ksuser.dll ` C:\Windows\System32\mf.dll ` C:\Windows\System32\mfcore.dll ` @@ -34,6 +35,7 @@ COPY --from=dlls ` C:\Windows\System32\opengl32.dll ` C:\Windows\System32\ResampleDMO.dll ` C:\Windows\System32\ResourcePolicyClient.dll ` + C:\Windows\System32\XInput1_4.dll ` C:\GatheredDLLs\ # Remove any DLL files that already exist in the target base image, to avoid permission errors when attempting to overwrite existing files with a COPY directive From edbb68c7eae1da63e7ec63f5d623f7d484ddb79c Mon Sep 17 00:00:00 2001 From: Jeff Smith Date: Fri, 5 Apr 2024 15:54:04 -0500 Subject: [PATCH 397/430] Windows: lock git version since choco version is locked and fails to install newest (#356) --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index b4479331..9e36705c 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -21,7 +21,7 @@ $env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" # Install the chocolatey packages we need -RunProcessChecked "choco" @("install", "--no-progress", "-y", "git", "--params", @' +RunProcessChecked "choco" @("install", "--no-progress", "-y", "git.install", "--version=2.43.0", "--params", @' "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'`" '@) From f37672bc53f37599f5b286d5604ccdcf6c8fb431 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 6 May 2024 11:04:35 +0300 Subject: [PATCH 398/430] Add changelist for UE-5.4 --- src/ue4docker/infrastructure/BuildConfiguration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 092d38fa..6aabd1fb 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -40,6 +40,7 @@ "5.1.0": 23058290, "5.2.0": 25360045, "5.3.0": 27405482, + "5.4.0": 33043543, } @@ -67,6 +68,7 @@ class VisualStudio(object): } UnsupportedSince = {VS2017: Version("5.0")} + UnsupportedSince = {VS2019: Version("5.4")} class ExcludedComponent(object): From 8a977627b57c15d8b4d3f96c6b1bb2a27e95e1f5 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 6 May 2024 11:07:05 +0300 Subject: [PATCH 399/430] Fix configuration of unsupported VS versions --- src/ue4docker/infrastructure/BuildConfiguration.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 6aabd1fb..e438e8cd 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -67,8 +67,10 @@ class VisualStudio(object): VS2022: Version("5.0.0"), } - UnsupportedSince = {VS2017: Version("5.0")} - UnsupportedSince = {VS2019: Version("5.4")} + UnsupportedSince = { + VS2017: Version("5.0"), + VS2019: Version("5.4"), + } class ExcludedComponent(object): From 9e2dfc38ef18a83205b1856b37f48c12f307b0e3 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 23 May 2024 00:34:31 -0700 Subject: [PATCH 400/430] Add temporary workaround for https://github.com/docker/docker-py/issues/3256 --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index a502e241..0fc24e21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,8 @@ dependencies = [ "Jinja2>=2.11.3", "packaging>=19.1", "psutil", + # Workaround for https://github.com/docker/docker-py/issues/3256 + "requests<2.32", "termcolor", ] From 9cebfa84a107e62e3c3cd6e72253f1f8afb50d81 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 23 May 2024 00:35:03 -0700 Subject: [PATCH 401/430] Refactor Visual Studio support --- src/ue4docker/build.py | 3 +- .../infrastructure/BuildConfiguration.py | 94 +++++++++++-------- 2 files changed, 59 insertions(+), 38 deletions(-) diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index 2b97c95c..b1ecf7f2 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -396,7 +396,8 @@ def build(): "--build-arg", "DLLSRCIMAGE=" + config.dllSrcImage, "--build-arg", - "VISUAL_STUDIO_BUILD_NUMBER=" + config.visualStudioBuildNumber, + "VISUAL_STUDIO_BUILD_NUMBER=" + + config.visualStudio.build_number, ] custom_prerequisites_dockerfile = config.args.prerequisites_dockerfile diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index e438e8cd..d55f9003 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -45,32 +45,52 @@ class VisualStudio(object): - VS2017 = "2017" - VS2019 = "2019" - VS2022 = "2022" - - BuildNumbers = { - VS2017: "15", - VS2019: "16", - VS2022: "17", - } - - SupportedSince = { + def __init__( + self, + name: str, + build_number: str, + supported_since: Version, + unsupported_since: Version | None, + pass_version_to_buildgraph: bool, + ): + self.name = name + self.build_number = build_number + self.supported_since = supported_since + self.unsupported_since = unsupported_since + self.pass_version_to_buildgraph = pass_version_to_buildgraph + + +DefaultVisualStudio = "2017" + +VisualStudios = { + "2017": VisualStudio( + name="2017", + build_number="15", # We do not support versions older than 4.20 - VS2017: Version("4.20"), + supported_since=Version("4.20"), + unsupported_since=Version("5.0"), + pass_version_to_buildgraph=False, + ), + "2019": VisualStudio( + name="2019", + build_number="16", # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa # # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 - VS2019: Version("4.25"), - VS2022: Version("5.0.0"), - } - - UnsupportedSince = { - VS2017: Version("5.0"), - VS2019: Version("5.4"), - } + supported_since=Version("4.25"), + unsupported_since=Version("5.4"), + pass_version_to_buildgraph=True, + ), + "2022": VisualStudio( + name="2022", + build_number="17", + supported_since=Version("5.0"), + unsupported_since=None, + pass_version_to_buildgraph=True, + ), +} class ExcludedComponent(object): @@ -184,8 +204,8 @@ def addArguments(parser): ) parser.add_argument( "--visual-studio", - default=VisualStudio.VS2017, - choices=VisualStudio.BuildNumbers.keys(), + default=DefaultVisualStudio, + choices=VisualStudios.keys(), help="Specify Visual Studio Build Tools version to use for Windows containers", ) parser.add_argument( @@ -591,24 +611,25 @@ def describeExcludedComponents(self): ) def _generateWindowsConfig(self): - self.visualStudio = self.args.visual_studio + self.visualStudio = VisualStudios.get(self.args.visual_studio) + if self.visualStudio is None: + raise RuntimeError( + f"unknown Visual Studio version: {self.args.visual_studio}" + ) if self.release is not None and not self.custom: # Check whether specified Unreal Engine release is compatible with specified Visual Studio - supportedSince = VisualStudio.SupportedSince.get(self.visualStudio, None) - if supportedSince is not None and Version(self.release) < supportedSince: + if ( + self.visualStudio.supported_since is not None + and Version(self.release) < self.visualStudio.supported_since + ): raise RuntimeError( - "specified version of Unreal Engine is too old for Visual Studio {}".format( - self.visualStudio - ) + f"specified version of Unreal Engine is too old for Visual Studio {self.visualStudio.name}" ) - unsupportedSince = VisualStudio.UnsupportedSince.get( - self.visualStudio, None - ) if ( - unsupportedSince is not None - and Version(self.release) >= unsupportedSince + self.visualStudio.unsupported_since is not None + and Version(self.release) >= self.visualStudio.unsupported_since ): raise RuntimeError( "Visual Studio {} is too old for specified version of Unreal Engine".format( @@ -616,14 +637,13 @@ def _generateWindowsConfig(self): ) ) - self.visualStudioBuildNumber = VisualStudio.BuildNumbers[self.visualStudio] # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. # Otherwise, UAT errors out with "Unknown argument: VS2019". - if self.visualStudio != VisualStudio.VS2017: + if self.visualStudio.pass_version_to_buildgraph: self.opts["buildgraph_args"] = ( self.opts.get("buildgraph_args", "") - + f" -set:VS{self.visualStudio}=true" + + f" -set:VS{self.visualStudio.name}=true" ) # Determine base tag for the Windows release of the host system @@ -641,7 +661,7 @@ def _generateWindowsConfig(self): self.baseImage = "mcr.microsoft.com/windows/servercore:" + self.basetag self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) - self.prereqsTag = self.basetag + "-vs" + self.visualStudio + self.prereqsTag = self.basetag + "-vs" + self.visualStudio.name # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect if self.args.isolation is not None: From 36b00cdf14e5d7db72e49f1d138c086f113779ab Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 23 May 2024 15:14:54 +0300 Subject: [PATCH 402/430] Fix compatibility with older Python --- src/ue4docker/infrastructure/BuildConfiguration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index d55f9003..5de35728 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -1,6 +1,7 @@ import json import platform import random +from typing import Optional import humanfriendly from packaging.version import Version, InvalidVersion @@ -50,7 +51,7 @@ def __init__( name: str, build_number: str, supported_since: Version, - unsupported_since: Version | None, + unsupported_since: Optional[Version], pass_version_to_buildgraph: bool, ): self.name = name From 41a252cd68cb5689509cd72c6d03b05af4b22044 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 23 May 2024 15:15:31 +0300 Subject: [PATCH 403/430] Revert "Add temporary workaround for https://github.com/docker/docker-py/issues/3256" Just-released docker-py 7.1.0 contains the fix: https://github.com/docker/docker-py/releases/tag/7.1.0 This reverts commit 9e2dfc38ef18a83205b1856b37f48c12f307b0e3. --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0fc24e21..a502e241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,6 @@ dependencies = [ "Jinja2>=2.11.3", "packaging>=19.1", "psutil", - # Workaround for https://github.com/docker/docker-py/issues/3256 - "requests<2.32", "termcolor", ] From ea67de4596281bbdf0cb4105fbe76c8da444b141 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Fri, 24 May 2024 15:30:49 +0300 Subject: [PATCH 404/430] Bump version to 0.0.113 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a502e241..75dcec92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.112" +version = "0.0.113" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 1aa50f2694754d74221576dea228e8d28a1fa2ad Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 18 Jun 2024 11:52:26 +1000 Subject: [PATCH 405/430] Warn users about IPv6 issues when using Docker < 26.0.0 --- src/ue4docker/build.py | 18 ++++++++++++++++++ src/ue4docker/infrastructure/DockerUtils.py | 10 ++++++++++ 2 files changed, 28 insertions(+) diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index b1ecf7f2..72ae3657 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -86,6 +86,24 @@ def build(): ) sys.exit(1) + # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 images without config changes + if ( + config.containerPlatform == "linux" + and DockerUtils.isVersionWithoutIPV6Loopback() + ): + logger.warning( + "\n".join( + [ + "Warning: detected a Docker version older than 26.0.0.", + "Older versions of Docker cannot build or run images for Unreal Engine 5.4 or", + "newer unless the Docker daemon is explicitly configured to enable IPv6 support.", + "For details, see: https://github.com/adamrehn/ue4-docker/issues/357", + "", + ] + ), + False, + ) + # Create an auto-deleting temporary directory to hold our build context with tempfile.TemporaryDirectory() as tempDir: contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles") diff --git a/src/ue4docker/infrastructure/DockerUtils.py b/src/ue4docker/infrastructure/DockerUtils.py index 54b447bf..05155935 100644 --- a/src/ue4docker/infrastructure/DockerUtils.py +++ b/src/ue4docker/infrastructure/DockerUtils.py @@ -1,5 +1,6 @@ import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re from docker.models.containers import Container +from packaging.version import Version from .FilesystemUtils import FilesystemUtils @@ -32,6 +33,15 @@ def info(): client = docker.from_env() return client.info() + @staticmethod + def isVersionWithoutIPV6Loopback(): + """ + Determines if the version of the Docker daemon lacks support for using the + IPv6 loopback address [::1] when using its default network configuration + """ + dockerVersion = Version(DockerUtils.version()["Version"]) + return dockerVersion < Version("26.0.0") + @staticmethod def exists(name): """ From f495b4571c3123c97d79168e2bc36810b33901ad Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Tue, 18 Jun 2024 20:36:54 +1000 Subject: [PATCH 406/430] Split subdirectories into separate layers under Linux (see #359) --- .../dockerfiles/ue4-minimal/linux/Dockerfile | 15 +++++++++++---- .../ue4-minimal/linux/split-components.py | 10 ++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 38cd9d6b..798a786c 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -45,10 +45,6 @@ RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC -# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers -COPY split-components.py /tmp/split-components.py -RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components - # Ensure UnrealVersionSelector is built, since the prebuilt binaries may not be up-to-date RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealVersionSelector Linux Shipping @@ -82,6 +78,11 @@ RUN rm -rf ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ bash -c 'set -e; shopt -s globstar; cd /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Extras/ushell && chmod +x ./**/*.sh' {% endif %} +# Split out both optional components (DDC, debug symbols, template projects) and large subdirectories so they can be copied +# into the final container image as separate filesystem layers, avoiding creating a single monolithic layer with everything +COPY split-components.py /tmp/split-components.py +RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components + # Copy the Installed Build into a clean image, discarding the source build {% if combine %} FROM prerequisites as minimal @@ -92,6 +93,12 @@ FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} # Copy the Installed Build files from the builder image COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Binaries /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Content /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Extras /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Intermediate /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Plugins /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Source /home/ue4/UnrealEngine {% if excluded_components.ddc == false %} COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DDC /home/ue4/UnrealEngine {% endif %} diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py index 742ce5aa..9c10a6b5 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py @@ -58,3 +58,13 @@ def extractComponent(inputDir, outputDir, component, description, items): extractComponent( rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs ) + +# Extract the larger non-optional subdirectories of the Engine directory +for subdir in ["Binaries", "Content", "Extras", "Intermediate", "Plugins", "Source"]: + extractComponent( + rootDir, + outputDir, + subdir, + f"{subdir} subdirectory", + [join(rootDir, "Engine", subdir)], + ) From 070dd3c617eaf31c3b237638301006f3c6ee6b26 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 21 Jun 2024 15:33:13 +1000 Subject: [PATCH 407/430] Add diagnostic for IPv6 support (see #357) --- src/ue4docker/diagnostics/__init__.py | 1 + src/ue4docker/diagnostics/base.py | 28 +++---- src/ue4docker/diagnostics/diagnostic_8gig.py | 2 +- src/ue4docker/diagnostics/diagnostic_all.py | 2 + src/ue4docker/diagnostics/diagnostic_ipv6.py | 76 +++++++++++++++++++ .../diagnostics/diagnostic_network.py | 2 +- src/ue4docker/diagnostics_cmd.py | 1 + .../diagnostics/ipv6/linux/Dockerfile | 7 ++ 8 files changed, 104 insertions(+), 15 deletions(-) create mode 100644 src/ue4docker/diagnostics/diagnostic_ipv6.py create mode 100644 src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile diff --git a/src/ue4docker/diagnostics/__init__.py b/src/ue4docker/diagnostics/__init__.py index f2c813bd..68997c6a 100644 --- a/src/ue4docker/diagnostics/__init__.py +++ b/src/ue4docker/diagnostics/__init__.py @@ -1,4 +1,5 @@ from .diagnostic_all import allDiagnostics from .diagnostic_8gig import diagnostic8Gig from .diagnostic_20gig import diagnostic20Gig +from .diagnostic_ipv6 import diagnosticIPv6 from .diagnostic_network import diagnosticNetwork diff --git a/src/ue4docker/diagnostics/base.py b/src/ue4docker/diagnostics/base.py index e538dcd1..a77f91e9 100644 --- a/src/ue4docker/diagnostics/base.py +++ b/src/ue4docker/diagnostics/base.py @@ -37,7 +37,7 @@ def _printAndRun(self, logger, prefix, command, check=False): logger.info(prefix + "Run: {}".format(command), False) subprocess.run(command, check=check) - def _checkPlatformMistmatch(self, logger, containerPlatform): + def _checkPlatformMistmatch(self, logger, containerPlatform, linuxFlagSupported): """ Verifies that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) """ @@ -51,12 +51,13 @@ def _checkPlatformMistmatch(self, logger, containerPlatform): ), False, ) - logger.error( - "[{}] Use the --linux flag if you want to test Linux containers instead.".format( - prefix - ), - False, - ) + if linuxFlagSupported: + logger.error( + "[{}] Use the --linux flag if you want to test Linux containers instead.".format( + prefix + ), + False, + ) raise RuntimeError elif containerPlatform == "linux" and dockerPlatform == "windows": logger.error( @@ -65,12 +66,13 @@ def _checkPlatformMistmatch(self, logger, containerPlatform): ), False, ) - logger.error( - "[{}] Remove the --linux flag if you want to test Windows containers instead.".format( - prefix - ), - False, - ) + if linuxFlagSupported: + logger.error( + "[{}] Remove the --linux flag if you want to test Windows containers instead.".format( + prefix + ), + False, + ) raise RuntimeError def _generateWindowsBuildArgs( diff --git a/src/ue4docker/diagnostics/diagnostic_8gig.py b/src/ue4docker/diagnostics/diagnostic_8gig.py index 0662034d..b6a464d9 100644 --- a/src/ue4docker/diagnostics/diagnostic_8gig.py +++ b/src/ue4docker/diagnostics/diagnostic_8gig.py @@ -80,7 +80,7 @@ def run(self, logger, args=[]): # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) try: - self._checkPlatformMistmatch(logger, containerPlatform) + self._checkPlatformMistmatch(logger, containerPlatform, True) except RuntimeError: return False diff --git a/src/ue4docker/diagnostics/diagnostic_all.py b/src/ue4docker/diagnostics/diagnostic_all.py index e94cf5ff..4872cd42 100644 --- a/src/ue4docker/diagnostics/diagnostic_all.py +++ b/src/ue4docker/diagnostics/diagnostic_all.py @@ -1,6 +1,7 @@ from .base import DiagnosticBase from .diagnostic_8gig import diagnostic8Gig from .diagnostic_20gig import diagnostic20Gig +from .diagnostic_ipv6 import diagnosticIPv6 from .diagnostic_network import diagnosticNetwork @@ -27,6 +28,7 @@ def run(self, logger, args=[]): diagnostics = [ diagnostic8Gig(), diagnostic20Gig(), + diagnosticIPv6(), diagnosticNetwork(), ] for index, diagnostic in enumerate(diagnostics): diff --git a/src/ue4docker/diagnostics/diagnostic_ipv6.py b/src/ue4docker/diagnostics/diagnostic_ipv6.py new file mode 100644 index 00000000..7fb78755 --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_ipv6.py @@ -0,0 +1,76 @@ +from .base import DiagnosticBase + + +class diagnosticIPv6(DiagnosticBase): + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:ipv6" + + def __init__(self): + pass + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check that Linux containers can access the IPv6 loopback address" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines whether Linux containers are able to access the IPv6,", + "loopback address ::1, which is required by Unreal Engine 5.4 and newer for", + "local ZenServer communication.", + "", + "This should work automatically under Docker 26.0.0 and newer, but older versions", + "require manual configuration by the user.", + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "ipv6" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # This diagnostic only applies to Linux containers + containerPlatform = "linux" + + # Verify that the user isn't trying to test Linux containers under Windows 10 when in Windows container mode + try: + self._checkPlatformMistmatch(logger, containerPlatform, False) + except RuntimeError: + return False + + # Attempt to build the Dockerfile + logger.action( + "[network] Attempting to build an image that accesses the IPv6 loopback address...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnosticIPv6.IMAGE_TAG, [] + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[network] Diagnostic succeeded! Linux containers can access the IPv6 loopback address without any issues.\n" + ) + else: + logger.error( + "[network] Diagnostic failed! Linux containers cannot access the IPv6 loopback address. Update to Docker 26.0.0+ or manually enable IPv6:", + True, + ) + logger.error( + "[network] https://docs.docker.com/config/daemon/ipv6/#use-ipv6-for-the-default-bridge-network\n", + False, + ) + + return built diff --git a/src/ue4docker/diagnostics/diagnostic_network.py b/src/ue4docker/diagnostics/diagnostic_network.py index aa56759e..4c8f99a9 100644 --- a/src/ue4docker/diagnostics/diagnostic_network.py +++ b/src/ue4docker/diagnostics/diagnostic_network.py @@ -70,7 +70,7 @@ def run(self, logger, args=[]): # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) try: - self._checkPlatformMistmatch(logger, containerPlatform) + self._checkPlatformMistmatch(logger, containerPlatform, True) except RuntimeError: return False diff --git a/src/ue4docker/diagnostics_cmd.py b/src/ue4docker/diagnostics_cmd.py index bd492d24..c112e02c 100644 --- a/src/ue4docker/diagnostics_cmd.py +++ b/src/ue4docker/diagnostics_cmd.py @@ -9,6 +9,7 @@ def diagnostics(): "all": allDiagnostics(), "8gig": diagnostic8Gig(), "20gig": diagnostic20Gig(), + "ipv6": diagnosticIPv6(), "network": diagnosticNetwork(), } diff --git a/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile new file mode 100644 index 00000000..12afdb7d --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Test that we can ping the IPv6 loopback address +RUN ping6 -c 5 '::1' From 632bee752291224bca1c63aae801cb3d4410d684 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 21 Jun 2024 15:49:39 +1000 Subject: [PATCH 408/430] Add IPv6 warning to `ue4-docker info` output (see #357) --- src/ue4docker/build.py | 12 ++------- src/ue4docker/info.py | 5 ++++ src/ue4docker/infrastructure/DockerUtils.py | 28 +++++++++++++++++++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index 72ae3657..20b79663 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -86,21 +86,13 @@ def build(): ) sys.exit(1) - # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 images without config changes + # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 Linux images without config changes if ( config.containerPlatform == "linux" and DockerUtils.isVersionWithoutIPV6Loopback() ): logger.warning( - "\n".join( - [ - "Warning: detected a Docker version older than 26.0.0.", - "Older versions of Docker cannot build or run images for Unreal Engine 5.4 or", - "newer unless the Docker daemon is explicitly configured to enable IPv6 support.", - "For details, see: https://github.com/adamrehn/ue4-docker/issues/357", - "", - ] - ), + DockerUtils.getIPV6WarningMessage() + "\n", False, ) diff --git a/src/ue4docker/info.py b/src/ue4docker/info.py index 9399514d..26ee0af1 100644 --- a/src/ue4docker/info.py +++ b/src/ue4docker/info.py @@ -101,3 +101,8 @@ def info(): item[0], " " * ((longestName + minSpaces) - len(item[0])), item[1] ) ) + + # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 Linux images without config changes + if DockerUtils.isVersionWithoutIPV6Loopback(): + logger = Logger(prefix="") + logger.warning(DockerUtils.getIPV6WarningMessage()) diff --git a/src/ue4docker/infrastructure/DockerUtils.py b/src/ue4docker/infrastructure/DockerUtils.py index 05155935..dd9310e5 100644 --- a/src/ue4docker/infrastructure/DockerUtils.py +++ b/src/ue4docker/infrastructure/DockerUtils.py @@ -1,4 +1,4 @@ -import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re +import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re, sys from docker.models.containers import Container from packaging.version import Version @@ -33,6 +33,14 @@ def info(): client = docker.from_env() return client.info() + @staticmethod + def minimumVersionForIPV6(): + """ + Returns the minimum version of the Docker daemon that supports IPv6 by default + without requiring manual configuration by the user + """ + return Version("26.0.0") + @staticmethod def isVersionWithoutIPV6Loopback(): """ @@ -40,7 +48,23 @@ def isVersionWithoutIPV6Loopback(): IPv6 loopback address [::1] when using its default network configuration """ dockerVersion = Version(DockerUtils.version()["Version"]) - return dockerVersion < Version("26.0.0") + return dockerVersion < DockerUtils.minimumVersionForIPV6() + + @staticmethod + def getIPV6WarningMessage(): + """ """ + return "\n".join( + [ + f"Warning: detected a Docker version older than {DockerUtils.minimumVersionForIPV6()}.", + "Older versions of Docker cannot build or run Linux images for Unreal Engine 5.4 or", + "newer unless the Docker daemon is explicitly configured to enable IPv6 support.", + "", + "To test whether IPv6 support is working, run the following diagnostic test:", + f"{sys.argv[0]} diagnostics ipv6", + "", + "For more details, see: https://github.com/adamrehn/ue4-docker/issues/357", + ] + ) @staticmethod def exists(name): From e365b45cf01572e7f9952101275a08c8a48cd12e Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Fri, 21 Jun 2024 15:56:49 +1000 Subject: [PATCH 409/430] Add stringification to `VisualStudio` objects for prettier log output --- src/ue4docker/infrastructure/BuildConfiguration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 5de35728..26035943 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -60,6 +60,9 @@ def __init__( self.unsupported_since = unsupported_since self.pass_version_to_buildgraph = pass_version_to_buildgraph + def __str__(self) -> str: + return self.name + DefaultVisualStudio = "2017" From 1a0f92cc93451554e3843733f6c04558892c4f80 Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 24 Jun 2024 15:12:33 +1000 Subject: [PATCH 410/430] Split subdirectories into separate layers under Windows (see #359) --- .../dockerfiles/ue4-minimal/windows/Dockerfile | 14 ++++++++++---- .../ue4-minimal/windows/split-components.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index fc2cc433..25b98f76 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -37,16 +37,16 @@ RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) && ` rmdir /s /q C:\UnrealEngine\Engine -# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers -COPY split-components.py C:\split-components.py -RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components - {% if (not disable_all_patches) and (not disable_target_patches) %} # Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini COPY fix-targets.py C:\fix-targets.py RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\Config\BaseEngine.ini {% endif %} +# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers +COPY split-components.py C:\split-components.py +RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components + # Copy the Installed Build into a clean image, discarding the source tree {% if combine %} FROM prerequisites as minimal @@ -57,6 +57,12 @@ FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} # Copy the Installed Build files from the builder image COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Binaries C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Content C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Extras C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Intermediate C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Plugins C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Source C:\UnrealEngine {% if excluded_components.ddc == false %} COPY --from=builder C:\UnrealEngine\Components\DDC C:\UnrealEngine {% endif %} diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py index 73149ffc..79fa8f31 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -56,3 +56,13 @@ def extractComponent(inputDir, outputDir, component, description, items): extractComponent( rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs ) + +# Extract the larger non-optional subdirectories of the Engine directory +for subdir in ["Binaries", "Content", "Extras", "Intermediate", "Plugins", "Source"]: + extractComponent( + rootDir, + outputDir, + subdir, + f"{subdir} subdirectory", + [join(rootDir, "Engine", subdir)], + ) From 8eeaf3a8baed726a889ef63daa2fa92cf5506e6c Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Mon, 24 Jun 2024 17:03:11 +1000 Subject: [PATCH 411/430] Bump version to 0.0.114 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 75dcec92..e4547fc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.113" +version = "0.0.114" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 4e53548b79ec11d7b5646730d33f8ca59d2417ab Mon Sep 17 00:00:00 2001 From: Adam Rehn Date: Thu, 27 Jun 2024 09:48:10 +1000 Subject: [PATCH 412/430] Fix 4.27 Windows breakage introduced in #346 --- .../dockerfiles/ue4-minimal/windows/split-components.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py index 79fa8f31..7c3384ba 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -48,7 +48,7 @@ def extractComponent(inputDir, outputDir, component, description, items): extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) # Extract debug symbols -symbolFiles = glob.glob(join(rootDir, "**", "*UnrealEditor*.pdb"), recursive=True) +symbolFiles = glob.glob(join(rootDir, "**", "*U*Editor*.pdb"), recursive=True) extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) # Extract template projects and samples From 8d48bc165a0e0d593a84b6d461463bfc5bfb7bde Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 14 Oct 2024 12:35:39 +0300 Subject: [PATCH 413/430] Normalize project urls according to PEP-753 --- pyproject.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e4547fc1..fb7ef873 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,10 +33,11 @@ dependencies = [ ] # See https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#urls +# See https://peps.python.org/pep-0753/#well-known-labels [project.urls] -Homepage = "https://github.com/adamrehn/ue4-docker" -Documentation = "https://adamrehn.github.io/ue4-docker" -Repository = "https://github.com/adamrehn/ue4-docker.git" +homepage = "https://github.com/adamrehn/ue4-docker" +documentation = "https://adamrehn.github.io/ue4-docker" +repository = "https://github.com/adamrehn/ue4-docker.git" [project.scripts] ue4-docker = "ue4docker:main" From d6a5a2855b37dcefa8c29d52bfde552637dfe66e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 23 Oct 2024 18:35:03 +0300 Subject: [PATCH 414/430] fix Docker deprecation warnings --- .../dockerfiles/ue4-build-prerequisites/linux/Dockerfile | 8 ++++---- .../ue4-build-prerequisites/windows/Dockerfile | 6 +++--- src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile | 2 +- src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile index d4dba0af..faeca470 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -1,5 +1,5 @@ ARG BASEIMAGE -FROM ${BASEIMAGE} as prerequisites +FROM ${BASEIMAGE} AS prerequisites {% if not disable_labels %} # Add a sentinel label so we can easily identify all derived images, including intermediate images @@ -11,15 +11,15 @@ ENV DEBIAN_FRONTEND=noninteractive # Enable CUDA support for NVIDIA GPUs (even when not using a CUDA base image), since evidently some versions of UE unconditionally assume # `libcuda.so.1` exists when using the NVIDIA proprietary drivers, and will fail to initialise the Vulkan RHI if it is missing -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},compute +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},compute # Add the "display" driver capability for NVIDIA GPUs # (This allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},display +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},display # Enable NVENC support for use by Unreal Engine plugins that depend on it (e.g. Pixel Streaming) # (Note that adding `video` seems to implicitly enable `compute` as well, but we include separate directives here to clearly indicate the purpose of both) -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES},video +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},video # Install our build prerequisites RUN apt-get update && apt-get install -y --no-install-recommends \ diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile index 280c044a..16b23050 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -2,9 +2,9 @@ ARG BASEIMAGE ARG DLLSRCIMAGE -FROM ${DLLSRCIMAGE} as dlls +FROM ${DLLSRCIMAGE} AS dlls -FROM ${BASEIMAGE} as deduplication +FROM ${BASEIMAGE} AS deduplication SHELL ["cmd", "/S", "/C"] {% if not disable_labels %} @@ -43,7 +43,7 @@ COPY remove-duplicate-dlls.ps1 C:\remove-duplicate-dlls.ps1 RUN powershell -ExecutionPolicy Bypass -File C:\remove-duplicate-dlls.ps1 # Copy the DLL files into a clean image -FROM ${BASEIMAGE} as prerequisites +FROM ${BASEIMAGE} AS prerequisites SHELL ["cmd", "/S", "/C"] COPY --from=deduplication C:\GatheredDlls\ C:\Windows\System32\ diff --git a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile index 368bdfea..b8da8539 100644 --- a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -42,7 +42,7 @@ USER ue4 # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) -ENV UE4CLI_CONFIG_DIR /home/ue4/.config/ue4cli +ENV UE4CLI_CONFIG_DIR=/home/ue4/.config/ue4cli # Copy the Conan configuration settings and package cache from the previous build stage COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan diff --git a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile index b30da1b4..2ce2b834 100644 --- a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -40,7 +40,7 @@ RUN pip install "%CONAN_VERSION%" "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue # Explicitly set the configuration directory for ue4cli # (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) -ENV UE4CLI_CONFIG_DIR C:\Users\ContainerAdministrator\AppData\Roaming\ue4cli +ENV UE4CLI_CONFIG_DIR=C:\Users\ContainerAdministrator\AppData\Roaming\ue4cli # Copy the Conan configuration settings and package cache from the previous build stage COPY --from=conan C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan From e1a8357347d41e3cb316b41c9e5e3f90220f4898 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 14 Nov 2024 22:57:19 +0300 Subject: [PATCH 415/430] Add changelist for ue-5.5 --- src/ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 26035943..0e850f4d 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -42,6 +42,7 @@ "5.2.0": 25360045, "5.3.0": 27405482, "5.4.0": 33043543, + "5.5.0": 37670630, } From bd38b0433491154cca39b0d49614b9835edfd35e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 14 Nov 2024 22:57:57 +0300 Subject: [PATCH 416/430] Bump version to 0.0.115 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fb7ef873..05a0c7cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.114" +version = "0.0.115" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.7" license = { file = "LICENSE" } From 3c4418fbab270d7313d3ac64f12468ee7064d7a6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 12 Dec 2024 10:53:02 +0300 Subject: [PATCH 417/430] Reorganize GitHub Actions test matrix (#368) * All Ubuntus now run tests with Python matching system one * Windows builds use latest Python. Even if we break something in older versions, we can just ask users to upgrade * Add Ubuntu 24.04 * Upgrade Python to 3.12 --- .github/workflows/ci.yml | 36 +++++++++++++++++++++--------------- docs/configuring-linux.adoc | 4 ++-- pyproject.toml | 2 +- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 315b5bf7..c41c9d1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,21 +31,27 @@ jobs: strategy: matrix: include: - - { os: ubuntu-20.04, python: "3.7" } - - { os: ubuntu-20.04, python: "3.10" } - - { os: ubuntu-22.04, python: "3.10" } - - { os: windows-2019, python: "3.7", visual-studio: 2017 } - - { os: windows-2019, python: "3.10", visual-studio: 2017 } - - { os: windows-2019, python: "3.7", visual-studio: 2019 } - - { os: windows-2019, python: "3.10", visual-studio: 2019 } - - { os: windows-2019, python: "3.7", visual-studio: 2022 } - - { os: windows-2019, python: "3.10", visual-studio: 2022 } - - { os: windows-latest, python: "3.7", visual-studio: 2017 } - - { os: windows-latest, python: "3.10", visual-studio: 2017 } - - { os: windows-latest, python: "3.7", visual-studio: 2019 } - - { os: windows-latest, python: "3.10", visual-studio: 2019 } - - { os: windows-latest, python: "3.7", visual-studio: 2022 } - - { os: windows-latest, python: "3.10", visual-studio: 2022 } + # All supported Ubuntu LTS with system python + # Be careful when removing EOL versions so that we still test our oldest supported python at least somewhere + - { os: ubuntu-20.04, python: "3.8" } + - { os: ubuntu-22.04, python: "3.10" } + - { os: ubuntu-24.04, python: "3.12" } + + # All supported Visual Studio on Windows Server 2019 + - { os: windows-2019, python: "3.12", visual-studio: 2017 } + - { os: windows-2019, python: "3.12", visual-studio: 2019 } + - { os: windows-2019, python: "3.12", visual-studio: 2022 } + + # All supported Visual Studio on Windows Server 2022 + - { os: windows-2022, python: "3.12", visual-studio: 2017 } + - { os: windows-2022, python: "3.12", visual-studio: 2019 } + - { os: windows-2022, python: "3.12", visual-studio: 2022 } + + # All supported Visual Studio on Windows Server 2025 + # TODO: Waiting for https://github.com/actions/runner-images/issues/10806 + # - { os: windows-2022, python: "3.12", visual-studio: 2017 } + # - { os: windows-2022, python: "3.12", visual-studio: 2019 } + # - { os: windows-2022, python: "3.12", visual-studio: 2022 } runs-on: ${{ matrix.os }} steps: - name: Checkout diff --git a/docs/configuring-linux.adoc b/docs/configuring-linux.adoc index ff9d9b08..585108c5 100644 --- a/docs/configuring-linux.adoc +++ b/docs/configuring-linux.adoc @@ -23,9 +23,9 @@ Follow the official installation instructions from the Docker Documentation for Once Docker is installed, follow the instructions from the https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user[Post-installation steps for Linux] page of the Docker Documentation to allow Docker commands to be run by a non-root user. This step is required in order to enable audio support when performing cloud rendering using the NVIDIA Container Toolkit. -== Step 2: Install Python 3.6 or newer +== Step 2: Install Python 3.8 or newer -WARNING: Note that older versions of these Linux distributions may not have Python 3.6 available in their system repositories by default. +WARNING: Note that older versions of these Linux distributions may not have Python 3.8 available in their system repositories by default. When working with an older distribution it may be necessary to configure community repositories that provide newer versions of Python. Under CentOS, run: diff --git a/pyproject.toml b/pyproject.toml index 05a0c7cf..5828c5c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "ue4-docker" version = "0.0.115" description = "Windows and Linux containers for Unreal Engine" -requires-python = ">= 3.7" +requires-python = ">= 3.8" license = { file = "LICENSE" } readme = "README.md" authors = [ From 3c46153f448425d835bb36d343dfc4b57e430999 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 6 Feb 2025 18:17:31 +0300 Subject: [PATCH 418/430] Install .NET Core 6.0 for UE-5.5+ See https://forums.unrealengine.com/t/unable-to-build-ue-5-5-installedbuild/2243395 --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index 9e36705c..325b5872 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -108,7 +108,8 @@ $vs_args = @( "--add", "Microsoft.Net.Component.4.6.2.TargetingPack", "--add", "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites", "--add", "Microsoft.NetCore.Component.SDK", - "--add", "Microsoft.NetCore.Component.Runtime.3.1" + "--add", "Microsoft.NetCore.Component.Runtime.3.1", + "--add", "Microsoft.NetCore.Component.Runtime.6.0" ) # Install the Visual Studio Build Tools workloads and components we need From a0bcb3369f3c38c4c6bc8eb8b1bc636574b37b4a Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Mon, 10 Feb 2025 23:59:12 -0800 Subject: [PATCH 419/430] resolves #371 fix broken url for Vulkan libs --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index 325b5872..d67d20a7 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -50,7 +50,7 @@ Copy-Item -Path "$env:TEMP\bin\x64\dxcompiler.dll" C:\Windows\System32\ Copy-Item -Path "$env:TEMP\bin\x64\dxil.dll" C:\Windows\System32\ # Gather the Vulkan runtime library -Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/latest/windows/vulkan-runtime-components.zip?u=" -OutFile "$env:TEMP\vulkan-runtime-components.zip" +Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.4.304.0/windows/VulkanRT-1.4.304.0-Components.zip?u=" -OutFile "$env:TEMP\vulkan-runtime-components.zip" Expand-Archive -Path "$env:TEMP\vulkan-runtime-components.zip" -DestinationPath "$env:TEMP" Copy-Item -Path "*\x64\vulkan-1.dll" -Destination C:\Windows\System32\ From e0c050d14b44a176c4e734204cee26a85e9b37b6 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 11 Feb 2025 00:35:15 -0800 Subject: [PATCH 420/430] Stop locking git version It seems newer install just fine (tested with 2.47.1). See #356 why it was locked initially. --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index d67d20a7..e040acfb 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -21,7 +21,7 @@ $env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" # Install the chocolatey packages we need -RunProcessChecked "choco" @("install", "--no-progress", "-y", "git.install", "--version=2.43.0", "--params", @' +RunProcessChecked "choco" @("install", "--no-progress", "-y", "git.install", "--params", @' "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'`" '@) From 4e4759728389e4c13d37615e17e008d1feee1303 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Tue, 11 Feb 2025 01:11:18 -0800 Subject: [PATCH 421/430] Revert "Install .NET Core 6.0 for UE-5.5+" This reverts commit 3c46153f448425d835bb36d343dfc4b57e430999. The reason is because this change doesn't fix anything. --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index e040acfb..f7202c5f 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -108,8 +108,7 @@ $vs_args = @( "--add", "Microsoft.Net.Component.4.6.2.TargetingPack", "--add", "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites", "--add", "Microsoft.NetCore.Component.SDK", - "--add", "Microsoft.NetCore.Component.Runtime.3.1", - "--add", "Microsoft.NetCore.Component.Runtime.6.0" + "--add", "Microsoft.NetCore.Component.Runtime.3.1" ) # Install the Visual Studio Build Tools workloads and components we need From d0955151600dd02782cc435f47134dc0db5f1ee8 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 26 Feb 2025 17:21:56 +0300 Subject: [PATCH 422/430] Bump version to 0.0.116 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5828c5c4..7cc1d7be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "ue4-docker" -version = "0.0.115" +version = "0.0.116" description = "Windows and Linux containers for Unreal Engine" requires-python = ">= 3.8" license = { file = "LICENSE" } From 80514e53e2bd7d5d642c5a3550c6fc5fc7fcf9f4 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 26 Mar 2025 06:34:21 -0700 Subject: [PATCH 423/430] Fix typo --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index f7202c5f..22b5f83f 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -4,7 +4,7 @@ function RunProcessChecked { param ([string] $Cmd, [string[]] $Argv) - Write-Output "Executing comand: $Cmd $Argv" + Write-Output "Executing command: $Cmd $Argv" $process = Start-Process -NoNewWindow -PassThru -Wait -FilePath $Cmd -ArgumentList $Argv if ($process.ExitCode -ne 0) From 95a1a9618ef2e8eecb5a693c63e53457fb24db75 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 27 Mar 2025 17:04:54 +0300 Subject: [PATCH 424/430] Drop leftover compatibility with Python older than 3.8 --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7cc1d7be..9552f485 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,6 @@ dependencies = [ "colorama", "docker>=6.1.0", "humanfriendly", - "importlib-metadata>=1.0;python_version<'3.8'", "Jinja2>=2.11.3", "packaging>=19.1", "psutil", From 9d0d2aa6de53b2f9eefbb5cdb34f1dfad91097ab Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 27 Mar 2025 07:05:18 -0700 Subject: [PATCH 425/430] Drop engines older than 4.27 (#370) --- README.md | 2 +- docs/advanced-build-options.adoc | 37 +++--------- docs/ue4-docker-build.adoc | 4 +- src/ue4docker/build.py | 26 +++++---- .../windows/install-prerequisites.ps1 | 19 +------ .../dockerfiles/ue4-minimal/linux/Dockerfile | 39 +------------ .../ue4-minimal/linux/copy-toolchain.py | 43 -------------- .../ue4-minimal/linux/enable-opengl.py | 19 ------- .../ue4-minimal/linux/fix-targets.py | 23 -------- .../ue4-minimal/linux/patch-build-graph.py | 57 ------------------- .../ue4-minimal/linux/patch-filters-xml.py | 26 --------- .../ue4-minimal/windows/Dockerfile | 33 +++-------- .../ue4-minimal/windows/fix-targets.py | 23 -------- .../ue4-minimal/windows/patch-build-graph.py | 37 ------------ .../ue4-minimal/windows/patch-filters-xml.py | 20 ------- .../dockerfiles/ue4-source/linux/Dockerfile | 34 ++++------- .../ue4-source/linux/linker-fixup.py | 16 ------ .../ue4-source/linux/patch-broken-releases.py | 52 +++++++---------- .../dockerfiles/ue4-source/linux/patch-ubt.py | 29 ---------- .../linux/set-changelist.py | 0 .../dockerfiles/ue4-source/windows/Dockerfile | 27 ++++----- .../windows/patch-broken-releases.py | 35 ------------ .../ue4-source/windows/patch-ubt.py | 29 ---------- .../windows/set-changelist.py | 0 .../infrastructure/BuildConfiguration.py | 24 ++------ test-suite/test-ue-releases.py | 5 +- 26 files changed, 87 insertions(+), 572 deletions(-) delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py delete mode 100644 src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py delete mode 100644 src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py delete mode 100644 src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py rename src/ue4docker/dockerfiles/{ue4-minimal => ue4-source}/linux/set-changelist.py (100%) delete mode 100644 src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py rename src/ue4docker/dockerfiles/{ue4-minimal => ue4-source}/windows/set-changelist.py (100%) diff --git a/README.md b/README.md index a3c0d436..4dd393b7 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ The ue4-docker Python package contains a set of Dockerfiles and accompanying bui Key features include: -- The six most recent versions of the Unreal Engine are supported (currently Unreal Engine 4.26 and newer). +- The six most recent versions of the Unreal Engine are supported (currently Unreal Engine 4.27 and newer). - Both Windows containers and Linux containers are supported. - Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc index 26352a0b..2d56e3f8 100644 --- a/docs/advanced-build-options.adoc +++ b/docs/advanced-build-options.adoc @@ -67,7 +67,7 @@ For example: [source,shell] ---- # Excludes both debug symbols and template projects -ue4-docker build 4.21.2 --exclude debug --exclude templates +ue4-docker build 4.27.0 --exclude debug --exclude templates ---- === Enabling system resource monitoring during builds @@ -78,19 +78,19 @@ You can also use the `-interval` flag to override the default interval of 20 sec [source,shell] ---- # Logs system resource levels every 20 seconds -ue4-docker build 4.24.2 --monitor +ue4-docker build 4.27.0 --monitor ---- [source,shell] ---- # Logs system resource levels every 20 seconds -ue4-docker build 4.24.2 --monitor +ue4-docker build 4.27.0 --monitor ---- [source,shell] ---- # Logs system resource levels every 5 seconds -ue4-docker build 4.24.2 --monitor -interval=5 +ue4-docker build 4.27.0 --monitor -interval=5 ---- [[exporting-generated-dockerfiles]] @@ -103,19 +103,19 @@ You can use these flags like so: [source,shell] ---- # Exports Dockerfiles for all images to the specified filesystem directory -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" ---- [source,shell] ---- # Exports Dockerfiles for all images -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" ---- [source,shell] ---- # Exports Dockerfiles for all images and combines them into a single Dockerfile -ue4-docker build 4.25.4 -layout "/path/to/Dockerfiles" --combine +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" --combine ---- Exporting Dockerfiles is useful for debugging or contributing to the development of ue4-docker itself. @@ -158,29 +158,10 @@ This includes the labels which specify the <. diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py index 20b79663..dc14af4a 100644 --- a/src/ue4docker/build.py +++ b/src/ue4docker/build.py @@ -465,34 +465,38 @@ def build(): "--build-arg", "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), ] + + changelistArgs = ( + ["--build-arg", "CHANGELIST={}".format(config.changelist)] + if config.changelist is not None + else [] + ) + builder.build_builtin_image( "ue4-source", mainTags, - commonArgs + config.platformArgs + ue4SourceArgs + credentialArgs, + commonArgs + + config.platformArgs + + ue4SourceArgs + + credentialArgs + + changelistArgs, secrets=secrets, ) builtImages.append("ue4-source") else: logger.info("Skipping ue4-source image build.") + # Build the minimal UE4 CI image, unless requested otherwise by the user if config.buildTargets["minimal"]: - ue4BuildArgs = prereqConsumerArgs + [ + minimalArgs = prereqConsumerArgs + [ "--build-arg", "TAG={}".format(mainTags[1]), ] - # Build the minimal UE4 CI image, unless requested otherwise by the user - if config.buildTargets["minimal"]: - minimalArgs = ( - ["--build-arg", "CHANGELIST={}".format(config.changelist)] - if config.changelist is not None - else [] - ) - builder.build_builtin_image( "ue4-minimal", mainTags, - commonArgs + config.platformArgs + ue4BuildArgs + minimalArgs, + commonArgs + config.platformArgs + minimalArgs, ) builtImages.append("ue4-minimal") else: diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index 22b5f83f..ce2f8444 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -56,26 +56,9 @@ Copy-Item -Path "*\x64\vulkan-1.dll" -Destination C:\Windows\System32\ $visual_studio_build = $args[0] -# Use the latest available Windows SDK. The motivation behind this is: -# 1. Newer SDKs allow developers to use newer APIs. Developers can guard that API usage with runtime version checks if they want to continue to support older Windows releases. -# 2. Unreal Engine slowly moves to newer Windows SDK. 4.27.0 no longer compiles with SDKs older than 18362 and even if it will be fixed in 4.27.x, -# this is just a question of a time when older SDKs support will be dropped completely -# 3. UE5 doesn't support VS2017 at all, so in the future that argument for continuing to use Windows SDK 17763 from VS2017 era will be weaker and weaker. -# -# We can't use newer SDK for VS2017 that is used to compile older engines because 18362 SDK support was only added in UE-4.23. -# -# See https://github.com/adamrehn/ue4-docker/issues/192 -# See https://forums.unrealengine.com/t/ndis_miniport_major_version-is-not-defined-error/135058 -# See https://github.com/EpicGames/UnrealEngine/blame/4.23.0-release/Engine/Source/Programs/UnrealBuildTool/Platform/Windows/UEBuildWindows.cs#L1822-L1823 -# See https://github.com/EpicGames/UnrealEngine/commit/ecc4872c3269e75a24adc40734cc8bcc9bbed1ca -# See https://udn.unrealengine.com/s/question/0D54z000079HcjJCAS/d3d12h427-error-c4668-winapipartitiongames-is-not-defined-as-a-preprocessor-macro-replacing-with-0-for-ifelif -# -# Keywords for Google: -# error C4668: 'NDIS_MINIPORT_MAJOR_VERSION' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif -# d3d12.h(427): error C4668: 'WINAPI_PARTITION_GAMES' is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' if ($visual_studio_build -eq "15") { - $windows_sdk_version = 17763 + $windows_sdk_version = 18362 } else { diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index 798a786c..d2cc8da2 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -7,29 +7,9 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder {% endif %} -# Set the changelist number in Build.version to ensure our Build ID is generated correctly -ARG CHANGELIST -COPY set-changelist.py /tmp/set-changelist.py -RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version $CHANGELIST - # Remove the .git directory to disable UBT `git status` calls and speed up the build process RUN rm -rf /home/ue4/UnrealEngine/.git -{% if (not disable_all_patches) and (not disable_opengl_patch) %} -# Enable the OpenGL RHI for Engine versions where it is present but deprecated -COPY enable-opengl.py /tmp/enable-opengl.py -RUN python3 /tmp/enable-opengl.py /home/ue4/UnrealEngine/Engine/Config/BaseEngine.ini -{% endif %} - -{% if (not disable_all_patches) and (not disable_buildgraph_patches) %} -COPY patch-filters-xml.py /tmp/patch-filters-xml.py -RUN python3 /tmp/patch-filters-xml.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineFilters.xml - -# Patch the default settings in InstalledEngineBuild.xml and increase the output verbosity of the DDC generation step -COPY patch-build-graph.py /tmp/patch-build-graph.py -RUN python3 /tmp/patch-build-graph.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml /home/ue4/UnrealEngine/Engine/Build/Build.version -{% endif %} - # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted @@ -41,7 +21,9 @@ RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ -target="Make Installed Build Linux" \ -script=Engine/Build/InstalledEngineBuild.xml \ -set:HostPlatformOnly=true \ + -set:WithClient=true \ -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} \ + -set:WithServer=true \ {{ buildgraph_args }} && \ rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC @@ -54,23 +36,6 @@ RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealVersionSelector Linux Shippin RUN cp /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Build/InstalledBuild.txt /home/ue4/UnrealEngine/Engine/Build/InstalledBuild.txt && \ ./Engine/Binaries/Linux/UnrealVersionSelector-Linux-Shipping -register -unattended -{% if (not disable_all_patches) and (not disable_target_patches) %} -# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini -COPY fix-targets.py /tmp/fix-targets.py -RUN python3 /tmp/fix-targets.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Config/BaseEngine.ini -{% endif %} - -{% if (not disable_all_patches) and (not disable_unrealpak_copy) %} -# Some versions of the Engine fail to include UnrealPak in the Installed Build, so copy it manually -RUN cp ./Engine/Binaries/Linux/UnrealPak ./LocalBuilds/Engine/Linux/Engine/Binaries/Linux/UnrealPak -{% endif %} - -{% if (not disable_all_patches) and (not disable_toolchain_copy) %} -# Ensure the bundled toolchain included in 4.20.0 and newer is copied to the Installed Build -COPY --chown=ue4:ue4 copy-toolchain.py /tmp/copy-toolchain.py -RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine -{% endif %} - {% if enable_ushell %} # Ensure ushell is copied to the Installed Build RUN rm -rf ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py deleted file mode 100644 index df70e6ec..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -from os.path import basename, dirname, exists, join, relpath -import glob, shutil, sys - -# Determine the root directory for the source build and the Installed Build -sourceRoot = sys.argv[1] -installedRoot = join(sourceRoot, "LocalBuilds", "Engine", "Linux") - -# Locate the bundled toolchain and copy it to the Installed Build -sdkGlob = join( - sourceRoot, - "Engine", - "Extras", - "ThirdPartyNotUE", - "SDKs", - "HostLinux", - "Linux_x64", - "*", - "x86_64-unknown-linux-gnu", -) -for bundled in glob.glob(sdkGlob): - # Extract the root path for the toolchain - toolchain = dirname(bundled) - - # Print progress output - print( - 'Copying bundled toolchain "{}" to Installed Build...'.format( - basename(toolchain) - ), - file=sys.stderr, - ) - sys.stderr.flush() - - # Perform the copy - dest = join(installedRoot, relpath(toolchain, sourceRoot)) - if exists(dest) == True: - print( - "Destination toolchain already exists: {}".format(dest), - file=sys.stderr, - flush=True, - ) - else: - shutil.copytree(toolchain, dest) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py deleted file mode 100644 index 597992ed..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/enable-opengl.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Enable the OpenGL RHI for Engine versions where it is present but deprecated -iniFile = sys.argv[1] -config = readFile(iniFile) -config = config.replace("; +TargetedRHIs=GLSL_430", "+TargetedRHIs=GLSL_430") -writeFile(iniFile, config) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py deleted file mode 100644 index 6ce9a427..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/fix-targets.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini -iniFile = sys.argv[1] -config = readFile(iniFile) -config = re.sub( - 'PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', - 'PlatformType="\\2", RequiredFile="\\1"', - config, -) -writeFile(iniFile, config) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py deleted file mode 100644 index 3bf2ddf1..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-build-graph.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -import json, os, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Read the build graph XML -buildXml = sys.argv[1] -code = readFile(buildXml) - -# Read the UE4 version information -versionFile = sys.argv[2] -versionData = json.loads(readFile(versionFile)) - -# Add verbose output flags to the `BuildDerivedDataCache` command -code = code.replace( - 'Command Name="BuildDerivedDataCache" Arguments="', - 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ', -) - -# Disable AArch64 by default (enabled since 4.24.0) -code = code.replace( - 'Property Name="DefaultWithLinuxAArch64" Value="true"', - 'Property Name="DefaultWithLinuxAArch64" Value="false"', -) -# AArch64 was renamed to Arm64 in UE-5.0, so also disable it -code = code.replace( - 'Property Name="DefaultWithLinuxArm64" Value="true"', - 'Property Name="DefaultWithLinuxArm64" Value="false"', -) - -# Enable client and server targets by default in 4.23.0 onwards, except for 4.24.0 - 4.24.2 where Linux server builds fail -# (See for details of the bug and its fix) -if ( - versionData["MajorVersion"] != 4 - or versionData["MinorVersion"] != 24 - or versionData["PatchVersion"] >= 3 -): - code = code.replace( - 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"', - ) - code = code.replace( - 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"', - ) - -# Write the modified XML back to disk -writeFile(buildXml, code) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py deleted file mode 100644 index cc2f1de7..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/patch-filters-xml.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -filtersXml = sys.argv[1] -code = readFile(filtersXml) - -# Add missing SetupDotnet.sh in Unreal Engine 5.0.0-early-access-1 -# See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 -# and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 -dotnet = "Engine/Binaries/ThirdParty/DotNet/Linux/..." -setup_dotnet = "Engine/Build/BatchFiles/Linux/SetupDotnet.sh" -if dotnet in code and setup_dotnet not in code: - code = code.replace(dotnet, f"{dotnet}\n{setup_dotnet}") - -writeFile(filtersXml, code) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile index 25b98f76..1c38ecdb 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -8,41 +8,22 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder {% endif %} -# Set the changelist number in Build.version to ensure our Build ID is generated correctly -ARG CHANGELIST -COPY set-changelist.py C:\set-changelist.py -RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% - # Remove the .git directory to disable UBT `git status` calls and speed up the build process RUN if exist C:\UnrealEngine\.git rmdir /s /q C:\UnrealEngine\.git -{% if (not disable_all_patches) and (not disable_buildgraph_patches) %} -# Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 -COPY patch-filters-xml.py C:\patch-filters-xml.py -RUN python C:\patch-filters-xml.py C:\UnrealEngine\Engine\Build\InstalledEngineFilters.xml - -# Patch out problematic entries in InstalledEngineBuild.xml introduced in UE4.23.0 -COPY patch-build-graph.py C:\patch-build-graph.py -RUN python C:\patch-build-graph.py C:\UnrealEngine\Engine\Build\InstalledEngineBuild.xml -{% endif %} - # Create an Installed Build of the Engine WORKDIR C:\UnrealEngine RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` - -target="Make Installed Build Win64" ` - -script=Engine/Build/InstalledEngineBuild.xml ` - -set:HostPlatformOnly=true ` - -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` - {{ buildgraph_args }} && ` + -target="Make Installed Build Win64" ` + -script=Engine/Build/InstalledEngineBuild.xml ` + -set:HostPlatformOnly=true ` + -set:WithClient=true ` + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` + -set:WithServer=true ` + {{ buildgraph_args }} && ` (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) && ` rmdir /s /q C:\UnrealEngine\Engine -{% if (not disable_all_patches) and (not disable_target_patches) %} -# Ensure Client and Server targets have their `PlatformType` field set correctly in BaseEngine.ini -COPY fix-targets.py C:\fix-targets.py -RUN python C:\fix-targets.py C:\UnrealEngine\LocalBuilds\Engine\Windows\Engine\Config\BaseEngine.ini -{% endif %} - # Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers COPY split-components.py C:\split-components.py RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py deleted file mode 100644 index 6ce9a427..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/fix-targets.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Ensure the `PlatformType` field is set correctly for Client and Server targets in BaseEngine.ini -iniFile = sys.argv[1] -config = readFile(iniFile) -config = re.sub( - 'PlatformType="Game", RequiredFile="(.+UE4(Client|Server).*\\.target)"', - 'PlatformType="\\2", RequiredFile="\\1"', - config, -) -writeFile(iniFile, config) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py deleted file mode 100644 index ae2de692..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-build-graph.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Read the build graph XML -buildXml = sys.argv[1] -code = readFile(buildXml) - -# Disable HoloLens by default on 4.23. -# On later versions, it is guarded by DefaultWithPlatform and disabled because of HostPlatformOnly=true -code = code.replace( - 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="$(DefaultWithWindows)"', - 'Option Name="WithHoloLens" Restrict="true|false" DefaultValue="false"', -) - -# Enable client and server targets by default in 4.23.0 onwards -code = code.replace( - 'Option Name="WithClient" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithClient" Restrict="true|false" DefaultValue="true"', -) -code = code.replace( - 'Option Name="WithServer" Restrict="true|false" DefaultValue="false"', - 'Option Name="WithServer" Restrict="true|false" DefaultValue="true"', -) - -# Write the modified XML back to disk -writeFile(buildXml, code) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py deleted file mode 100644 index 20c9e254..00000000 --- a/src/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -# Remove the dependency on Linux cross-compilation debug tools introduced in UE4.20.0 -filtersXml = sys.argv[1] -code = readFile(filtersXml) -code = code.replace("Engine/Binaries/Linux/dump_syms.exe", "") -code = code.replace("Engine/Binaries/Linux/BreakpadSymbolEncoder.exe", "") -writeFile(filtersXml, code) diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile index b952f903..1e5a1837 100644 --- a/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -6,11 +6,6 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} -{% if not disable_all_patches %} -# Enable verbose output for steps that patch files? -ARG VERBOSE_OUTPUT=0 -{% endif %} - {% if source_mode == "copy" %} # Copy the Unreal Engine source code from the host system @@ -83,8 +78,13 @@ RUN mkdir /home/ue4/UnrealEngine && \ {% endif %} +{% if not disable_all_patches %} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 +{% endif %} + {% if (not disable_all_patches) and (not disable_release_patches) %} -# Apply our bugfix patches to broken Engine releases such as 4.25.4 +# Apply our bugfix patches to broken Engine releases # (Make sure we do this before the post-clone setup steps are run) COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT @@ -114,21 +114,7 @@ RUN sudo apt-get update && \ {% endif %} -{% if (not disable_all_patches) and (not disable_linker_fixup) %} -# The linker bundled with UE4.20.0 onwards chokes on system libraries built with newer compilers, -# so redirect the bundled clang to use the system linker instead -COPY --chown=ue4:ue4 linker-fixup.py /tmp/linker-fixup.py -RUN python3 /tmp/linker-fixup.py /home/ue4/UnrealEngine/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64 `which ld` -{% endif %} - -{% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} -# Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards -# (For details of what this is, see: ) -RUN rm -r -f /home/ue4/UnrealEngine/Engine/Platforms/XXX -{% endif %} - -{% if (not disable_all_patches) and (not disable_ubt_patches) %} -# Apply our bugfix patches to UnrealBuildTool (UBT) -COPY --chown=ue4:ue4 patch-ubt.py /tmp/patch-ubt.py -RUN python3 /tmp/patch-ubt.py /home/ue4/UnrealEngine/Engine/Source/Programs/UnrealBuildTool -{% endif %} +# Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST +COPY set-changelist.py /tmp/set-changelist.py +RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version $CHANGELIST diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py b/src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py deleted file mode 100644 index f63a12b4..00000000 --- a/src/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -import glob, os, shutil, sys - -# Retrieve the Linux SDK root directory and system ld location from our command-line arguments -sdkRoot = sys.argv[1] -systemLd = sys.argv[2] - -# Locate the bundled version(s) of ld and replace them with symlinks to the system ld -for bundled in glob.glob( - os.path.join( - sdkRoot, "*", "x86_64-unknown-linux-gnu", "bin", "x86_64-unknown-linux-gnu-ld" - ) -): - os.unlink(bundled) - os.symlink(systemLd, bundled) - print("{} => {}".format(bundled, systemLd), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py index 5fb4f0f2..5dc2bf55 100644 --- a/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py +++ b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -14,41 +14,31 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" versionDetails = json.loads( readFile(join(engineRoot, "Engine", "Build", "Build.version")) ) + if ( - versionDetails["MajorVersion"] == 4 - and versionDetails["MinorVersion"] == 25 - and versionDetails["PatchVersion"] == 4 + versionDetails["MajorVersion"] == 5 + and versionDetails["MinorVersion"] == 0 + and versionDetails["PatchVersion"] == 0 ): - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them - # (See: ) - gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") - gitdepsXml = readFile(gitdepsFile) - if '', - '\n ' - + '', - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' - + '', - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' - + '', - ) - - writeFile(gitdepsFile, gitdepsXml) - - if verboseOutput: - print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) - else: - print("PATCHED {}".format(gitdepsFile), file=sys.stderr) + buildFile = join(engineRoot, "Engine", "Build", "InstalledEngineFilters.xml") + buildXml = readFile(buildFile) + + # Add missing SetupDotnet.sh in Unreal Engine 5.0.0-early-access-1 + # See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 + # and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 + dotnet = "Engine/Binaries/ThirdParty/DotNet/Linux/..." + setup_dotnet = "Engine/Build/BatchFiles/Linux/SetupDotnet.sh" + if dotnet in buildXml and setup_dotnet not in buildXml: + buildXml = buildXml.replace(dotnet, f"{dotnet}\n{setup_dotnet}") + + writeFile(buildFile, buildXml) + + if verboseOutput: + print("PATCHED {}:\n\n{}".format(buildFile, buildXml), file=sys.stderr) + else: + print("PATCHED {}".format(buildFile), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py deleted file mode 100644 index 435ddd90..00000000 --- a/src/ue4docker/dockerfiles/ue4-source/linux/patch-ubt.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -def patchFile(filename, search, replace): - contents = readFile(filename) - patched = contents.replace(search, replace) - writeFile(filename, patched) - - -# Apply our bugfixes to UnrealBuildTool (UBT) - -# Ensure modules always have a binary output directory value set when exporting JSON -# (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) -patchFile( - os.path.join(sys.argv[1], "Configuration", "UEBuildTarget.cs"), - "Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);", - "Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);", -) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/src/ue4docker/dockerfiles/ue4-source/linux/set-changelist.py similarity index 100% rename from src/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py rename to src/ue4docker/dockerfiles/ue4-source/linux/set-changelist.py diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile index ddd5c8a3..a2c13b8a 100644 --- a/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -7,11 +7,6 @@ ARG PREREQS_TAG FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} {% endif %} -{% if not disable_all_patches %} -# Enable verbose output for steps that patch files? -ARG VERBOSE_OUTPUT=0 -{% endif %} - {% if source_mode == "copy" %} # Copy the Unreal Engine source code from the host system @@ -63,8 +58,13 @@ COPY patch-setup-win.py C:\patch-setup-win.py RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat %VERBOSE_OUTPUT% {% endif %} +{% if not disable_all_patches %} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 +{% endif %} + {% if (not disable_all_patches) and (not disable_release_patches) %} -# Apply our bugfix patches to broken Engine releases such as 4.25.4 +# Apply our bugfix patches to broken Engine releases # (Make sure we do this before the post-clone setup steps are run) COPY patch-broken-releases.py C:\patch-broken-releases.py RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% @@ -75,14 +75,7 @@ RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% WORKDIR C:\UnrealEngine RUN Setup.bat -no-cache {{ gitdependencies_args }} -{% if (not disable_all_patches) and (not disable_example_platform_cleanup) %} -# Remove the sample `XXX` example platform code, since this breaks builds from 4.24.0 onwards -# (For details of what this is, see: ) -RUN rmdir /s /q C:\UnrealEngine\Engine\Platforms\XXX 2>NUL || exit 0 -{% endif %} - -{% if (not disable_all_patches) and (not disable_ubt_patches) %} -# Apply our bugfix patches to UnrealBuildTool (UBT) -COPY patch-ubt.py C:\patch-ubt.py -RUN python C:\patch-ubt.py C:\UnrealEngine\Engine\Source\Programs\UnrealBuildTool -{% endif %} +# Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST +COPY set-changelist.py C:\set-changelist.py +RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py index 980d8f02..577e8525 100644 --- a/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py +++ b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -14,47 +14,12 @@ def writeFile(filename, data): f.write(data.encode("utf-8")) -# Determine if we are building UE 4.25.4 engineRoot = sys.argv[1] verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" versionDetails = json.loads( readFile(join(engineRoot, "Engine", "Build", "Build.version")) ) -if ( - versionDetails["MajorVersion"] == 4 - and versionDetails["MinorVersion"] == 25 - and versionDetails["PatchVersion"] == 4 -): - # If `Commit.gitdeps.xml` is missing the changes from CL 14469950 then inject them - # (See: ) - gitdepsFile = join(engineRoot, "Engine", "Build", "Commit.gitdeps.xml") - gitdepsXml = readFile(gitdepsFile) - if '', - '\n ' - + '', - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' - + '', - ) - gitdepsXml = gitdepsXml.replace( - '', - '\n ' - + '', - ) - - writeFile(gitdepsFile, gitdepsXml) - - if verboseOutput: - print("PATCHED {}:\n\n{}".format(gitdepsFile, gitdepsXml), file=sys.stderr) - else: - print("PATCHED {}".format(gitdepsFile), file=sys.stderr) - -# Determine if we are building UE 5.1.0 if ( versionDetails["MajorVersion"] == 5 and versionDetails["MinorVersion"] == 1 diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py deleted file mode 100644 index 435ddd90..00000000 --- a/src/ue4docker/dockerfiles/ue4-source/windows/patch-ubt.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - - -def readFile(filename): - with open(filename, "rb") as f: - return f.read().decode("utf-8") - - -def writeFile(filename, data): - with open(filename, "wb") as f: - f.write(data.encode("utf-8")) - - -def patchFile(filename, search, replace): - contents = readFile(filename) - patched = contents.replace(search, replace) - writeFile(filename, patched) - - -# Apply our bugfixes to UnrealBuildTool (UBT) - -# Ensure modules always have a binary output directory value set when exporting JSON -# (In some Engine versions, External modules do not have an output directory set, breaking modules that reference `$(BinaryOutputDir)`) -patchFile( - os.path.join(sys.argv[1], "Configuration", "UEBuildTarget.cs"), - "Module.ExportJson(Module.Binary?.OutputDir, GetExecutableDir(), Writer);", - "Module.ExportJson((Module.Binary != null || Binaries.Count == 0) ? Module.Binary?.OutputDir : Binaries[0].OutputDir, GetExecutableDir(), Writer);", -) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/src/ue4docker/dockerfiles/ue4-source/windows/set-changelist.py similarity index 100% rename from src/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py rename to src/ue4docker/dockerfiles/ue4-source/windows/set-changelist.py diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 0e850f4d..0a81064d 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -29,13 +29,6 @@ # The Perforce changelist numbers for each supported .0 release of the Unreal Engine UNREAL_ENGINE_RELEASE_CHANGELISTS = { - "4.20.0": 4212847, - "4.21.0": 4541578, - "4.22.0": 5660361, - "4.23.0": 8386587, - "4.24.0": 10570792, - "4.25.0": 13144385, - "4.26.0": 14830424, "4.27.0": 17155196, "5.0.0": 19505902, "5.1.0": 23058290, @@ -71,20 +64,15 @@ def __str__(self) -> str: "2017": VisualStudio( name="2017", build_number="15", - # We do not support versions older than 4.20 - supported_since=Version("4.20"), + # We do not support versions older than 4.27 + supported_since=Version("4.27"), unsupported_since=Version("5.0"), pass_version_to_buildgraph=False, ), "2019": VisualStudio( name="2019", build_number="16", - # Unreal Engine 4.23.1 is the first that successfully builds with Visual Studio v16.3 - # See https://github.com/EpicGames/UnrealEngine/commit/2510d4fd07a35ba5bff6ac2c7becaa6e8b7f11fa - # - # Unreal Engine 4.25 is the first that works with .NET SDK 4.7+ - # See https://github.com/EpicGames/UnrealEngine/commit/5256eedbdef30212ab69fdf4c09e898098959683 - supported_since=Version("4.25"), + supported_since=Version("4.27"), unsupported_since=Version("5.4"), pass_version_to_buildgraph=True, ), @@ -133,12 +121,12 @@ def addArguments(parser): parser.add_argument( "release", nargs="?", # aka "required = False", but that doesn't work in positionals - help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)', + help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)', ) parser.add_argument( "--ue-version", default=None, - help='UE4 release to build, in semver format (e.g. 4.20.0) or "custom" for a custom repo and branch', + help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch', ) parser.add_argument( "--linux", @@ -450,7 +438,7 @@ def __init__(self, parser, argv, logger): ) except InvalidVersion: raise RuntimeError( - 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.20.0")'.format( + 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.27.0")'.format( self.args.release ) ) diff --git a/test-suite/test-ue-releases.py b/test-suite/test-ue-releases.py index a3931f9a..6ebfa410 100755 --- a/test-suite/test-ue-releases.py +++ b/test-suite/test-ue-releases.py @@ -14,7 +14,7 @@ class UERelease: def __init__( - self, name: str, tag: str, repo: str, vsVersion: int, ubuntuVersion: str + self, name: str, tag: str, repo: str, vsVersion: int, ubuntuVersion: str | None ) -> None: self.name = name self.tag = tag @@ -29,12 +29,13 @@ def __init__( # The list of Unreal Engine releases that are currently supported by ue4-docker SUPPORTED_RELEASES = [ - UERelease("4.26", "4.26.2-fixed", COMMITDEPS_REPO, 2017, None), UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, 2017, None), UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, 2019, "20.04"), UERelease("5.1", "5.1.1-fixed", COMMITDEPS_REPO, 2019, None), UERelease("5.2", "5.2.1-release", UPSTREAM_REPO, 2022, None), UERelease("5.3", "5.3.2-release", UPSTREAM_REPO, 2022, None), + UERelease("5.4", "5.4.4-release", UPSTREAM_REPO, 2022, None), + UERelease("5.5", "5.5.1-release", UPSTREAM_REPO, 2022, None), ] From 90fb3a6f71b2cc6593a8b013af0e15f1c18214dd Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 24 Apr 2025 00:28:19 -0700 Subject: [PATCH 426/430] resolves #374 replace dependency on windows-sdk-10-version-1809-windbg with windowsdriverkit10 --- .../ue4-build-prerequisites/windows/install-prerequisites.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 index ce2f8444..db157acb 100644 --- a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -26,7 +26,7 @@ RunProcessChecked "choco" @("install", "--no-progress", "-y", "git.install", "-- '@) # pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine -RunProcessChecked "choco" @("install", "--no-progress", "-y", "choco-cleaner", "python", "vcredist-all", "windows-sdk-10-version-1809-windbg") +RunProcessChecked "choco" @("install", "--no-progress", "-y", "choco-cleaner", "python", "vcredist-all", "windowsdriverkit10") # Reload our environment variables from the registry so the `git` command works Update-SessionEnvironment From 10c1078940e433cbfb0c54493ac1c1929be1ef85 Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Thu, 24 Apr 2025 10:30:57 +0300 Subject: [PATCH 427/430] Drop Ubuntu 20.04 from CI It is no longer supported by GitHub Actions. See https://github.com/actions/runner-images/issues/11101 --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c41c9d1c..59ab26fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,6 @@ jobs: include: # All supported Ubuntu LTS with system python # Be careful when removing EOL versions so that we still test our oldest supported python at least somewhere - - { os: ubuntu-20.04, python: "3.8" } - { os: ubuntu-22.04, python: "3.10" } - { os: ubuntu-24.04, python: "3.12" } From 2a1e922c4e0637862d532fba054b03330e11a42e Mon Sep 17 00:00:00 2001 From: Marat Radchenko Date: Wed, 4 Jun 2025 10:40:55 +0300 Subject: [PATCH 428/430] Add changelist for 5.6 --- src/ue4docker/infrastructure/BuildConfiguration.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py index 0a81064d..200f1cf6 100644 --- a/src/ue4docker/infrastructure/BuildConfiguration.py +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -36,6 +36,7 @@ "5.3.0": 27405482, "5.4.0": 33043543, "5.5.0": 37670630, + "5.6.0": 43139311, } From e0d2b2e09670b8b1f41d367f85980ae57f77bf44 Mon Sep 17 00:00:00 2001 From: Sam Collier Date: Mon, 23 Jun 2025 23:37:15 +0100 Subject: [PATCH 429/430] build ubt by calling BuildUBT.sh --- src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index d2cc8da2..b965d5db 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -13,7 +13,7 @@ RUN rm -rf /home/ue4/UnrealEngine/.git # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild -buildubt +RUN ./Engine/Build/BatchFiles/BuildUBT.sh # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine From 8756c7c453eb5aa34d6182a282776d1f9afd2195 Mon Sep 17 00:00:00 2001 From: Sam Collier Date: Tue, 24 Jun 2025 20:45:29 +0100 Subject: [PATCH 430/430] only run BuildUBT.sh if it's present otherwise fallback on Build.sh --- src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile index b965d5db..9fec768f 100644 --- a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -13,7 +13,11 @@ RUN rm -rf /home/ue4/UnrealEngine/.git # Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the # target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, # which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/BuildUBT.sh +RUN if [ -f ./Engine/Build/BatchFiles/BuildUBT.sh ]; then \ + ./Engine/Build/BatchFiles/BuildUBT.sh; \ + else \ + ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild -buildubt; \ + fi # Create an Installed Build of the Engine WORKDIR /home/ue4/UnrealEngine