diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..50ca329f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh eol=lf diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index d29555d6..00000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,20 +0,0 @@ -Guidelines for contributing to ue4-docker -========================================= - - -## Contents - -- [Creating issues](#creating-issues) -- [Creating pull requests](#creating-pull-requests) - - -## Creating issues - -When creating an issue to report a bug, please follow the provided issue template. Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. - -If you are creating an issue for a feature request, you can disregard the default issue template. However, please make sure you have thoroughly read [the documentation](https://adamrehn.com/docs/ue4-docker/) to ensure the requested feature does not already exist before creating an issue. - - -## Creating pull requests - -Before creating a pull request, please ensure sure you have tested your changes on all platforms to which the change applies (Linux / Windows Server / Windows 10 / macOS). diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 72a6f46a..7a264e17 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,4 +4,8 @@ Output of the `ue4-docker info` command: FULL OUTPUT OF ue4-docker info GOES HERE ``` -(The rest of the issue description goes here. If you're reporting a problem with building the container images, be sure to include the full output of the `ue4-docker build` command, along with the command line parameters used to invoke the build. If you're making a feature request, you can remove the template contents entirely, since the `ue4-docker info` output is only needed for helping diagnose and reproduce bugs.) +Additional details: + +- Are you accessing the network through a proxy server? **Yes/No** + +(The rest of the issue description goes here. If you're reporting a problem with building container images, be sure to include the full output of the `ue4-docker build` command, including the initial output lines that display the command line parameters used to invoke the build. If you're making a feature request, you can remove the template contents entirely, since the `ue4-docker info` output and related information is only needed for helping diagnose and reproduce bugs.) diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..e6c28dc8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml new file mode 100644 index 00000000..79c3ed97 --- /dev/null +++ b/.github/release-drafter.yml @@ -0,0 +1,8 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +version-resolver: + default: patch +template: | + ## Changes + + $CHANGES diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..59ab26fe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: CI +on: [push, pull_request] +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: psf/black@stable + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies + run: pip install --upgrade build twine + - name: Build package + run: python -m build + - name: Check package + run: twine check --strict dist/* + - name: Publish to PyPI + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + test: + strategy: + matrix: + include: + # All supported Ubuntu LTS with system python + # Be careful when removing EOL versions so that we still test our oldest supported python at least somewhere + - { os: ubuntu-22.04, python: "3.10" } + - { os: ubuntu-24.04, python: "3.12" } + + # All supported Visual Studio on Windows Server 2019 + - { os: windows-2019, python: "3.12", visual-studio: 2017 } + - { os: windows-2019, python: "3.12", visual-studio: 2019 } + - { os: windows-2019, python: "3.12", visual-studio: 2022 } + + # All supported Visual Studio on Windows Server 2022 + - { os: windows-2022, python: "3.12", visual-studio: 2017 } + - { os: windows-2022, python: "3.12", visual-studio: 2019 } + - { os: windows-2022, python: "3.12", visual-studio: 2022 } + + # All supported Visual Studio on Windows Server 2025 + # TODO: Waiting for https://github.com/actions/runner-images/issues/10806 + # - { os: windows-2022, python: "3.12", visual-studio: 2017 } + # - { os: windows-2022, python: "3.12", visual-studio: 2019 } + # - { os: windows-2022, python: "3.12", visual-studio: 2022 } + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install + run: pip install . --user + - name: Build prerequisites + run: ue4-docker build --target=build-prerequisites --visual-studio ${{ matrix.visual-studio || '2017' }} + - name: Run diagnostics + run: ue4-docker diagnostics all diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..047a94b4 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,47 @@ +name: Deploy docs to GitHub Pages + +# Only run this workflow when the YAML file itself changes or files change in the "docs" folder +on: + push: + branches: [master] + paths: + - ".github/workflows/docs.yml" + - "docs/**" + +# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages +permissions: + contents: read + pages: write + id-token: write + +# Allow one concurrent deployment +concurrency: + group: "pages" + cancel-in-progress: true + +jobs: + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build docs + run: docs/build.sh + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: "build/gh-pages" + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 00000000..f8bae4b3 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,13 @@ +name: Release Drafter + +on: + push: + branches: [ master ] + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v6 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 36b9f1fa..cb94132f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ +.idea/ .vscode/ __pycache__/ build dist +venv/ *.egg-info *.pyc diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 7990a448..00000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -if: tag IS present -dist: xenial -language: python -python: - - "3.7" -install: - - pip install wheel -script: echo "No build-time tests to run." -deploy: - provider: pypi - user: adamrehn - distributions: bdist_wheel - on: - tags: true - password: - secure: On10RovRYLb2S9TmoqzHCnHF1Bld5UJSbnoMCWytjwXKtp/+FphcT5htSNBKUpoBbjxdCT0we0Vy1GtGJi8auNZa+fhcegIikXVDg5iBn2UvgLVWrWZBQWDPCjkdL7fCij5MSAjJxBEuYRcfeGTM76wyYfYaO4omvNE96CVoWmkD3Ev0h1wrxeUoy7mAQ7Qy/46gaQ73LH0YnhaZ1s5frtESkoFsfqNwkrFY14Iu2gfnDZB6WDVqS00oOXhjiAjEyyGfCk6eW4GyUdVa2dWaapHDVeruvr6ICBj5DOXpf37dsIZvjqpWAqO76+4rbJ8tyPkjsr+DyQ2hq7a1Lq+TWlVLh78FkVoQATDRy/Y1Ai+X/nhnFkNrrIHHaS1B82Cnyq2XWLp/iUoxAZLTP7q11L4vrpOccSx8Ed5LaPoucTNvEVyM//eD8J8Z12OCue+ZAmWdq1E1slkfeBcp0bAkfTQ/9K0HJ7t0fCUj3+Y4DbSLB3H7zT0Ev6+roZWHzcwPf1Wct+Qwe2RdTZRdAt0C/tdSCTgB82/GrUWnaKLO4f2Flyr2BOUFMZDnOYR5MOs9826n8RnDz9mXDSqEZWOy01jjMS7H7jvKBQQwbVllpfxZqkd+JXumsf7k2l4nV3ETIQuU4NYExoTSe+Y32zP8Zgbe+Y9V0kte8a4m9FO7HvI= diff --git a/CONTRIBUTING.adoc b/CONTRIBUTING.adoc new file mode 100644 index 00000000..a63a2448 --- /dev/null +++ b/CONTRIBUTING.adoc @@ -0,0 +1,27 @@ += Guidelines for contributing to ue4-docker +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Creating issues + +When creating an issue to report a bug, please follow the provided issue template. +Make sure you include the full output of the `ue4-docker info` command in the provided section, as well as the full output of the `ue4-docker build` command and the command line parameters used to invoke the build if you are reporting a problem with building the container images. + +If you are creating an issue for a feature request, you can disregard the default issue template. +However, please make sure you have thoroughly read https://adamrehn.github.io/ue4-docker[the documentation] to ensure the requested feature does not already exist before creating an issue. + +== Creating pull requests + +Before creating a pull request, please ensure sure you have tested your changes on all platforms to which the change applies (Linux / Windows Server / Windows 10 / macOS). + +== Autoformatting your changes + +ue4-docker uses https://github.com/psf/black[Black] for its code formatting. +If you are submitting changes, make sure to install and run Black: + +* `pip install --user black` +* `python -m black .` (invoke in repository root) +* Now your code is properly formatted diff --git a/LICENSE b/LICENSE index e3d824d0..49363f78 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 - 2019 Adam Rehn +Copyright (c) 2018 - 2024 Adam Rehn Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index d900361a..4dd393b7 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,38 @@

Unreal Engine and Docker Logos

-

Unreal Engine 4 Docker Containers

-

Continuous Integration • Cloud Rendering • UE4-Powered Microservices

+

Unreal Engine Docker Containers

+

Continuous Integration • Pixel Streaming • Unreal Engine Powered Microservices

 

-**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://adamrehn.com/docs/ue4-docker/) to view details specific to using the ue4-docker project.** +**Looking for a place to start? Check out the [Unreal Containers community hub](https://unrealcontainers.com/) for implementation-agnostic information on using the Unreal Engine inside Docker containers, and then head to the [comprehensive ue4-docker documentation](https://adamrehn.github.io/ue4-docker) to view details specific to using the ue4-docker project.** -The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine 4](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' [Unreal Engine](https://www.unrealengine.com/). The images also incorporate the infrastructure from [ue4cli](https://github.com/adamrehn/ue4cli), [conan-ue4cli](https://github.com/adamrehn/conan-ue4cli), and [ue4-ci-helpers](https://github.com/adamrehn/ue4-ci-helpers) to facilitate a wide variety of use cases. Key features include: -- Unreal Engine 4.19.0 and newer is supported. +- The six most recent versions of the Unreal Engine are supported (currently Unreal Engine 4.27 and newer). - Both Windows containers and Linux containers are supported. -- Building and packaging UE4 projects is supported. +- Building and packaging Unreal Engine projects is supported. - Running automation tests is supported. -- Running built UE4 projects with offscreen rendering is supported via NVIDIA Docker under Linux. +- Running built Unreal Engine projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. Resources: -- **Documentation:** +- **Documentation:** - **GitHub repository:** -- **Package on PyPI:** +- **Package on PyPI:** - **Related articles:** - **Unreal Containers community hub:** +- **Development channel on the Unreal Containers Community Discord server**: ## Contributing -See the file [CONTRIBUTING.md](https://github.com/adamrehn/ue4-docker/blob/master/.github/CONTRIBUTING.md) for guidelines on how to contribute to the development of ue4-docker. +See the file [CONTRIBUTING.adoc](https://github.com/adamrehn/ue4-docker/blob/master/CONTRIBUTING.adoc) for guidelines on how to contribute to the development of ue4-docker. ## Legal -Copyright © 2018 - 2019, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. +Copyright © 2018 - 2024, Adam Rehn. Licensed under the MIT License, see the file [LICENSE](https://github.com/adamrehn/ue4-docker/blob/master/LICENSE) for details. Unreal and its logo are Epic Games' trademarks or registered trademarks in the US and elsewhere. diff --git a/docs/advanced-build-options.adoc b/docs/advanced-build-options.adoc new file mode 100644 index 00000000..2d56e3f8 --- /dev/null +++ b/docs/advanced-build-options.adoc @@ -0,0 +1,237 @@ += Advanced build options +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== General options + +=== Specifying Git credentials + +The xref:ue4-docker-build.adoc[ue4-docker build] command supports three methods for specifying the credentials that will be used to clone the Unreal Engine Git repository: + +- **Command-line arguments**: the `-username` and `-password` command-line arguments can be used to specify the username and password, respectively. + +- **Environment variables**: the `UE4DOCKER_USERNAME` and `UE4DOCKER_PASSWORD` environment variables can be used to specify the username and password, respectively. +Note that credentials specified via command-line arguments will take precedence over values defined in environment variables. + +- **Standard input**: if either the username or password has not been specified via a command-line argument or environment variable then the build command will prompt the user to enter the credential(s) for which values have not already been specified. + +Note that the username and password are handled independently, which means you can use different methods to specify the two credentials (e.g. username specified via command-line argument and password supplied via standard input.) + +Users who have enabled https://help.github.com/en/articles/about-two-factor-authentication[Two-Factor Authentication (2FA)] for their GitHub account will need to generate a https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line[personal access token] and use that in place of their password. + +=== Building a custom version of the Unreal Engine + +If you would like to build a custom version of Unreal Engine rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +When building a custom Engine version, **both the repository URL and branch/tag must be specified**: + +If you would like to build a custom version of Unreal Engine rather than one of the official releases from Epic, you can specify "custom" as the release string and specify the Git repository and branch/tag that should be cloned. +When building a custom Engine version, **both the repository URL and branch/tag must be specified**: + +[source,shell] +---- +ue4-docker build custom -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +---- + +This will produce images tagged `adamrehn/ue4-source:custom`, `adamrehn/ue4-minimal:custom`, etc. + +If you are performing multiple custom builds and wish to differentiate between them, it is recommended to also specify a name for the custom build: + +[source,shell] +---- +ue4-docker build custom:my-custom-build -repo=https://github.com/MyUser/UnrealEngine.git -branch=MyBranch +---- + +This will produce images tagged `adamrehn/ue4-source:my-custom-build`, `adamrehn/ue4-minimal:my-custom-build`, etc. + +[[exclude-components]] +=== Excluding Engine components to reduce the final image size + +Starting in ue4-docker version 0.0.30, you can use the `--exclude` flag when running the xref:ue4-docker-build.adoc[ue4-docker build] command to specify that certain Engine components should be excluded from the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:available-container-images.adoc#ue4-full[ue4-full] images. +The following components can be excluded: + +- `ddc`: disables building the DDC for the Engine. +This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. + +- `debug`: removes all debug symbols from the built images. +(When building Windows containers the files are actually truncated instead of removed, so they still exist but have a size of zero bytes. +This is done for compatibility reasons.) + +- `templates`: removes the template projects and samples that ship with the Engine. + +You can specify the `--exclude` flag multiple times to exclude as many components as you like. +For example: + +[source,shell] +---- +# Excludes both debug symbols and template projects +ue4-docker build 4.27.0 --exclude debug --exclude templates +---- + +=== Enabling system resource monitoring during builds + +Starting in ue4-docker version 0.0.46, you can use the `--monitor` flag to enable a background thread that will log information about system resource usage (available disk space and memory, CPU usage, etc.) at intervals during the build. +You can also use the `-interval` flag to override the default interval of 20 seconds: + +[source,shell] +---- +# Logs system resource levels every 20 seconds +ue4-docker build 4.27.0 --monitor +---- + +[source,shell] +---- +# Logs system resource levels every 20 seconds +ue4-docker build 4.27.0 --monitor +---- + +[source,shell] +---- +# Logs system resource levels every 5 seconds +ue4-docker build 4.27.0 --monitor -interval=5 +---- + +[[exporting-generated-dockerfiles]] +=== Exporting generated Dockerfiles + +Since ue4-docker version 0.0.78, the xref:ue4-docker-build.adoc[ue4-docker build] command supports a flag called `-layout` that allows the generated Dockerfiles to be exported to a filesystem directory instead of being built. +In addition, version 0.0.80 of ue4-docker added support for a flag called `--combine` that allows you to combine multiple generated Dockerfiles into a single Dockerfile that performs a https://docs.docker.com/develop/develop-images/multistage-build/[multi-stage build]. +You can use these flags like so: + +[source,shell] +---- +# Exports Dockerfiles for all images to the specified filesystem directory +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" +---- + +[source,shell] +---- +# Exports Dockerfiles for all images +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" +---- + +[source,shell] +---- +# Exports Dockerfiles for all images and combines them into a single Dockerfile +ue4-docker build 4.27.0 -layout "/path/to/Dockerfiles" --combine +---- + +Exporting Dockerfiles is useful for debugging or contributing to the development of ue4-docker itself. +You can also use the generated Dockerfiles to build container images independently of ue4-docker, but only under the following circumstances: + +- When building Windows container images, you must specify the <> `source_mode` and set it to `copy`. +This generates Dockerfiles that copy the Unreal Engine source code from the host filesystem rather than cloning it from a git repository, thus eliminating the dependency on ue4-docker's credential endpoint to securely provide git credentials and allowing container images to be built without the need for ue4-docker itself. + +[[advanced-options-for-dockerfile-generation]] +=== Advanced options for Dockerfile generation + +NOTE: Note that option names are all listed with underscores between words below (e.g. `source_mode`), but in some examples you will see dashes used as the delimiter instead (e.g. `source-mode`). **These uses are actually equivalent, since ue4-docker automatically converts any dashes in the option name into underscores.** This is because dashes are more stylistically consistent with command-line flags (and thus preferable in examples), but underscores must be used in the underlying Dockerfile template code since dashes cannot be used in https://jinja.palletsprojects.com/en/2.11.x/api/#notes-on-identifiers[Jinja identifiers]. + +Since ue4-docker version 0.0.78, the xref:ue4-docker-build.adoc[ue4-docker build] command supports a flag called `--opt` that allows users to directly set the context values passed to the underlying https://jinja.palletsprojects.com/[Jinja templating engine] used to generate Dockerfiles. +Some of these options (such as `source_mode`) can only be used when <>, whereas others can be used with the regular ue4-docker build process. **Note that incorrect use of these options can break build behaviour, so only use an option if you have read through both this documentation and the ue4-docker source code itself and understand exactly what that option does.** The following options are supported as of the latest version of ue4-docker: + +- **`source_mode`**: *(string)* controls how the xref:available-container-images.adoc#ue4-source[ue4-source] Dockerfile obtains the source code for the Unreal Engine. +Valid options are: + +- `git`: the default mode, whereby the Unreal Engine source code is cloned from a git repository. +This is the only mode that can be used when not <>. + +- `copy`: copies the Unreal Engine source code from the host filesystem. +The filesystem path can be specified using the `SOURCE_LOCATION` Docker build argument, and of course must be a child path of the build context. + +- **`credential_mode`**: *(string)* controls how the xref:available-container-images.adoc#ue4-source[ue4-source] Dockerfile securely obtains credentials for authenticating with remote git repositories when `source_mode` is set to `git`. +Valid options are: + +- `endpoint`: the default mode for Windows Containers, whereby ue4-docker exposes an HTTP endpoint that responds with credentials when presented with a randomly-generated security token, which is injected into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process by way of a Docker build argument. +This mode will not work when <>, since the credential endpoint will not be available during the build process. + +- `secrets`: **(Linux containers only)** default mode for Linux Containers, uses https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information[BuildKit build secrets] to securely inject the git credentials into the xref:available-container-images.adoc#ue4-source[ue4-source] container during the build process. + +- **`buildgraph_args`**: *(string)* allows you to specify additional arguments to pass to the https://docs.unrealengine.com/en-US/ProductionPipelines/BuildTools/AutomationTool/BuildGraph/index.html[BuildGraph system] when creating an Installed Build of the Unreal Engine in the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image. + +- **`disable_labels`**: *(boolean)* prevents ue4-docker from applying labels to built container images. +This includes the labels which specify the <> as well as the sentinel labels that the xref:ue4-docker-clean.adoc[ue4-docker clean] command uses to identify container images, and will therefore break the functionality of that command. + +- **`disable_all_patches`**: *(boolean)* disables all the patches that ue4-docker ordinarily applies to the Unreal Engine source code. +This is useful when building a custom fork of the Unreal Engine to which the appropriate patches have already been applied, **but will break the build process when used with a version of the Unreal Engine that requires one or more patches**. +It is typically safer to disable individual patches using the specific flag for each patch instead of simply disabling everything: + +- **`disable_release_patches`**: *(boolean)* disables the patches that ue4-docker ordinarily applies to versions of the Unreal Engine which are known to contain bugs. This will obviously break the build process when building these known broken releases, but will have no effect when building other versions of the Unreal Engine. + +- **`disable_windows_setup_patch`**: *(boolean)* prevents ue4-docker from patching `Setup.bat` under Windows to comment out the calls to the Unreal Engine prerequisites installer and UnrealVersionSelector, both of which are known to cause issues during the build process for Windows containers. + +== Windows-specific options + +[[windows-base-tag]] +=== Specifying the Windows Server Core base image tag + +NOTE: The `-basetag` flag controls how the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `-basetag` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. + +By default, Windows container images are based on the Windows Server Core release that best matches the version of the host operating system. +However, Windows containers cannot run a newer kernel version than that of the host operating system, rendering the latest images unusable under older versions of Windows 10 and Windows Server. +(See the https://docs.microsoft.com/en-us/virtualization/windowscontainers/deploy-containers/version-compatibility[Windows Container Version Compatibility] page for a table detailing which configurations are supported.) + +If you are building images with the intention of subsequently running them under an older version of Windows 10 or Windows Server, you will need to build images based on the same kernel version as the target system (or older.) The kernel version can be specified by providing the appropriate base OS image tag via the `-basetag=TAG` flag when invoking the build command: + +[source,shell] +---- +ue4-docker build 4.27.0 -basetag=ltsc2019 # Uses Windows Server 2019 (Long Term Support Channel) +---- + +For a list of supported base image tags, see the https://hub.docker.com/r/microsoft/windowsservercore/[Windows Server Core base image on Docker Hub]. + +[[windows-isolation-mode]] +=== Specifying the isolation mode under Windows + +The isolation mode can be explicitly specified via the `-isolation=MODE` flag when invoking the build command. +Valid values are `process` (supported under Windows Server and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[Windows 10 version 1809 or newer]) or `hyperv` (supported under both Windows 10 and Windows Server.) If you do not explicitly specify an isolation mode then the appropriate default for the host system will be used. + +=== Specifying Visual Studio Build Tools version under Windows + +=== Keeping or excluding Installed Build debug symbols under Windows + +WARNING: Excluding debug symbols is necessary under some versions of Docker as a workaround for a bug that limits the amount of data that a `COPY` directive can process to 8GB. +See xref:troubleshooting-build-issues.adoc#copy-8gb-20gb[this section of the Troubleshooting Build Issues page] for further details on this issue. + +Prior to version 0.0.30, ue4-docker defaulted to truncating all `.pdb` files when building the Installed Build for the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] Windows image. +This was done primarily to address the bug described in the warning alert above, and also had the benefit of reducing the overall size of the built container images. +However, if you required the debug symbols for producing debuggable builds, you had to opt to retain all `.pdb` files by specifying the `--keep-debug` flag when invoking the build command. +(This flag was removed in ue4-docker version 0.0.30, when the default behaviour was changed and replaced with a more generic, cross-platform approach.) + +Since ue4-docker version 0.0.30, debug symbols are kept intact by default, and can be removed by using the `--exclude debug` flag as described in the section <>. + +=== Building Linux container images under Windows + +By default, Windows container images are built when running the build command under Windows. +To build Linux container images instead, simply specify the `--linux` flag when invoking the build command. + +== Linux-specific options + +[[cuda]] +=== Enabling CUDA support for GPU-enabled Linux images + +IMPORTANT: The `--cuda` flag controls how the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image is built and tagged, which has a flow-on effect to all the other images. +If you are building multiple related images over separate invocations of the build command (e.g. building the xref:available-container-images.adoc#ue4-source[ue4-source] image in one command and then subsequently building the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image in another command), be sure to specify the same `--cuda` flag each time to avoid unintentionally building two sets of unrelated images with different configurations. + +By default, the Linux images built by ue4-docker support hardware-accelerated OpenGL when run via the NVIDIA Container Toolkit. +If you would like CUDA support in addition to OpenGL support, simply specify the `--cuda` flag when invoking the build command. + +You can also control the version of the CUDA base image that is used by appending a version number when specifying the `--cuda` flag, as demonstrated below: + +[source,shell] +---- +# Uses the default CUDA base image (currently CUDA 9.2) +ue4-docker build RELEASE --cuda +---- + +[source,shell] +---- +# Uses the CUDA 12.2.0 base image +ue4-docker build RELEASE --cuda=12.2.0 +{% endhighlight %} +---- + +For a list of supported CUDA versions, see the list of Ubuntu 22.04 image tags for the https://hub.docker.com/r/nvidia/cuda/[nvidia/cuda] base image. diff --git a/docs/available-container-images.adoc b/docs/available-container-images.adoc new file mode 100644 index 00000000..4efe2c62 --- /dev/null +++ b/docs/available-container-images.adoc @@ -0,0 +1,97 @@ += List of available container images +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +You can build the following images using the xref:ue4-docker-build.adoc[ue4-docker build] command: + +- <> +- <> +- <> +- <> + +By default, all available images will be built. +You can prevent unwanted images from being built by appending the relevant image-specific flag to the build command (see the *"Flag to disable"* entry for each image below.) + +[[ue4-build-prerequisites]] +== ue4-build-prerequisites + +**Tags:** + +* **Windows containers**: `adamrehn/ue4-build-prerequisites:BASETAG` where `BASETAG` is the xref:advanced-build-options.adoc#windows-base-tag[Windows Server Core base image tag] + +* **Linux containers**: `adamrehn/ue4-build-prerequisites:CONFIGURATION` where `CONFIGURATION` is as follows: + +** `opengl` if CUDA support is not enabled + +** `cudaVERSION` where `VERSION` is the CUDA version if xref:advanced-build-options.adoc#cuda[CUDA support is enabled] (e.g. `cuda9.2`, `cuda10.0`, etc.) + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the build prerequisites common to all Engine versions. + +**Uses:** + +* Keep this image on disk to speed up subsequent container image builds. + +[[ue4-source]] +== ue4-source + +**Tags:** + +* `adamrehn/ue4-source:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-source:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-source/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the cloned source code for Unreal Engine, along with its downloaded dependency data. +The ue4-minimal image uses this source code as the starting point for its build. + +**Uses:** + +* Only needed during the build process. +Afterwards, this image can be removed using `ue4-docker clean --source` to save disk space. + +[[ue4-minimal]] +== ue4-minimal + +**Tags:** + +* `adamrehn/ue4-minimal:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-minimal:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/master/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains the absolute minimum set of components required for use in a Continuous Integration (CI) pipeline, consisting of only the build prerequisites and an Installed Build of the Engine. + +**Uses:** + +* Use this image for xref:continuous-integration.adoc[CI pipelines] that do not require ue4cli, conan-ue4cli, or ue4-ci-helpers. + +[[ue4-full]] +== ue4-full + +**Tags:** + +* `adamrehn/ue4-full:RELEASE` where `RELEASE` is the Engine release number + +* `adamrehn/ue4-full:RELEASE-PREREQS` where `RELEASE` is as above and `PREREQS` is the <> image tag + +**Dockerfiles:** https://github.com/adamrehn/ue4-docker/tree/src/master/ue4docker/dockerfiles/ue4-full/windows/Dockerfile[icon:windows[] Windows] | https://github.com/adamrehn/ue4-docker/tree/master/ue4docker/dockerfiles/ue4-full/linux/Dockerfile[icon:linux[] Linux] + +**Contents:** Contains everything from the `ue4-minimal` image, and adds the following: + +* ue4cli +* conan-ue4cli +* ue4-ci-helpers +* PulseAudio support (Linux image only) +* X11 support (Linux image only) + +**Uses:** + +* xref:continuous-integration.adoc[CI pipelines] that require ue4cli, conan-ue4cli, or ue4-ci-helpers +* Packaging xref:microservices.adoc[Unreal Engine-powered microservices] diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..53028626 --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# See http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -euo pipefail +IFS=$'\n\t' + +project_dir="$(git rev-parse --show-toplevel)" + +function docker_asciidoctor() { + docker run --rm --user "$(id -u):$(id -g)" --volume "${project_dir}:/project/" asciidoctor/docker-asciidoctor "$@" +} + +# HTML +docker_asciidoctor asciidoctor -a linkcss /project/docs/index.adoc -D /project/build/gh-pages/ + +# PDF +docker_asciidoctor asciidoctor-pdf /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.pdf + +# EPUB +docker_asciidoctor asciidoctor-epub3 /project/docs/index.adoc -o /project/build/gh-pages/ue4-docker.epub + +# Manpages +for f in docs/ue4-docker-*.adoc +do + docker_asciidoctor asciidoctor -b manpage "/project/${f}" -D /project/build/gh-pages/ +done diff --git a/docs/configuring-linux.adoc b/docs/configuring-linux.adoc new file mode 100644 index 00000000..585108c5 --- /dev/null +++ b/docs/configuring-linux.adoc @@ -0,0 +1,72 @@ += Configuring Linux +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Requirements + +- 64-bit version of one of Docker's https://docs.docker.com/install/#supported-platforms[supported Linux distributions] (CentOS 7+, Debian 7.7+, Fedora 26+, Ubuntu 14.04+) +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker CE + +Follow the official installation instructions from the Docker Documentation for your distribution: + +- https://docs.docker.com/install/linux/docker-ce/centos/[CentOS] +- https://docs.docker.com/install/linux/docker-ce/debian/[Debian] +- https://docs.docker.com/install/linux/docker-ce/fedora/[Fedora] +- https://docs.docker.com/install/linux/docker-ce/ubuntu/[Ubuntu] + +Once Docker is installed, follow the instructions from the https://docs.docker.com/install/linux/linux-postinstall/#manage-docker-as-a-non-root-user[Post-installation steps for Linux] page of the Docker Documentation to allow Docker commands to be run by a non-root user. +This step is required in order to enable audio support when performing cloud rendering using the NVIDIA Container Toolkit. + +== Step 2: Install Python 3.8 or newer + +WARNING: Note that older versions of these Linux distributions may not have Python 3.8 available in their system repositories by default. +When working with an older distribution it may be necessary to configure community repositories that provide newer versions of Python. + +Under CentOS, run: + +[source,shell] +---- +sudo yum install python3 python3-devel python3-pip +---- + +Under Debian and Ubuntu, run: + +[source,shell] +---- +sudo apt-get install python3 python3-dev python3-pip +---- + +Under Fedora, run: + +[source,shell] +---- +sudo dnf install python3 python3-devel python3-pip +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command: + +[source,shell] +---- +sudo pip3 install ue4-docker +---- + +== Step 4: Use ue4-docker to automatically configure the Linux firewall + +If the host system is running an active firewall that blocks access to port 9876 (required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image) then it is necessary to create a firewall rule to permit access to this port. +The xref:ue4-docker-setup.adoc[ue4-docker setup] command will detect this scenario and perform the appropriate firewall configuration automatically. +Simply run: + +[source,shell] +---- +sudo ue4-docker setup +---- + +Note that the `iptables-persistent` service will need to be installed for the newly-created firewall rule to persist after the host system reboots. diff --git a/docs/configuring-macos.adoc b/docs/configuring-macos.adoc new file mode 100644 index 00000000..ca4434a7 --- /dev/null +++ b/docs/configuring-macos.adoc @@ -0,0 +1,51 @@ += Configuring macOS +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +WARNING: macOS provides a suboptimal experience when running Linux containers, due to the following factors: +Linux containers are unable to use GPU acceleration via the xref:nvidia-docker-primer.adoc[NVIDIA Container Toolkit]. + +== Requirements + +- 2010 or newer model Mac hardware +- macOS 10.10.3 Yosemite or newer +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker CE for Mac + +Download and install https://store.docker.com/editions/community/docker-ce-desktop-mac[Docker CE for Mac from the Docker Store]. + +== Step 2: Install Python 3 via Homebrew + +The simplest way to install Python 3 and pip under macOS is to use the https://brew.sh/[Homebrew package manager]. +To do so, run the following commands from a Terminal prompt: + +[source,shell] +---- +# Install Homebrew +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +---- + +== Install Python + +[source,shell] +---- +brew install python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from a Terminal prompt: + +[source,shell] +---- +sudo pip3 install ue4-docker +---- + +== Step 4: Manually configure Docker daemon settings + +Use https://docs.docker.com/desktop/mac/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 800GB. diff --git a/docs/configuring-windows-10.adoc b/docs/configuring-windows-10.adoc new file mode 100644 index 00000000..03ed51f4 --- /dev/null +++ b/docs/configuring-windows-10.adoc @@ -0,0 +1,75 @@ += Configuring Windows 10 and 11 +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Warning + +Windows 10 and 11 provide an optimal experience when running Windows containers, but **only when process isolation mode is used**. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:windows-container-primer.adoc[several issues that impact performance and stability]. +The default isolation mode depends on the specific version of Windows being used: + +- Under Windows 10, Hyper-V isolation mode is the default isolation mode and https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-[process isolation mode must be manually enabled] each time a container is built or run. +The xref:ue4-docker-build.adoc[ue4-docker build] command will automatically pass the flag to enable process isolation mode where possible. **This requires Windows 10 version 1809 or newer.** + +- Under Windows 11, process isolation mode is the default isolation mode. + +== Requirements + +- 64-bit Windows 10 Pro, Enterprise, or Education (Version 1607 or newer) +- Hardware-accelerated virtualization enabled in the system BIOS/EFI +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker CE for Windows + +Download and install https://store.docker.com/editions/community/docker-ce-desktop-windows[Docker CE for Windows from the Docker Store]. + +== Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the https://chocolatey.org/[Chocolatey package manager]. +To do so, run the following command from an elevated PowerShell prompt: + +[source,powershell] +---- +Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +---- + +You may need to restart your shell for it to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. + +Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +[source,powershell] +---- +choco install -y python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +[source,powershell] +---- +pip install ue4-docker +---- + +== Step 4: Manually configure Docker daemon settings + +For building and running Windows containers: + +- Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Windows containers] rather than Linux containers. +- Configure the Docker daemon to increase the maximum container disk size from the default 20GB limit by following https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits[the instructions provided by Microsoft]. +The 120GB limit specified in the instructions is not quite enough, so set an 800GB limit instead. **Be sure to restart the Docker daemon after applying the changes to ensure they take effect.** + +WARNING: **The ue4-docker maintainers do not provide support for building and running Linux containers under Windows**, due to the various technical limitations of the Hyper-V and WSL2 backends for Docker Desktop (see https://github.com/adamrehn/ue4-docker/issues/205[this issue] for details of these limitations). +This functionality is still present in ue4-docker for those who choose to use it, but users are solely responsible for troubleshooting any issues they encounter when doing so. + +For building and running Linux containers: + +- Configure the Docker daemon to https://docs.docker.com/desktop/windows/#switch-between-windows-and-linux-containers[use Linux containers] rather than Windows containers. + +- **If you are using the Hyper-V backend** then use https://docs.docker.com/desktop/windows/#resources[the Advanced section under the Resources tab of the Docker Desktop settings pane] to set the memory allocation for the Moby VM to 8GB and the maximum VM disk image size to 800GB. + +- **If you are using the WSL2 backend** then https://docs.microsoft.com/en-us/windows/wsl/compare-versions#expanding-the-size-of-your-wsl-2-virtual-hard-disk[expand the WSL2 virtual hard disk] to at least 800GB. diff --git a/docs/configuring-windows-server.adoc b/docs/configuring-windows-server.adoc new file mode 100644 index 00000000..2d1b2de8 --- /dev/null +++ b/docs/configuring-windows-server.adoc @@ -0,0 +1,70 @@ += Configuring Windows Server +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +IMPORTANT: Windows Server provides an optimal experience when running Windows containers, but **only when process isolation mode is used**. +Using https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode] will result in a suboptimal experience due to xref:windows-container-primer.adoc[several issues that impact performance and stability]. +Process isolation mode is the default isolation mode under Windows Server. + +== Requirements + +- Windows Server 2016 or newer +- Minimum 8GB of RAM +- Minimum 800GB available disk space for building container images + +== Step 1: Install Docker EE + +As per the instructions provided by the https://docs.docker.com/install/windows/docker-ee/[Install Docker Engine - Enterprise on Windows Servers] page of the Docker Documentation, run the following commands from an elevated PowerShell prompt: + +[source,powershell] +---- +# Add the Docker provider to the PowerShell package manager +Install-Module DockerMsftProvider -Force + +# Install Docker EE +Install-Package Docker -ProviderName DockerMsftProvider -Force + +# Restart the computer to enable the containers feature +Restart-Computer +---- + +== Step 2: Install Python 3 via Chocolatey + +The simplest way to install Python and pip under Windows is to use the https://chocolatey.org/[Chocolatey package manager]. +To do so, run the following command from an elevated PowerShell prompt: + +[source,powershell] +---- +Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +---- + +You may need to restart the system for your shell to recognise the updates that the Chocolatey installer makes to the system `PATH` environment variable. +Once these changes are recognised, you can install Python by running the following command from either an elevated PowerShell prompt or an elevated Command Prompt: + +[source,shell] +---- +choco install -y python +---- + +== Step 3: Install ue4-docker + +Install the ue4-docker Python package by running the following command from an elevated Command Prompt: + +[source,shell] +---- +pip install ue4-docker +---- + +== Step 4: Use ue4-docker to automatically configure Docker and Windows Firewall + +To automatically configure the required system settings, run the xref:ue4-docker-setup.adoc[ue4-docker setup] command from an elevated Command Prompt: + +[source,shell] +---- +ue4-docker setup +---- + +This will configure the Docker daemon to set the maximum image size to 800GB, create a Windows Firewall rule to allow Docker containers to communicate with the host system (which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image), and download any required DLL files under Windows Server version 1809 and newer. diff --git a/docs/continuous-integration.adoc b/docs/continuous-integration.adoc new file mode 100644 index 00000000..5e9127fa --- /dev/null +++ b/docs/continuous-integration.adoc @@ -0,0 +1,11 @@ += Continuous Integration (CI) +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Use a controlled and reproducible environment to build, test, and package Unreal projects. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/continuous-integration[Use Cases: Continuous Integration and Deployment (CI/CD)]. diff --git a/docs/feedback.adoc b/docs/feedback.adoc new file mode 100644 index 00000000..37973b61 --- /dev/null +++ b/docs/feedback.adoc @@ -0,0 +1,3 @@ +ifdef::backend-html5[] +image::https://img.shields.io/badge/improve-this%20doc-orange.svg[link=https://github.com/adamrehn/ue4-docker/edit/master/docs/{filename},float=right] +endif::[] diff --git a/docs/frequently-asked-questions.adoc b/docs/frequently-asked-questions.adoc new file mode 100644 index 00000000..ebe121f8 --- /dev/null +++ b/docs/frequently-asked-questions.adoc @@ -0,0 +1,30 @@ += Frequently Asked Questions +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== Why are the Dockerfiles written in such an inefficient manner? There are a large number of `RUN` directives that could be combined to improve both build efficiency and overall image size. + +With the exception of the ../building-images/available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] and ../building-images/available-container-images.adoc#ue4-minimal[ue4-minimal] images, the Dockerfiles have been deliberately written in an inefficient way because doing so serves two very important purposes. + +The first purpose is self-documentation. +These Docker images are the first publicly-available Windows and Linux images to provide comprehensive build capabilities for Unreal Engine 4. +Along with the supporting documentation and https://adamrehn.com/articles/tag/Unreal%20Engine/[articles on adamrehn.com], the code in this repository represents an important source of information regarding the steps that must be taken to get Unreal Engine working correctly inside a container. +The readability of the Dockerfiles is key, which is why they contain so many individual `RUN` directives with explanatory comments. +Combining `RUN` directives would reduce readability and potentially obfuscate the significance of critical steps. + +The second purpose is debuggability. +Updating the Dockerfiles to ensure compatibility with new Unreal Engine releases is an extremely involved process that typically requires building the Engine many times over. +By breaking the Dockerfiles into many fine-grained `RUN` directives, the Docker build cache can be leveraged to ensure only the failing steps need to be repeated when rebuilding the images during debugging. +Combining `RUN` directives would increase the amount of processing that needs to be redone each time one of the commands in a given directive fails, significantly increasing overall debugging times. + +== Can Windows containers be used to perform cloud rendering in the same manner as Linux containers with the NVIDIA Container Toolkit? + +See the answer here: https://unrealcontainers.com/docs/concepts/nvidia-docker#is-there-any-support-for-other-platforms + +== Is it possible to build Unreal projects for macOS or iOS using the Docker containers? + +Building projects for macOS or iOS requires a copy of macOS and Xcode. +Since macOS cannot run inside a Docker container, there is unfortunately no way to perform macOS or iOS builds using Docker containers. diff --git a/docs/index.adoc b/docs/index.adoc new file mode 100644 index 00000000..8c69755f --- /dev/null +++ b/docs/index.adoc @@ -0,0 +1,133 @@ += ue4-docker user manual +Adam Rehn ; other contributors +{docdate} +:doctype: book +:icons: font +:idprefix: +:idseparator: - +:sectanchors: +:sectlinks: +:source-highlighter: rouge +:toc: left + +== Read these first + +:filename: introduction-to-ue4-docker.adoc +include::feedback.adoc[] +include::introduction-to-ue4-docker.adoc[leveloffset=+2] + +:filename: large-container-images-primer.adoc +include::feedback.adoc[] +include::large-container-images-primer.adoc[leveloffset=+2] + +:filename: windows-container-primer.adoc +include::feedback.adoc[] +include::windows-container-primer.adoc[leveloffset=+2] + +:filename: nvidia-docker-primer.adoc +include::feedback.adoc[] +include::nvidia-docker-primer.adoc[leveloffset=+2] + +[[configuration]] +== Configuration + +:filename: supported-host-configurations.adoc +include::feedback.adoc[] +include::supported-host-configurations.adoc[leveloffset=+2] + +:filename: configuring-linux.adoc +include::feedback.adoc[] +include::configuring-linux.adoc[leveloffset=+2] + +:filename: configuring-windows-server.adoc +include::feedback.adoc[] +include::configuring-windows-server.adoc[leveloffset=+2] + +:filename: configuring-windows-10.adoc +include::feedback.adoc[] +include::configuring-windows-10.adoc[leveloffset=+2] + +:filename: configuring-macos.adoc +include::feedback.adoc[] +include::configuring-macos.adoc[leveloffset=+2] + +[[use-cases]] +== Use cases + +:filename: use-cases-overview.adoc +include::feedback.adoc[] +include::use-cases-overview.adoc[leveloffset=+2] + +:filename: continuous-integration.adoc +include::feedback.adoc[] +include::continuous-integration.adoc[leveloffset=+2] + +:filename: microservices.adoc +include::feedback.adoc[] +include::microservices.adoc[leveloffset=+2] + +:filename: linux-installed-builds +include::feedback.adoc[] +include::linux-installed-builds.adoc[leveloffset=+2] + +== Building images + +:filename: available-container-images.adoc +include::feedback.adoc[] +include::available-container-images.adoc[leveloffset=+2] + +:filename: advanced-build-options.adoc +include::feedback.adoc[] +include::advanced-build-options.adoc[leveloffset=+2] + +:filename: troubleshooting-build-issues.adoc +include::feedback.adoc[] +include::troubleshooting-build-issues.adoc[leveloffset=+2] + +== Command reference + +:filename: ue4-docker-build.adoc +include::feedback.adoc[] +include::ue4-docker-build.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-clean.adoc +include::feedback.adoc[] +include::ue4-docker-clean.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-diagnostics.adoc +include::feedback.adoc[] +include::ue4-docker-diagnostics.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-export.adoc +include::feedback.adoc[] +include::ue4-docker-export.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-info.adoc +include::feedback.adoc[] +include::ue4-docker-info.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-setup.adoc +include::feedback.adoc[] +include::ue4-docker-setup.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-test.adoc +include::feedback.adoc[] +include::ue4-docker-test.adoc[leveloffset=+2] + +<<< + +:filename: ue4-docker-version.adoc +include::feedback.adoc[] +include::ue4-docker-version.adoc[leveloffset=+2] diff --git a/docs/introduction-to-ue4-docker.adoc b/docs/introduction-to-ue4-docker.adoc new file mode 100644 index 00000000..55a100e4 --- /dev/null +++ b/docs/introduction-to-ue4-docker.adoc @@ -0,0 +1,49 @@ += Introduction to ue4-docker +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +The ue4-docker Python package contains a set of Dockerfiles and accompanying build infrastructure that allows you to build Docker images for Epic Games' https://www.unrealengine.com/[Unreal Engine]. +The images also incorporate the infrastructure from `ue4cli`, `conan-ue4cli`, and `ue4-ci-helpers` to facilitate a wide variety of use cases. + +Key features include: + +- Unreal Engine 4.20.0 and newer is supported. +- Both Windows containers and Linux containers are supported. +- Building and packaging Unreal Engine projects is supported. +- Running automation tests is supported. +- Running built Unreal Engine projects with offscreen rendering is supported via the NVIDIA Container Toolkit under Linux. + +== Important legal notice + +**Except for the xref:available-container-images.adoc#ue4-build-prerequisites[ue4-build-prerequisites] image, the Docker images produced by the ue4-docker Python package contain the Unreal Engine Tools in both source code and object code form. +As per Section 1A of the https://www.unrealengine.com/eula[Unreal Engine EULA], Engine Licensees are prohibited from public distribution of the Engine Tools unless such distribution takes place via the Unreal Marketplace or a fork of the Epic Games Unreal Engine GitHub repository. +**Public distribution of the built images via an openly accessible Docker Registry (e.g. Docker Hub) is a direct violation of the license terms.** It is your responsibility to ensure that any private distribution to other Engine Licensees (such as via an organisation's internal Docker Registry) complies with the terms of the Unreal Engine EULA.** + +For more details, see the https://unrealcontainers.com/docs/obtaining-images/eula-restrictions[Unreal Engine EULA Restrictions] page on the https://unrealcontainers.com/[Unreal Containers community hub]. + +== Getting started + +Multipurpose Docker images for large, cross-platform projects such as the Unreal Engine involve a great deal more complexity than most typical Docker images. +Before you start using ue4-docker it may be helpful to familiarise yourself with some of these complexities: + +- If you've never built large (multi-gigabyte) Docker images before, be sure to read the xref:large-container-images-primer.adoc[Large container images primer]. +- If you've never used Windows containers before, be sure to read the xref:windows-container-primer.adoc[Windows containers primer]. +- If you've never used GPU-accelerated Linux containers with the NVIDIA Container Toolkit before, be sure to read the xref:nvidia-docker-primer.adoc[NVIDIA Container Toolkit primer]. + +Once you're familiar with all the relevant background material, you can dive right in: + +1. First up, head to the xref:#configuration[Configuration] section for details on how to install ue4-docker and configure your host system so that it is ready to build and run the Docker images. +2. Next, check out the xref:#use-cases[Use Cases] section for details on the various scenarios in which the Docker images can be used. +Once you've selected the use case you're interested in, you'll find step-by-step instructions on how to build the necessary container images and start using them. +3. If you run into any issues or want to customise your build with advanced options, the <<#building-images,Building Images>> section provides all the relevant details. +4. For more information, check out the xref:frequently-asked-questions.adoc[FAQ] and the <<#command-reference,Command Reference>> section. + +== Links + +- https://github.com/adamrehn/ue4-docker[ue4-docker GitHub repository] +- https://pypi.org/project/ue4-docker/[ue4-docker package on PyPI] +- https://adamrehn.com/articles/tag/Unreal%20Engine/[Related articles on adamrehn.com] +- https://unrealcontainers.com/[Unreal Containers community hub] diff --git a/docs/large-container-images-primer.adoc b/docs/large-container-images-primer.adoc new file mode 100644 index 00000000..0d2563b9 --- /dev/null +++ b/docs/large-container-images-primer.adoc @@ -0,0 +1,32 @@ += Large container images primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Although large container images are in no way different to smaller container images at a technical level, there are several aspects of the Docker build process that impact large images to a far greater extent than smaller images. +This page provides an overview for users who have never built large (multi-gigabyte) container images before and may therefore be unfamiliar with these impacts. +This information applies equally to both Linux containers and Windows containers. + +== Filesystem layer commit performance + +The time taken to commit filesystem layers to disk when building smaller Docker images is low enough that many users may not even perceive this process as a distinct aspect of a `RUN` step in a Dockerfile. +However, when a step generates a filesystem layer that is multiple gigabytes in size, the time taken to commit this data to disk becomes immediately noticeable. +For some larger layers in the container images built by ue4-docker, the filesystem layer commit process can take well over 40 minutes to complete on consumer-grade hardware. +(The Installed Build layer in the multi-stage build of the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] image is the largest of all the filesystem layers, and has been observed to take well over an hour and a half to commit to disk on some hardware.) + +Since Docker does not emit any output during the layer commit process, users may become concerned that the build has hung. +After the ue4-docker provided output `Performing filesystem layer commit...`, the only indication that any processing is taking place is the high quantity of CPU usage and disk I/O present during this stage. +There is no need for concern, as none of the steps in the ue4-docker Dockerfiles can run indefinitely without failing and emitting an error. +When a build step ceases to produce output, it is merely a matter of waiting for the filesystem layer commit to complete. + +== Disk space consumption during the build process + +Due to overheads associated with temporary layers in multi-stage builds and layer difference computation, the Docker build process for an image will consume more disk space than is required to hold the final built image. +These overheads are relatively modest when building smaller container images. +However, these overheads are exacerbated significantly when building large container images, and it is important to be aware of the quantity of available disk space that is required to build any given image or set of images. + +Although none of the container images produced by ue4-docker currently exceed 100GB in size, the build process requires at least 400GB of available disk space under Linux and at least 800GB of available disk space under Windows. +Once a build is complete, the xref:ue4-docker-clean.adoc[ue4-docker clean] command can be used to clean up temporary layers leftover from multi-stage builds and reclaim all the disk space not occupied by the final built images. +The https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] command can also be useful for cleaning up data that is not used by any of the tagged images present on the system. diff --git a/docs/linux-installed-builds.adoc b/docs/linux-installed-builds.adoc new file mode 100644 index 00000000..38a3fa59 --- /dev/null +++ b/docs/linux-installed-builds.adoc @@ -0,0 +1,12 @@ += Linux Installed Builds +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Under Windows and macOS, Engine licensees can easily download and manage Installed Builds of Unreal Engine through the Epic Games Launcher. +Since version 4.21.0 of the Engine, ue4-docker provides an alternative source of Installed Builds under Linux in lieu of a native version of the launcher. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/linux-installed-builds[Use Cases: Linux Installed Builds]. diff --git a/docs/microservices.adoc b/docs/microservices.adoc new file mode 100644 index 00000000..9591d7e9 --- /dev/null +++ b/docs/microservices.adoc @@ -0,0 +1,11 @@ += Microservices +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +Thanks to the inclusion of conan-ue4cli infrastructure, the ue4-full image makes it easy to build Unreal Engine-powered microservices with Google's popular gRPC framework. + +NOTE: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/use-cases/microservices[Use Cases: Microservices]. diff --git a/docs/nvidia-docker-primer.adoc b/docs/nvidia-docker-primer.adoc new file mode 100644 index 00000000..8d023cc1 --- /dev/null +++ b/docs/nvidia-docker-primer.adoc @@ -0,0 +1,9 @@ += NVIDIA Container Toolkit primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +IMPORTANT: This page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/concepts/nvidia-docker[Key Concepts: NVIDIA Container Toolkit]. diff --git a/docs/supported-host-configurations.adoc b/docs/supported-host-configurations.adoc new file mode 100644 index 00000000..619d7dee --- /dev/null +++ b/docs/supported-host-configurations.adoc @@ -0,0 +1,43 @@ += Supported host configurations +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +The table below lists the host operating systems can be used to build and run the container images produced by ue4-docker, as well as which features are supported under each system. + +**Click on an operating system's name to view the configuration instructions for that platform.** + +[%autowidth.stretch] +|=== +| Host OS | Linux containers | Windows containers | NVIDIA Container Toolkit | Optimality + +| xref:configuring-linux.adoc[Linux] +| icon:check[] +| icon:times[] +| icon:check[] +| Optimal for Linux containers + +| xref:configuring-windows-server.adoc[Windows Server] +| icon:times[] +| icon:check[] +| icon:times[] +| Optimal for Windows containers when using process isolation mode + +| xref:configuring-windows-10.adoc[Windows 10 and 11] +| Works but not tested or supported +| icon:check[] +| icon:times[] +| Optimal for Windows containers when using process isolation mode + +| xref:configuring-macos.adoc[macOS] +| icon:check[] +| icon:times[] +| icon:times[] +| Suboptimal for Linux containers + +|=== + +The *Optimality* column indicates whether a given host operating system provides the best experience for running the container types that it supports. +The configuration instructions page for each operating system provides further details regarding the factors that make it either optimal or suboptimal. diff --git a/docs/troubleshooting-build-issues.adoc b/docs/troubleshooting-build-issues.adoc new file mode 100644 index 00000000..b7f94a01 --- /dev/null +++ b/docs/troubleshooting-build-issues.adoc @@ -0,0 +1,152 @@ += Troubleshooting build issues +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +== General issues + +=== Building the `ue4-build-prerequisites` image fails with a network-related error + +This indicates an underlying network or proxy server issue outside ue4-docker itself that you will need to troubleshoot. +You can use the xref:ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to test container network connectivity during the troubleshooting process. +Here are some steps to try: + +- If your host system accesses the network through a proxy server, make sure that https://docs.docker.com/network/proxy/[Docker is configured to use the correct proxy server settings]. +- If DNS resolution is failing then try adding the following entry to your https://docs.docker.com/engine/reference/commandline/dockerd/#daemon-configuration-file[Docker daemon configuration file] (accessed through the Docker Engine settings pane if you're using Docker Desktop) and restarting the daemon: + +[source,json] +---- +{ + "dns": ["8.8.8.8"] +} +---- + +=== Cloning the UnrealEngine Git repository fails with the message `error: unable to read askpass response from 'C:\git-credential-helper.bat'` (for Windows containers) or `'/tmp/git-credential-helper.sh'` (for Linux containers) + +This typically indicates that the firewall on the host system is blocking connections from the Docker container, preventing it from retrieving the Git credentials supplied by the build command. +This is particularly noticeable under a clean installation of Windows Server, which blocks connections from other subnets by default. +The firewall will need to be configured appropriately to allow the connection, or else temporarily disabled. +(Under Windows Server, the xref:ue4-docker-setup.adoc[ue4-docker setup] command can configure the firewall rule for you automatically.) + +=== Building the Derived Data Cache (DDC) for the Installed Build of the Engine fails with a message about failed shader compilation or being unable to open a `.uasset` file + +This is typically caused by insufficient available disk space. +To fix this, simply free up some disk space and run the build again. +Running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune] can be helpful for freeing up space occupied by untagged images. +Note that restarting the Docker daemon and/or rebooting the host system may also help, since some versions of Docker have a bug that results in the amount of required disk space slowly increasing as more and more builds are run. + +=== Building Windows containers fails with the message `hcsshim::ImportLayer failed in Win32: The system cannot find the path specified` or building Linux containers fails with a message about insufficient disk space + +Assuming you haven't actually run out of disk space, this means that the maximum Docker image size has not been configured correctly. + +- For Windows containers, follow https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size[the instructions provided by Microsoft], making sure you restart the Docker daemon after you've modified the config JSON. +(Under Windows Server, the xref:ue4-docker-setup.adoc[ue4-docker setup] command can configure this for you automatically.) +- For Linux containers, use the https://docs.docker.com/docker-for-windows/#advanced[Docker for Windows "Advanced" settings tab] under Windows or the https://docs.docker.com/docker-for-mac/#disk[Docker for Mac "Disk" settings tab] under macOS. + +[[copy-8gb-20gb]] +=== Building the `ue4-minimal` image fails on the `COPY --from=builder` directive that copies the Installed Build from the intermediate image into the final image + +WARNING: Modern versions of Docker Desktop for Windows and Docker EE for Windows Server suffer from issues with 8GiB filesystem layers, albeit due to different underlying bugs. +Since ue4-docker version 0.0.47, you can use the xref:ue4-docker-diagnostics.adoc[ue4-docker diagnostics] command to check whether the Docker daemon on your system suffers from this issue. +If it does, you may need to xref:advanced-build-options.adoc#exclude-components[exclude debug symbols] when building Windows images. + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger: + +- https://github.com/moby/moby/issues/37581 (affects all platforms) +- https://github.com/moby/moby/issues/40444 (affects Windows containers only) + +https://github.com/moby/moby/issues/37581[#37581] was https://github.com/moby/moby/pull/37771[fixed] in Docker CE 18.09.0, whilst https://github.com/moby/moby/issues/40444[#40444] was https://github.com/moby/moby/pull/41430[fixed] in Docker CE 20.10.0. + +If you are using a version of Docker that contains one of these bugs then you will need to xref:advanced-build-options.adoc#exclude-components[exclude debug symbols], which reduces the size of the Installed Build below the 8GiB threshold. +If debug symbols are required then it will be necessary to upgrade or downgrade to a version of Docker that does not suffer from the 8GiB size limit issue (although finding such a version under Windows may prove quite difficult.) + +== Linux-specific issues + +=== Building the Engine in a Linux container fails with an error indicating that a compatible version of clang cannot be found or the file `ToolchainVersion.txt` is missing + +This is typically caused by the download of the Unreal Engine's toolchain archive from the Epic Games CDN becoming corrupted and failing to extract correctly. +This issue can occur both inside containers and when running directly on a host system, and the fix is to simply delete the corrupted files and try again: + +. Untag the available-container-images.adoc#ue4-source[ue4-source] image. +. Clear the Docker filesystem layer cache by running https://docs.docker.com/engine/reference/commandline/system_prune/[docker system prune]. +. Re-run the xref:ue4-docker-build.adoc[ue4-docker build] command. + +=== Building Unreal Engine 5.0 in a Linux container fails with the error message `No usable version of libssl was found` + +This is a known compatibility issue that affects Unreal Engine 5.0.0 through to Unreal Engine 5.0.3 when running under Ubuntu 22.04 or newer. In order to build Unreal Engine 5.0 in a Linux container, you will need to use an Ubuntu 20.04 base image by specifying the flags `-basetag ubuntu20.04`. + +== Windows-specific issues + +=== Building the `ue4-build-prerequisites` image fails with an unknown error + +Microsoft issued a security update in February 2020 that https://support.microsoft.com/en-us/help/4542617/you-might-encounter-issues-when-using-windows-server-containers-with-t[broke container compatibility for all versions of Windows Server and caused 32-bit applications to fail silently when run]. +The issue is resolved by ensuring that both the host system and the container image are using versions of Windows that incorporate the fix: + +- Make sure your host system is up-to-date and all available Windows updates are installed. +- Make sure you are using the latest version of ue4-docker, which automatically uses container images that incorporate the fix. + +[[hcsshim-timeout]] +=== Building Windows containers fails with the message `hcsshim: timeout waiting for notification extra info` or the message `This operation ended because the timeout has expired` + +Recent versions of Docker under Windows may sometimes encounter the error https://github.com/Microsoft/hcsshim/issues/152[hcsshim: timeout waiting for notification extra info] when building or running Windows containers. +This is a known issue when using Windows containers in https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/hyperv-container[Hyper-V isolation mode]. +At the time of writing, Microsoft have stated that they are aware of the problem, but an official fix is yet to be released. + +As a workaround until a proper fix is issued, it seems that altering the memory limit for containers between subsequent invocations of the `docker` command can reduce the frequency with which this error occurs. +(Changing the memory limit when using Hyper-V isolation likely forces Docker to provision a new Hyper-V VM, preventing it from re-using an existing one that has become unresponsive.) Please note that this workaround has been devised based on my own testing under Windows 10 and may not hold true when using Hyper-V isolation under Windows Server. + +To enable the workaround, specify the `--random-memory` flag when invoking the build command. +This will set the container memory limit to a random value between 10GB and 12GB when the build command starts. +If a build fails with the `hcsshim` timeout error, simply re-run the build command and in most cases the build will continue successfully, even if only for a short while. +Restarting the Docker daemon may also help. + +Note that some older versions of UnrealBuildTool will crash with an error stating *"The process cannot access the file because it is being used by another process"* when using a memory limit that is not a multiple of 4GB. +If this happens, simply run the build command again with an appropriate memory limit (e.g. `-m 8GB` or `-m 12GB`.) If the access error occurs even when using an appropriate memory limit, this likely indicates that Windows is unable to allocate the full amount of memory to the container. +Rebooting the host system may help to alleviate this issue. + +=== Building or running Windows containers fails with the message `The operating system of the container does not match the operating system of the host` + +This error is shown in two situations: + +- The host system is running an **older kernel version** than the container image. +In this case, you will need to build the images using the same kernel version as the host system or older. +See xref:advanced-build-options.adoc#windows-base-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. +- The host system is running a **newer kernel version** than the container image, and you are attempting to use process isolation mode instead of Hyper-V isolation mode. +(Process isolation mode is the default under Windows Server.) In this case, you will need to use Hyper-V isolation mode instead. +See xref:advanced-build-options.adoc#windows-isolation-mode[Specifying the isolation mode under Windows] for details on how to do this. + +=== Pulling the .NET Framework base image fails with the message `ProcessUtilityVMImage \\?\`(long path here)`\UtilityVM: The system cannot find the path specified` + +This is a known issue when the host system is running an older kernel version than the container image. +Just like in the case of *"The operating system of the container does not match the operating system of the host"* error mentioned above, you will need to build the images using the same kernel version as the host system or older. +See xref:advanced-build-options.adoc#windows-base-tag[Specifying the Windows Server Core base image tag] for details on specifying the correct kernel version when building Windows container images. + +=== Building the Engine in a Windows container fails with the message `The process cannot access the file because it is being used by another process` + +This is a known bug in some older versions of UnrealBuildTool when using a memory limit that is not a multiple of 4GB. +To alleviate this issue, specify an appropriate memory limit override (e.g. `-m 8GB` or `-m 12GB`.) For more details on this issue, see the last paragraph of the <> section. + +=== Building the Engine in a Windows container fails with the message `fatal error LNK1318: Unexpected PDB error; OK (0)` + +This is a known bug in some versions of Visual Studio which only appears to occur intermittently. +The simplest fix is to simply reboot the host system and then re-run the build command. +Insufficient available memory may also contribute to triggering this bug. +Note that a linker wrapper https://docs.unrealengine.com/en-US/Support/Builds/ReleaseNotes/4_24/index.html[was added in Unreal Engine 4.24.0] to automatically retry link operations in the event that this bug occurs, so it shouldn't be an issue when building version 4.24.0 or newer. + +[[pagefile]] +=== Building the Engine in a Windows container fails with the message `fatal error C1060: the compiler is out of heap space` + +This error typically occurs when the Windows pagefile size is not large enough. +As stated in the xref:troubleshooting-build-issues.adoc#pagefile[Troubleshooting build issues], there is currently no exposed mechanism to control the pagefile size for containers running in Hyper-V isolation mode. +However, containers running in process isolation mode will use the pagefile settings of the host system. +When using process isolation mode, this error can be resolved by increasing the pagefile size on the host system. +(Note that the host system will usually need to be rebooted for the updated pagefile settings to take effect.) + +[[windows-bind-mount]] +=== Building an Unreal project in a Windows container fails when the project files are located in a directory that is bind-mounted from the host operating system + +The paths associated with Windows bind-mounted directories inside Hyper-V isolation mode VMs can cause issues for certain build tools, including UnrealBuildTool and CMake. +As a result, building Unreal projects located in Windows bind-mounted directories is not advised when using Hyper-V isolation mode. +The solution is to copy the Unreal project to a temporary directory within the container's filesystem and build it there, copying any produced build artifacts back to the host system via the bind-mounted directory as necessary. diff --git a/docs/ue4-docker-build.adoc b/docs/ue4-docker-build.adoc new file mode 100644 index 00000000..20ce169a --- /dev/null +++ b/docs/ue4-docker-build.adoc @@ -0,0 +1,167 @@ +[[ue4-docker-build]] += ue4-docker-build (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-build - build container image for a specific version of Unreal Engine + +== Synopsis + +*ue4-docker build* [_option_]... _version_ + +== Description + +To build container images for a specific version of the Unreal Engine, simply specify the _version_ that you would like to build using full https://semver.org/[semver] syntax. +For example, to build Unreal Engine 4.27.0, run: + +[source,shell] +---- +ue4-docker build 4.27.0 +---- + +You will be prompted for the Git credentials to be used when cloning the Unreal Engine GitHub repository (this will be the GitHub username and password you normally use when cloning . +The build process will then start automatically, displaying progress output from each of the `docker build` commands that are being run in turn. + +By default, all available images will be built. +See the xref:available-container-images.adoc[List of available container images] for details on customising which images are built. + +== Options + +*-basetag* _basetag_:: +Operating system base image tag to use. +For Linux this is the version of Ubuntu (default is ubuntu22.04). +For Windows this is the Windows Server Core base image tag (default is the host OS version) + +*-branch* _branch_:: +Set the custom branch/tag to clone when *custom* is specified as the _version_. + +*--combine*:: +Combine generated Dockerfiles into a single multi-stage build Dockerfile + +*-conan-ue4cli* _conan_ue4cli_:: +Override the default version of conan-ue4cli installed in the ue4-full image + +*--docker-build-args* _args_:: +Allows passing custom args to `docker build` command. +For example, this can be useful for pushing images. + +*--dry-run*:: +Use this if you would like to see what Docker commands would be run by `ue4-docker build` without actually building anything. +Execution will proceed as normal, but no Git credentials will be requested and all Docker commands will be printed to standard output instead of being executed as child processes. + +*--exclude {ddc,debug,templates}*:: +Exclude the specified component from the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] and xref:available-container-images.adoc#ue4-full[ue4-full] images. ++ +The following components can be excluded: ++ +- `ddc`: disables building the DDC for the Engine. +This significantly speeds up building the Engine itself but results in far longer cook times when subsequently packaging Unreal projects. +- `debug`: removes all debug symbols from the built images. +- `templates`: removes the template projects and samples that ship with the Engine. ++ +You can specify the `--exclude` flag multiple times to exclude as many components as you like. +For example: ++ +[source,shell] +---- +# Excludes both debug symbols and template projects +ue4-docker build 4.21.2 --exclude debug --exclude templates +---- + +*-h, --help*:: +Print help and exit + +*-interval* _inverval_:: +Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds) + +*-layout* _layout_:: +Copy generated Dockerfiles to the specified directory and don't build the images + +*-m* _memory_:: +Override the default memory limit under Windows (also overrides --random-memory) + +*--monitor*:: +Monitor resource usage during builds (useful for debugging) + +*--no-cache*:: +Disable Docker build cache + +*--no-full*:: +Don't build the ue4-full image (deprecated, use *--target* _target_ instead) + +*--no-minimal*:: +Don't build the ue4-minimal image (deprecated, use *--target* _target_ instead) + +*--opt* _opt_:: +Set an advanced configuration option (can be specified multiple times to specify multiple options) + +*-password* _password_:: +Specify access token or password to use when cloning the git repository + +*--rebuild*:: +Rebuild images even if they already exist + +*-repo* _repo_:: +Set the URL of custom git repository to clone when *custom* is specified as the _version_ + +*-suffix* _suffix_:: +Add a suffix to the tags of the built images + +*--target* _target_:: +Tells ue4-docker to build specific image (including its dependencies). ++ +Supported values: `all`, `build-prerequisites`, `full`, `minimal`, `source`. ++ +You can specify the `--target` option multiple times. + +*-ue4cli* _ue4cli_:: +Override the default version of ue4cli installed in the ue4-full image + +*-username* _username_:: +Specify the username to use when cloning the git repository + +*-v*, *--verbose*:: +Enable verbose output during builds (useful for debugging) + +== Linux-specific options + +*--cuda* _version_:: +Add CUDA support as well as OpenGL support + +== Windows-specific options + +*--ignore-blacklist*:: +Run builds even on blacklisted versions of Windows (advanced use only) + +*-isolation {process,hyperv}*:: +Set the isolation mode to use + +*--linux*:: +Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support) + +*--random-memory*:: +Use a random memory limit for Windows containers + +*--visual-studio {2017,2019,2022}*:: +Specify Visual Studio Build Tools version. ++ +By default, ue4-docker uses Visual Studio Build Tools 2017 to build Unreal Engine. +Starting with Unreal Engine 4.25, you may choose to use Visual Studio Build Tools 2019 instead. ++ +Unreal Engine 5.0 adds support for VS2022 but removes support for VS2017. + +== Environment + +This section describes several environment variables that affect how `ue4-docker build` operates. + +*UE4DOCKER_TAG_NAMESPACE*:: +If you would like to override the default `adamrehn/` prefix that is used when generating the tags for all built images, you can do so by specifying a custom value using the `UE4DOCKER_TAG_NAMESPACE` environment variable. + +== See also + +xref:ue4-docker-clean.adoc#ue4-docker-clean[*ue4-docker-clean*(1)] diff --git a/docs/ue4-docker-clean.adoc b/docs/ue4-docker-clean.adoc new file mode 100644 index 00000000..8ab13319 --- /dev/null +++ b/docs/ue4-docker-clean.adoc @@ -0,0 +1,36 @@ +[[ue4-docker-clean]] += ue4-docker-clean (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-clean - cleans built container images. + +== Synopsis + +*ue4-docker clean* [*-tag* _tag_] [*--source*] [*--all*] [*--dry-run*] + +== Description + +By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed. + +== Options + +*--all*:: +Remove all ue4-docker images, applying the tag filter if one was specified + +*--dry-run*:: +If you're unsure as to exactly what images will be removed by a given invocation of the command, append the `--dry-run` flag to have ue4-docker print the generated `docker rmi` commands instead of running them. + +*--prune*:: +Run `docker system prune` after cleaning + +*--source*:: +Remove ../building-images/available-container-images.adoc#ue4-source[ue4-source] images, applying the tag filter if one was specified + +*-tag* _tag_:: +Apply a filter for the three flags below, restricting them to removing only images with the specified _tag_ (e.g. `-tag 4.21.0` will only remove images for 4.21.0) diff --git a/docs/ue4-docker-diagnostics.adoc b/docs/ue4-docker-diagnostics.adoc new file mode 100644 index 00000000..23e87b10 --- /dev/null +++ b/docs/ue4-docker-diagnostics.adoc @@ -0,0 +1,42 @@ +[[ue4-docker-diagnostics]] += ue4-docker-diagnostics (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-diagnostics - run diagnostics to detect issues with the host system configuration. + +== Synopsis + +*ue4-docker diagnostics* _diagnostic_ + +== Description + +This command can be used to run the following diagnostics: + +=== Checking for the Docker 8GiB filesystem layer bug + +Some versions of Docker contain one or more of a series of separate but related bugs that prevent the creation of filesystem layers which are 8GiB in size or larger. +This also causes `COPY` directives to fail when copying data in excess of 8GiB in size, xref:troubleshooting-build-issues.adoc#copy-8gb-20gb[breaking Dockerfile steps during the creation of Installed Builds that contain debug symbols]. + +This diagnostic tests whether the host system's Docker daemon suffers from this issue, by attempting to build a simple test Dockerfile with an 8GiB filesystem layer: + +[source,shell] +---- +ue4-docker diagnostics 8gig +---- + +// TODO: Add docs on ue4-docker diagnostics 20gig + +=== Checking for container network connectivity issues + +This diagnostic tests whether running containers are able to access the internet, resolve DNS entries, and download remote files: + +[source,shell] +---- +ue4-docker diagnostics network +---- diff --git a/docs/ue4-docker-export.adoc b/docs/ue4-docker-export.adoc new file mode 100644 index 00000000..b78186f4 --- /dev/null +++ b/docs/ue4-docker-export.adoc @@ -0,0 +1,63 @@ +[[ue4-docker-export]] += ue4-docker-export (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-export - export components from built container image to the host system + +== Synopsis + +*ue4-docker export* _component_ _tag_ _destination_ + +This command can be used to export the following components: + +== Description + +=== Exporting Installed Builds under Linux + +Installed Builds of Unreal Engine can be exported to the host system starting with version 4.21.0. Once you have built either the xref:available-container-images.adoc#ue4-minimal[ue4-minimal] or xref:available-container-images.adoc#ue4-full[ue4-full] image for the UE4 version that you want to export, you can export it to the host system like so: + +[source,shell] +---- +# Example: specify a version without a full image tag (assumes `adamrehn/ue4-full`) +# Exports the Installed Build from `adamrehn/ue4-full:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "4.27.0" ~/UnrealInstalled +---- + +[source,shell] +---- +# Example: specify a full image tag +# Exports the Installed Build from `adamrehn/ue4-minimal:4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "adamrehn/ue4-minimal:4.27.0" ~/UnrealInstalled +---- + +[source,shell] +---- +# Example: use a container image based on ue4-minimal with a completely different tag +# Exports the Installed Build from `ghcr.io/epicgames/unreal-engine:dev-4.27.0` to the directory `~/UnrealInstalled` on the host system +ue4-docker export installed "ghcr.io/epicgames/unreal-engine:dev-4.27.0" ~/UnrealInstalled +---- + +=== Exporting Conan packages + +The Conan wrapper packages generated by `conan-ue4cli` can be exported from the xref:available-container-images.adoc#ue4-full[ue4-full] image to the local Conan package cache on the host system like so: + +[source,shell] +---- +ue4-docker export packages 4.27.0 cache +---- + +https://conan.io/[Conan] will need to be installed on the host system for this to work. +To use the exported packages for development on the host system, you will also need to generate the accompanying profile-wide packages by running the command: + +[source,shell] +---- +ue4 conan generate --profile-only +---- + +This will require both `ue4cli` and `conan-ue4cli` to be installed on the host system. diff --git a/docs/ue4-docker-info.adoc b/docs/ue4-docker-info.adoc new file mode 100644 index 00000000..9d5e6a76 --- /dev/null +++ b/docs/ue4-docker-info.adoc @@ -0,0 +1,27 @@ +[[ue4-docker-info]] += ue4-docker-info (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-info - display information about the host system and Docker daemon + +== Synopsis + +*ue4-docker info* + +== Description + +The command will output the following information: + +- The ue4-docker version +- The host OS version +- The Docker daemon version +- Whether the NVIDIA Container Toolkit is supported under the current host configuration +- The detected configuration value for the maximum image size for Windows containers +- The total amount of detected system memory +- The number of detected physical and logical CPUs diff --git a/docs/ue4-docker-setup.adoc b/docs/ue4-docker-setup.adoc new file mode 100644 index 00000000..e76985e6 --- /dev/null +++ b/docs/ue4-docker-setup.adoc @@ -0,0 +1,32 @@ +[[ue4-docker-setup]] += ue4-docker-setup (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-setup - automatically configure the host system where possible + +== Synopsis + +*ue4-docker setup* + +== Description + +This command will automatically configure a Linux or Windows Server host system with the settings required in order to build and run containers produced by ue4-docker. + +**Under Linux:** + +- If an active firewall is detected then a firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image. + +**Under Windows Server:** + +- The Docker daemon will be configured to set the maximum image size for Windows containers to 800GB. +- A Windows Firewall rule will be created to allow Docker containers to communicate with the host system, which is required during the build of the xref:available-container-images.adoc#ue4-source[ue4-source] image. +- Under Windows Server Core version 1809 and newer, any required DLL files will be copied to the host system from the https://hub.docker.com/_/microsoft-windows[full Windows base image]. +Note that the full base image was only introduced in Windows Server version 1809, so this step will not be performed under older versions of Windows Server. + +**Under Windows 10 and macOS** this command will print a message informing the user that automatic configuration is not supported under their platform and that they will need to configure the system manually. diff --git a/docs/ue4-docker-test.adoc b/docs/ue4-docker-test.adoc new file mode 100644 index 00000000..d4fe2bba --- /dev/null +++ b/docs/ue4-docker-test.adoc @@ -0,0 +1,21 @@ +[[ue4-docker-test]] += ue4-docker-test (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-test - run tests to verify the correctness of built container images + +== Synopsis + +*ue4-docker test* _tag_ + +== Description + +This command runs a suite of tests to verify that built xref:available-container-images.adoc#ue4-full[ue4-full] container images are functioning correctly and can be used to build and package Unreal projects and plugins. + +This command is primarily intended for use by developers who are contributing to the ue4-docker project itself. diff --git a/docs/ue4-docker-version.adoc b/docs/ue4-docker-version.adoc new file mode 100644 index 00000000..4831c225 --- /dev/null +++ b/docs/ue4-docker-version.adoc @@ -0,0 +1,19 @@ +[[ue4-docker-version]] += ue4-docker-version (1) +:doctype: manpage +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge + +== Name + +ue4-docker-version - display version information about ue4-docker + +== Synopsis + +*ue4-docker version* + +== Description + +Prints ue4-docker version on the standard output. diff --git a/docs/use-cases-overview.adoc b/docs/use-cases-overview.adoc new file mode 100644 index 00000000..b9dda084 --- /dev/null +++ b/docs/use-cases-overview.adoc @@ -0,0 +1,14 @@ += Use cases overview +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +The container images produced by ue4-docker incorporate infrastructure to facilitate a wide variety of use cases. +A number of key use cases are listed below. +Select a use case to see detailed instructions on how to build and run the appropriate container images. + +* xref:continuous-integration.adoc[Continuous Integration] +* xref:linux-installed-builds.adoc[Linux installed builds] +* xref:microservices.adoc[Microservices] diff --git a/docs/windows-container-primer.adoc b/docs/windows-container-primer.adoc new file mode 100644 index 00000000..f1320f77 --- /dev/null +++ b/docs/windows-container-primer.adoc @@ -0,0 +1,20 @@ += Windows containers primer +:icons: font +:idprefix: +:idseparator: - +:source-highlighter: rouge +:toc: + +NOTE: The implementation-agnostic information from this page has migrated to the https://unrealcontainers.com/[Unreal Containers community hub]. +You can find the new version here: https://unrealcontainers.com/docs/concepts/windows-containers[Key Concepts: Windows Containers]. + +*Details specific to ue4-docker:* + +Due to the performance and stability issues currently associated with containers running in Hyper-V isolation mode, it is strongly recommended that process isolation mode be used for building and running Windows containers. +This necessitates the use of Windows Server as the host system (https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/faq#can-i-run-windows-containers-in-process-isolated-mode-on-windows-10-enterprise-or-professional[or Windows 10 version 1809 or newer for development and testing purposes]) and requires that all container images use the same Windows version as the host system. +A number of ue4-docker commands provide specific functionality to facilitate this: + +* The xref:ue4-docker-build.adoc[ue4-docker build] command will automatically attempt to build images based on the same kernel version as the host system, and will default to process isolation mode if the operating system version and Docker daemon version allow it. +Hyper-V isolation mode will still be used if the user explicitly xref:advanced-build-options.adoc#windows-base-tag[specifies a different kernel version] than that of the host system or xref:advanced-build-options.adoc#windows-isolation-mode[explicitly requests Hyper-V isolation mode]. + +* The xref:ue4-docker-setup.adoc[ue4-docker setup] command automates the configuration of Windows Server hosts, in order to provide a smoother experience for users who migrate their container hosts to the latest versions of Windows Server as they are released. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..9552f485 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,57 @@ +[project] +name = "ue4-docker" +version = "0.0.116" +description = "Windows and Linux containers for Unreal Engine" +requires-python = ">= 3.8" +license = { file = "LICENSE" } +readme = "README.md" +authors = [ + { name = "Adam Rehn", email = "adam@adamrehn.com" }, + { name = "Marat Radchenko", email = "marat@slonopotamus.org" }, +] +keywords = [ + "unreal engine", + "docker", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Build Tools", +] +dependencies = [ + "colorama", + "docker>=6.1.0", + "humanfriendly", + "Jinja2>=2.11.3", + "packaging>=19.1", + "psutil", + "termcolor", +] + +# See https://packaging.python.org/en/latest/specifications/declaring-project-metadata/#urls +# See https://peps.python.org/pep-0753/#well-known-labels +[project.urls] +homepage = "https://github.com/adamrehn/ue4-docker" +documentation = "https://adamrehn.github.io/ue4-docker" +repository = "https://github.com/adamrehn/ue4-docker.git" + +[project.scripts] +ue4-docker = "ue4docker:main" + +[build-system] +requires = [ + "setuptools>=61", + "wheel", +] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-data] +ue4docker = [ + "dockerfiles/*/*/.dockerignore", + "dockerfiles/diagnostics/*/*/*", + "dockerfiles/*/*/*", + "tests/*.py", +] diff --git a/setup.py b/setup.py deleted file mode 100644 index 483b4a74..00000000 --- a/setup.py +++ /dev/null @@ -1,55 +0,0 @@ -from os.path import abspath, dirname, join -from setuptools import setup - -# Read the README markdown data from README.md -with open(abspath(join(dirname(__file__), 'README.md')), 'rb') as readmeFile: - __readme__ = readmeFile.read().decode('utf-8') - -# Read the version number from version.py -with open(abspath(join(dirname(__file__), 'ue4docker', 'version.py'))) as versionFile: - __version__ = versionFile.read().strip().replace('__version__ = ', '').replace("'", '') - -setup( - name='ue4-docker', - version=__version__, - description='Windows and Linux containers for Unreal Engine 4', - long_description=__readme__, - long_description_content_type='text/markdown', - classifiers=[ - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Software Development :: Build Tools', - 'Environment :: Console' - ], - keywords='epic unreal engine docker', - url='http://github.com/adamrehn/ue4-docker', - author='Adam Rehn', - author_email='adam@adamrehn.com', - license='MIT', - packages=['ue4docker', 'ue4docker.exports', 'ue4docker.infrastructure'], - zip_safe=False, - python_requires = '>=3.6', - install_requires = [ - 'colorama', - 'docker>=3.0.0', - 'flask', - 'humanfriendly', - 'psutil', - 'requests', - 'semver>=2.7.9', - 'setuptools>=38.6.0', - 'termcolor', - 'twine>=1.11.0', - 'wheel>=0.31.0' - ], - package_data = { - 'ue4docker': [ - 'dockerfiles/*/*/.dockerignore', - 'dockerfiles/*/*/*' - ] - }, - entry_points = { - 'console_scripts': ['ue4-docker=ue4docker:main'] - } -) diff --git a/ue4docker/__init__.py b/src/ue4docker/__init__.py similarity index 90% rename from ue4docker/__init__.py rename to src/ue4docker/__init__.py index cbf7415d..6824568e 100644 --- a/ue4docker/__init__.py +++ b/src/ue4docker/__init__.py @@ -4,5 +4,6 @@ from .info import info from .main import main from .setup_cmd import setup +from .test import test from .version_cmd import version from .version import __version__ diff --git a/src/ue4docker/__main__.py b/src/ue4docker/__main__.py new file mode 100644 index 00000000..91bbc937 --- /dev/null +++ b/src/ue4docker/__main__.py @@ -0,0 +1,8 @@ +from .main import main +import os, sys + +if __name__ == "__main__": + # Rewrite sys.argv[0] so our help prompts display the correct base command + interpreter = sys.executable if sys.executable not in [None, ""] else "python3" + sys.argv[0] = "{} -m ue4docker".format(os.path.basename(interpreter)) + main() diff --git a/src/ue4docker/build.py b/src/ue4docker/build.py new file mode 100644 index 00000000..dc14af4a --- /dev/null +++ b/src/ue4docker/build.py @@ -0,0 +1,616 @@ +import argparse, getpass, humanfriendly, json, os, platform, shutil, sys, tempfile, time +from .infrastructure import * +from .version import __version__ +from os.path import join + + +def _getCredential(args, name, envVar, promptFunc): + # Check if the credential was specified via the command-line + if getattr(args, name, None) is not None: + print("Using {} specified via `-{}` command-line argument.".format(name, name)) + return getattr(args, name) + + # Check if the credential was specified via an environment variable + if envVar in os.environ: + print("Using {} specified via {} environment variable.".format(name, envVar)) + return os.environ[envVar] + + # Fall back to prompting the user for the value + return promptFunc() + + +def _getUsername(args): + return _getCredential( + args, "username", "UE4DOCKER_USERNAME", lambda: input("Username: ") + ) + + +def _getPassword(args): + return _getCredential( + args, + "password", + "UE4DOCKER_PASSWORD", + lambda: getpass.getpass("Access token or password: "), + ) + + +def build(): + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} build] ".format(sys.argv[0])) + + # Register our supported command-line arguments + parser = argparse.ArgumentParser(prog="{} build".format(sys.argv[0])) + BuildConfiguration.addArguments(parser) + + # If no command-line arguments were supplied, display the help message and exit + if len(sys.argv) < 2: + parser.print_help() + sys.exit(0) + + # Parse the supplied command-line arguments + try: + config = BuildConfiguration(parser, sys.argv[1:], logger) + except RuntimeError as e: + logger.error("Error: {}".format(e)) + sys.exit(1) + + # Verify that Docker is installed + if DockerUtils.installed() == False: + logger.error( + "Error: could not detect Docker version. Please ensure Docker is installed." + ) + sys.exit(1) + + # Verify that we aren't trying to build Windows containers under Windows 10 when in Linux container mode (or vice versa) + # (Note that we don't bother performing this check when we're just copying Dockerfiles to an output directory) + if config.layoutDir is None: + dockerPlatform = DockerUtils.info()["OSType"].lower() + if config.containerPlatform == "windows" and dockerPlatform == "linux": + logger.error( + "Error: cannot build Windows containers when Docker Desktop is in Linux container", + False, + ) + logger.error( + "mode. Use the --linux flag if you want to build Linux containers instead.", + False, + ) + sys.exit(1) + elif config.containerPlatform == "linux" and dockerPlatform == "windows": + logger.error( + "Error: cannot build Linux containers when Docker Desktop is in Windows container", + False, + ) + logger.error( + "mode. Remove the --linux flag if you want to build Windows containers instead.", + False, + ) + sys.exit(1) + + # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 Linux images without config changes + if ( + config.containerPlatform == "linux" + and DockerUtils.isVersionWithoutIPV6Loopback() + ): + logger.warning( + DockerUtils.getIPV6WarningMessage() + "\n", + False, + ) + + # Create an auto-deleting temporary directory to hold our build context + with tempfile.TemporaryDirectory() as tempDir: + contextOrig = join(os.path.dirname(os.path.abspath(__file__)), "dockerfiles") + + # Create the builder instance to build the Docker images + builder = ImageBuilder( + join(tempDir, "dockerfiles"), + config.containerPlatform, + logger, + config.rebuild, + config.dryRun, + config.layoutDir, + config.opts, + config.combine, + ) + + # Resolve our main set of tags for the generated images; this is used only for Source and downstream + if config.buildTargets["source"]: + mainTags = [ + "{}{}-{}".format(config.release, config.suffix, config.prereqsTag), + config.release + config.suffix, + ] + + # Print the command-line invocation that triggered this build, masking any supplied passwords + args = [ + ( + "*******" + if config.args.password is not None and arg == config.args.password + else arg + ) + for arg in sys.argv + ] + logger.info("COMMAND-LINE INVOCATION:", False) + logger.info(str(args), False) + + # Print the details of the Unreal Engine version being built + logger.info("UNREAL ENGINE VERSION SETTINGS:") + logger.info( + "Custom build: {}".format("Yes" if config.custom == True else "No"), False + ) + if config.custom == True: + logger.info("Custom name: " + config.release, False) + elif config.release is not None: + logger.info("Release: " + config.release, False) + if config.repository is not None: + logger.info("Repository: " + config.repository, False) + logger.info("Branch/tag: " + config.branch + "\n", False) + + # Determine if we are using a custom version for ue4cli or conan-ue4cli + if config.ue4cliVersion is not None or config.conanUe4cliVersion is not None: + logger.info("CUSTOM PACKAGE VERSIONS:", False) + logger.info( + "ue4cli: {}".format( + config.ue4cliVersion + if config.ue4cliVersion is not None + else "default" + ), + False, + ) + logger.info( + "conan-ue4cli: {}\n".format( + config.conanUe4cliVersion + if config.conanUe4cliVersion is not None + else "default" + ), + False, + ) + + # Report any advanced configuration options that were specified + if len(config.opts) > 0: + logger.info("ADVANCED CONFIGURATION OPTIONS:", False) + for key, value in sorted(config.opts.items()): + logger.info("{}: {}".format(key, json.dumps(value)), False) + print("", file=sys.stderr, flush=True) + + # Determine if we are building Windows or Linux containers + if config.containerPlatform == "windows": + # Provide the user with feedback so they are aware of the Windows-specific values being used + logger.info("WINDOWS CONTAINER SETTINGS", False) + logger.info( + "Isolation mode: {}".format(config.isolation), False + ) + logger.info( + "Base OS image: {}".format(config.baseImage), False + ) + logger.info( + "Dll source image: {}".format(config.dllSrcImage), False + ) + logger.info( + "Host OS: {}".format(WindowsUtils.systemString()), + False, + ) + logger.info( + "Memory limit: {}".format( + "No limit" + if config.memLimit is None + else "{:.2f}GB".format(config.memLimit) + ), + False, + ) + logger.info( + "Detected max image size: {:.0f}GB".format(DockerUtils.maxsize()), + False, + ) + logger.info( + "Visual Studio: {}".format(config.visualStudio), False + ) + + # Verify that the host OS is not a release that is blacklisted due to critical bugs + if ( + config.ignoreBlacklist == False + and WindowsUtils.isBlacklistedWindowsHost() == True + ): + logger.error( + "Error: detected blacklisted host OS version: {}".format( + WindowsUtils.systemString() + ), + False, + ) + logger.error("", False) + logger.error( + "This version of Windows contains one or more critical bugs that", + False, + ) + logger.error( + "render it incapable of successfully building UE4 container images.", + False, + ) + logger.error( + "You will need to use an older or newer version of Windows.", False + ) + logger.error("", False) + logger.error("For more information, see:", False) + logger.error( + "https://unrealcontainers.com/docs/concepts/windows-containers", + False, + ) + sys.exit(1) + + # Verify that the user is not attempting to build images with a newer kernel version than the host OS + newer_check = WindowsUtils.isNewerBaseTag( + config.hostBasetag, config.basetag + ) + if newer_check: + logger.error( + "Error: cannot build container images with a newer kernel version than that of the host OS!" + ) + sys.exit(1) + elif newer_check is None: + logger.warning( + "Warning: unable to determine whether host system is new enough to use specified base tag" + ) + + # Ensure the Docker daemon is configured correctly + requiredLimit = WindowsUtils.requiredSizeLimit() + if DockerUtils.maxsize() < requiredLimit and config.buildTargets["source"]: + logger.error("SETUP REQUIRED:") + logger.error( + "The max image size for Windows containers must be set to at least {}GB.".format( + requiredLimit + ) + ) + logger.error( + "See the Microsoft documentation for configuration instructions:" + ) + logger.error( + "https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-storage#storage-limits" + ) + logger.error( + "Under Windows Server, the command `{} setup` can be used to automatically configure the system.".format( + sys.argv[0] + ) + ) + sys.exit(1) + + elif config.containerPlatform == "linux": + logger.info("LINUX CONTAINER SETTINGS", False) + logger.info( + "Base OS image: {}\n".format(config.baseImage), + False, + ) + + # Report which Engine components are being excluded (if any) + logger.info("GENERAL SETTINGS", False) + logger.info( + "Build targets: {}".format( + " ".join( + sorted( + [ + target + for target, enabled in config.buildTargets.items() + if enabled + ] + ) + ) + ), + False, + ) + logger.info( + "Changelist override: {}".format( + config.changelist + if config.changelist is not None + else "(None specified)" + ), + False, + ) + if len(config.excludedComponents) > 0: + logger.info("Excluding the following Engine components:", False) + for component in config.describeExcludedComponents(): + logger.info("- {}".format(component), False) + else: + logger.info("Not excluding any Engine components.", False) + + # Print a warning if the user is attempting to build Linux images under Windows + if config.containerPlatform == "linux" and ( + platform.system() == "Windows" or WindowsUtils.isWSL() + ): + logger.warning( + "Warning: attempting to build Linux container images under Windows (e.g. via WSL)." + ) + logger.warning( + "The ue4-docker maintainers do not provide support for building and running Linux", + False, + ) + logger.warning( + "containers under Windows, and this configuration is not tested to verify that it", + False, + ) + logger.warning( + "functions correctly. Users are solely responsible for troubleshooting any issues", + False, + ) + logger.warning( + "they encounter when attempting to build Linux container images under Windows.", + False, + ) + + # Determine if we need to prompt for credentials + if config.dryRun == True: + # Don't bother prompting the user for any credentials during a dry run + logger.info( + "Performing a dry run, `docker build` commands will be printed and not executed." + ) + username = "" + password = "" + + elif config.layoutDir is not None: + # Don't bother prompting the user for any credentials when we're just copying the Dockerfiles to a directory + logger.info( + "Copying generated Dockerfiles to: {}".format(config.layoutDir), False + ) + username = "" + password = "" + + elif ( + not config.buildTargets["source"] + or builder.willBuild("ue4-source", mainTags) == False + ): + # Don't bother prompting the user for any credentials if we're not building the ue4-source image + logger.info( + "Not building the ue4-source image, no Git credentials required.", False + ) + username = "" + password = "" + + else: + # Retrieve the Git username and password from the user when building the ue4-source image + print( + "\nRetrieving the Git credentials that will be used to clone the UE4 repo" + ) + username = _getUsername(config.args) + password = _getPassword(config.args) + print() + + # If resource monitoring has been enabled, start the resource monitoring background thread + resourceMonitor = ResourceMonitor(logger, config.args.interval) + if config.args.monitor == True: + resourceMonitor.start() + + # Prep for endpoint cleanup, if necessary + endpoint = None + + try: + # Keep track of our starting time + startTime = time.time() + + # If we're copying Dockerfiles to an output directory then make sure it exists and is empty + if config.layoutDir is not None: + if os.path.exists(config.layoutDir): + shutil.rmtree(config.layoutDir) + os.makedirs(config.layoutDir) + + # Keep track of the images we've built + builtImages = [] + + commonArgs = [ + "--build-arg", + "NAMESPACE={}".format(GlobalConfiguration.getTagNamespace()), + ] + config.args.docker_build_args + + # Build the UE4 build prerequisites image + if config.buildTargets["build-prerequisites"]: + # Compute the build options for the UE4 build prerequisites image + # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) + prereqsArgs = ["--build-arg", "BASEIMAGE=" + config.baseImage] + if config.containerPlatform == "windows": + prereqsArgs = prereqsArgs + [ + "--build-arg", + "DLLSRCIMAGE=" + config.dllSrcImage, + "--build-arg", + "VISUAL_STUDIO_BUILD_NUMBER=" + + config.visualStudio.build_number, + ] + + custom_prerequisites_dockerfile = config.args.prerequisites_dockerfile + if custom_prerequisites_dockerfile is not None: + builder.build_builtin_image( + "ue4-base-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + builtin_name="ue4-build-prerequisites", + ) + builtImages.append("ue4-base-build-prerequisites") + else: + builder.build_builtin_image( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqsArgs, + ) + + prereqConsumerArgs = [ + "--build-arg", + "PREREQS_TAG={}".format(config.prereqsTag), + ] + + if custom_prerequisites_dockerfile is not None: + builder.build( + "ue4-build-prerequisites", + [config.prereqsTag], + commonArgs + config.platformArgs + prereqConsumerArgs, + dockerfile_template=custom_prerequisites_dockerfile, + context_dir=os.path.dirname(custom_prerequisites_dockerfile), + ) + + builtImages.append("ue4-build-prerequisites") + else: + logger.info("Skipping ue4-build-prerequisities image build.") + + # Build the UE4 source image + if config.buildTargets["source"]: + # Start the HTTP credential endpoint as a child process and wait for it to start + if config.opts["credential_mode"] == "endpoint": + endpoint = CredentialEndpoint(username, password) + endpoint.start() + + # If we're using build secrets then pass the Git username and password to the UE4 source image as secrets + secrets = {} + if config.opts["credential_mode"] == "secrets": + secrets = {"username": username, "password": password} + credentialArgs = [] if len(secrets) > 0 else endpoint.args() + + ue4SourceArgs = prereqConsumerArgs + [ + "--build-arg", + "GIT_REPO={}".format(config.repository), + "--build-arg", + "GIT_BRANCH={}".format(config.branch), + "--build-arg", + "VERBOSE_OUTPUT={}".format("1" if config.verbose == True else "0"), + ] + + changelistArgs = ( + ["--build-arg", "CHANGELIST={}".format(config.changelist)] + if config.changelist is not None + else [] + ) + + builder.build_builtin_image( + "ue4-source", + mainTags, + commonArgs + + config.platformArgs + + ue4SourceArgs + + credentialArgs + + changelistArgs, + secrets=secrets, + ) + builtImages.append("ue4-source") + else: + logger.info("Skipping ue4-source image build.") + + # Build the minimal UE4 CI image, unless requested otherwise by the user + if config.buildTargets["minimal"]: + minimalArgs = prereqConsumerArgs + [ + "--build-arg", + "TAG={}".format(mainTags[1]), + ] + + builder.build_builtin_image( + "ue4-minimal", + mainTags, + commonArgs + config.platformArgs + minimalArgs, + ) + builtImages.append("ue4-minimal") + else: + logger.info("Skipping ue4-minimal image build.") + + # Build the full UE4 CI image, unless requested otherwise by the user + if config.buildTargets["full"]: + # If custom version strings were specified for ue4cli and/or conan-ue4cli, use them + infrastructureFlags = [] + if config.ue4cliVersion is not None: + infrastructureFlags.extend( + [ + "--build-arg", + "UE4CLI_VERSION={}".format(config.ue4cliVersion), + ] + ) + if config.conanUe4cliVersion is not None: + infrastructureFlags.extend( + [ + "--build-arg", + "CONAN_UE4CLI_VERSION={}".format(config.conanUe4cliVersion), + ] + ) + + # Build the image + builder.build_builtin_image( + "ue4-full", + mainTags, + commonArgs + + config.platformArgs + + ue4BuildArgs + + infrastructureFlags, + ) + builtImages.append("ue4-full") + else: + logger.info("Skipping ue4-full image build.") + + # If we are generating Dockerfiles then include information about the options used to generate them + if config.layoutDir is not None: + # Determine whether we generated a single combined Dockerfile or a set of Dockerfiles + if config.combine == True: + # Generate a comment to place at the top of the single combined Dockerfile + lines = [ + "This file was generated by ue4-docker version {} with the following options:".format( + __version__ + ), + "", + ] + lines.extend( + [ + "- {}: {}".format(key, json.dumps(value)) + for key, value in sorted(config.opts.items()) + ] + ) + lines.extend( + [ + "", + "This Dockerfile combines the steps for the following images:", + "", + ] + ) + lines.extend(["- {}".format(image) for image in builtImages]) + comment = "\n".join(["# {}".format(line) for line in lines]) + + # Inject the comment at the top of the Dockerfile, being sure to place it after any `escape` parser directive + dockerfile = join(config.layoutDir, "combined", "Dockerfile") + dockerfileContents = FilesystemUtils.readFile(dockerfile) + if dockerfileContents.startswith("# escape"): + newline = dockerfileContents.index("\n") + dockerfileContents = ( + dockerfileContents[0 : newline + 1] + + "\n" + + comment + + "\n\n" + + dockerfileContents[newline + 1 :] + ) + else: + dockerfileContents = comment + "\n\n" + dockerfileContents + FilesystemUtils.writeFile(dockerfile, dockerfileContents) + + else: + # Create a JSON file to accompany the set of generated Dockerfiles + FilesystemUtils.writeFile( + join(config.layoutDir, "generated.json"), + json.dumps( + { + "version": __version__, + "images": builtImages, + "opts": config.opts, + }, + indent=4, + sort_keys=True, + ), + ) + + # Report the total execution time + endTime = time.time() + logger.action( + "Total execution time: {}".format( + humanfriendly.format_timespan(endTime - startTime) + ) + ) + + # Stop the resource monitoring background thread if it is running + resourceMonitor.stop() + + # Stop the HTTP server + if endpoint is not None: + endpoint.stop() + + except (Exception, KeyboardInterrupt) as e: + # One of the images failed to build + logger.error("Error: {}".format(e)) + resourceMonitor.stop() + if endpoint is not None: + endpoint.stop() + sys.exit(1) diff --git a/src/ue4docker/clean.py b/src/ue4docker/clean.py new file mode 100644 index 00000000..016a3338 --- /dev/null +++ b/src/ue4docker/clean.py @@ -0,0 +1,78 @@ +import argparse, itertools, subprocess, sys +from .infrastructure import * + + +def _isIntermediateImage(image): + sentinel = "com.adamrehn.ue4-docker.sentinel" + labels = image.attrs["ContainerConfig"]["Labels"] + return labels is not None and sentinel in labels + + +def _cleanMatching(cleaner, filter, tag, dryRun): + tagSuffix = ":{}".format(tag) if tag is not None else "*" + matching = DockerUtils.listImages(tagFilter=filter + tagSuffix) + cleaner.cleanMultiple( + itertools.chain.from_iterable([image.tags for image in matching]), dryRun + ) + + +def clean(): + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} clean] ".format(sys.argv[0])) + + # Our supported command-line arguments + parser = argparse.ArgumentParser( + prog="{} clean".format(sys.argv[0]), + description="Cleans built container images. " + + "By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed.", + ) + parser.add_argument( + "-tag", default=None, help="Only clean images with the specified tag" + ) + parser.add_argument("--source", action="store_true", help="Clean ue4-source images") + parser.add_argument( + "--all", action="store_true", help="Clean all ue4-docker images" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print docker commands instead of running them", + ) + parser.add_argument( + "--prune", action="store_true", help="Run `docker system prune` after cleaning" + ) + + # Parse the supplied command-line arguments + args = parser.parse_args() + + # Create our image cleaner + cleaner = ImageCleaner(logger) + + # Remove any intermediate images leftover from our multi-stage builds + dangling = DockerUtils.listImages(filters={"dangling": True}) + dangling = [image.id for image in dangling if _isIntermediateImage(image)] + cleaner.cleanMultiple(dangling, args.dry_run) + + # If requested, remove ue4-source images + if args.source == True: + _cleanMatching( + cleaner, + GlobalConfiguration.resolveTag("ue4-source"), + args.tag, + args.dry_run, + ) + + # If requested, remove everything + if args.all == True: + _cleanMatching( + cleaner, GlobalConfiguration.resolveTag("ue4-*"), args.tag, args.dry_run + ) + + # If requested, run `docker system prune` + if args.prune == True: + logger.action("Running `docker system prune`...") + pruneCommand = ["docker", "system", "prune", "-f"] + if args.dry_run == True: + print(pruneCommand) + else: + subprocess.call(pruneCommand) diff --git a/src/ue4docker/diagnostics/__init__.py b/src/ue4docker/diagnostics/__init__.py new file mode 100644 index 00000000..68997c6a --- /dev/null +++ b/src/ue4docker/diagnostics/__init__.py @@ -0,0 +1,5 @@ +from .diagnostic_all import allDiagnostics +from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_20gig import diagnostic20Gig +from .diagnostic_ipv6 import diagnosticIPv6 +from .diagnostic_network import diagnosticNetwork diff --git a/src/ue4docker/diagnostics/base.py b/src/ue4docker/diagnostics/base.py new file mode 100644 index 00000000..a77f91e9 --- /dev/null +++ b/src/ue4docker/diagnostics/base.py @@ -0,0 +1,159 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from os.path import abspath, dirname, join +import subprocess + + +class DiagnosticBase(object): + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + raise NotImplementedError + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + raise NotImplementedError + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + raise NotImplementedError + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + raise NotImplementedError + + # Helper functionality for derived classes + + def _printAndRun(self, logger, prefix, command, check=False): + """ + Prints a command and then executes it + """ + logger.info(prefix + "Run: {}".format(command), False) + subprocess.run(command, check=check) + + def _checkPlatformMistmatch(self, logger, containerPlatform, linuxFlagSupported): + """ + Verifies that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + """ + prefix = self.getPrefix() + dockerInfo = DockerUtils.info() + dockerPlatform = dockerInfo["OSType"].lower() + if containerPlatform == "windows" and dockerPlatform == "linux": + logger.error( + "[{}] Error: attempting to test Windows containers while Docker Desktop is in Linux container mode.".format( + prefix + ), + False, + ) + if linuxFlagSupported: + logger.error( + "[{}] Use the --linux flag if you want to test Linux containers instead.".format( + prefix + ), + False, + ) + raise RuntimeError + elif containerPlatform == "linux" and dockerPlatform == "windows": + logger.error( + "[{}] Error: attempting to test Linux containers while Docker Desktop is in Windows container mode.".format( + prefix + ), + False, + ) + if linuxFlagSupported: + logger.error( + "[{}] Remove the --linux flag if you want to test Windows containers instead.".format( + prefix + ), + False, + ) + raise RuntimeError + + def _generateWindowsBuildArgs( + self, logger, basetagOverride=None, isolationOverride=None + ): + """ + Generates the build arguments for testing Windows containers, with optional overrides for base tag and isolation mode + """ + + # Determine the appropriate container image base tag for the host system release unless the user specified a base tag + buildArgs = [] + hostBaseTag = WindowsUtils.getHostBaseTag() + baseTag = basetagOverride if basetagOverride is not None else hostBaseTag + + if baseTag is None: + raise RuntimeError( + "unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag" + ) + + buildArgs = ["--build-arg", "BASETAG={}".format(baseTag)] + + # Use the default isolation mode unless requested otherwise + dockerInfo = DockerUtils.info() + isolation = ( + isolationOverride + if isolationOverride is not None + else dockerInfo["Isolation"] + ) + buildArgs += ["--isolation={}".format(isolation)] + + # If the user specified process isolation mode and a different base tag to the host system then warn them + prefix = self.getPrefix() + if isolation == "process" and baseTag != hostBaseTag: + logger.info( + "[{}] Warning: attempting to use different Windows container/host versions".format( + prefix + ), + False, + ) + logger.info( + "[{}] when running in process isolation mode, this will usually break!".format( + prefix + ), + False, + ) + + # Set a sensible memory limit when using Hyper-V isolation mode + if isolation == "hyperv": + buildArgs += ["-m", "4GiB"] + + return buildArgs + + def _buildDockerfile(self, logger, containerPlatform, tag, buildArgs): + """ + Attempts to build the diagnostic's Dockerfile for the specified container platform, with the specified parameters + """ + + # Attempt to build the Dockerfile + prefix = self.getPrefix() + contextDir = join( + dirname(dirname(abspath(__file__))), + "dockerfiles", + "diagnostics", + prefix, + containerPlatform, + ) + try: + command = ["docker", "build", "-t", tag, contextDir] + buildArgs + self._printAndRun(logger, "[{}] ".format(prefix), command, check=True) + built = True + except: + logger.error("[{}] Build failed!".format(prefix)) + built = False + + # Remove any built images, including intermediate images + logger.action("[{}] Cleaning up...".format(prefix), False) + if built == True: + self._printAndRun(logger, "[{}] ".format(prefix), ["docker", "rmi", tag]) + self._printAndRun( + logger, "[{}] ".format(prefix), ["docker", "system", "prune", "-f"] + ) + + # Report the success or failure of the build + return built diff --git a/src/ue4docker/diagnostics/diagnostic_20gig.py b/src/ue4docker/diagnostics/diagnostic_20gig.py new file mode 100644 index 00000000..2947d0ef --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_20gig.py @@ -0,0 +1,92 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase + +import argparse, os, platform +from os.path import abspath, dirname, join + + +class diagnostic20Gig(DiagnosticBase): + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:20gig" + + def __init__(self): + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 20gig") + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check for Docker 20GiB COPY bug" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if the Docker daemon suffers from 20GiB COPY bug", + "", + "See https://github.com/adamrehn/ue4-docker/issues/99#issuecomment-1079702817 for details and workarounds", + "", + self._parser.format_help(), + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "20gig" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which platform we are running on + containerPlatform = platform.system().lower() + + if containerPlatform != "windows": + logger.action( + "[20gig] Diagnostic skipped. Current platform is not affected by the bug this diagnostic checks\n" + ) + return True + + buildArgs = self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + + # Attempt to build the Dockerfile + logger.action( + "[20gig] Attempting to COPY more than 20GiB between layers...", False + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnostic20Gig.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[20gig] Diagnostic succeeded! The Docker daemon can COPY more than 20GiB between layers.\n" + ) + else: + logger.error( + "[20gig] Diagnostic failed! The Docker daemon cannot COPY more than 20GiB between layers.\n", + True, + ) + + return built diff --git a/src/ue4docker/diagnostics/diagnostic_8gig.py b/src/ue4docker/diagnostics/diagnostic_8gig.py new file mode 100644 index 00000000..b6a464d9 --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_8gig.py @@ -0,0 +1,114 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase + +import argparse, os, platform +from os.path import abspath, dirname, join + + +class diagnostic8Gig(DiagnosticBase): + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:8gig" + + def __init__(self): + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics 8gig") + self._parser.add_argument( + "--linux", + action="store_true", + help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)", + ) + self._parser.add_argument( + "--random", + action="store_true", + help="Create a file filled with random bytes instead of zeroes under Windows", + ) + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check for Docker 8GiB filesystem layer bug" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if the Docker daemon suffers from one of the 8GiB filesystem", + "layer bugs reported at https://github.com/moby/moby/issues/37581 (affects all platforms)", + "or https://github.com/moby/moby/issues/40444 (affects Windows containers only)", + "", + "#37581 was fixed in Docker CE 18.09.0 and #40444 was fixed in Docker CE 20.10.0", + "", + self._parser.format_help(), + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "8gig" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = ( + "linux" + if args.linux == True or platform.system().lower() != "windows" + else "windows" + ) + + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + try: + self._checkPlatformMistmatch(logger, containerPlatform, True) + except RuntimeError: + return False + + # Set our build arguments when testing Windows containers + buildArgs = ( + self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + if containerPlatform == "windows" + else [] + ) + + # Attempt to build the Dockerfile + logger.action( + "[8gig] Attempting to build an image with an 8GiB filesystem layer...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnostic8Gig.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[8gig] Diagnostic succeeded! The Docker daemon can build images with 8GiB filesystem layers.\n" + ) + else: + logger.error( + "[8gig] Diagnostic failed! The Docker daemon cannot build images with 8GiB filesystem layers.\n", + True, + ) + + return built diff --git a/src/ue4docker/diagnostics/diagnostic_all.py b/src/ue4docker/diagnostics/diagnostic_all.py new file mode 100644 index 00000000..4872cd42 --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_all.py @@ -0,0 +1,55 @@ +from .base import DiagnosticBase +from .diagnostic_8gig import diagnostic8Gig +from .diagnostic_20gig import diagnostic20Gig +from .diagnostic_ipv6 import diagnosticIPv6 +from .diagnostic_network import diagnosticNetwork + + +class allDiagnostics(DiagnosticBase): + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Run all available diagnostics" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "This diagnostic runs all available diagnostics in sequence." + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Run all available diagnostics in turn, storing the results + results = [] + diagnostics = [ + diagnostic8Gig(), + diagnostic20Gig(), + diagnosticIPv6(), + diagnosticNetwork(), + ] + for index, diagnostic in enumerate(diagnostics): + # Run the diagnostic and report its result + logger.info( + '[all] Running individual diagnostic: "{}"'.format( + diagnostic.getName() + ), + True, + ) + results.append(diagnostic.run(logger)) + logger.info( + "[all] Individual diagnostic result: {}".format( + "passed" if results[-1] == True else "failed" + ), + False, + ) + + # Print a newline after the last diagnostic has run + if index == len(diagnostics) - 1: + print() + + # Only report success if all diagnostics succeeded + return False not in results diff --git a/src/ue4docker/diagnostics/diagnostic_ipv6.py b/src/ue4docker/diagnostics/diagnostic_ipv6.py new file mode 100644 index 00000000..7fb78755 --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_ipv6.py @@ -0,0 +1,76 @@ +from .base import DiagnosticBase + + +class diagnosticIPv6(DiagnosticBase): + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:ipv6" + + def __init__(self): + pass + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check that Linux containers can access the IPv6 loopback address" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines whether Linux containers are able to access the IPv6,", + "loopback address ::1, which is required by Unreal Engine 5.4 and newer for", + "local ZenServer communication.", + "", + "This should work automatically under Docker 26.0.0 and newer, but older versions", + "require manual configuration by the user.", + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "ipv6" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # This diagnostic only applies to Linux containers + containerPlatform = "linux" + + # Verify that the user isn't trying to test Linux containers under Windows 10 when in Windows container mode + try: + self._checkPlatformMistmatch(logger, containerPlatform, False) + except RuntimeError: + return False + + # Attempt to build the Dockerfile + logger.action( + "[network] Attempting to build an image that accesses the IPv6 loopback address...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnosticIPv6.IMAGE_TAG, [] + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[network] Diagnostic succeeded! Linux containers can access the IPv6 loopback address without any issues.\n" + ) + else: + logger.error( + "[network] Diagnostic failed! Linux containers cannot access the IPv6 loopback address. Update to Docker 26.0.0+ or manually enable IPv6:", + True, + ) + logger.error( + "[network] https://docs.docker.com/config/daemon/ipv6/#use-ipv6-for-the-default-bridge-network\n", + False, + ) + + return built diff --git a/src/ue4docker/diagnostics/diagnostic_network.py b/src/ue4docker/diagnostics/diagnostic_network.py new file mode 100644 index 00000000..4c8f99a9 --- /dev/null +++ b/src/ue4docker/diagnostics/diagnostic_network.py @@ -0,0 +1,108 @@ +from ..infrastructure import DockerUtils, WindowsUtils +from .base import DiagnosticBase +import argparse, platform + + +class diagnosticNetwork(DiagnosticBase): + # The tag we use for built images + IMAGE_TAG = "adamrehn/ue4-docker/diagnostics:network" + + def __init__(self): + # Setup our argument parser so we can use its help message output in our description text + self._parser = argparse.ArgumentParser(prog="ue4-docker diagnostics network") + self._parser.add_argument( + "--linux", + action="store_true", + help="Use Linux containers under Windows hosts (useful when testing Docker Desktop or LCOW support)", + ) + self._parser.add_argument( + "--isolation", + default=None, + choices=["hyperv", "process"], + help="Override the default isolation mode when testing Windows containers", + ) + self._parser.add_argument( + "-basetag", + default=None, + choices=WindowsUtils.getKnownBaseTags(), + help="Override the default base image tag when testing Windows containers", + ) + + def getName(self): + """ + Returns the human-readable name of the diagnostic + """ + return "Check that containers can access the internet correctly" + + def getDescription(self): + """ + Returns a description of what the diagnostic does + """ + return "\n".join( + [ + "This diagnostic determines if running containers are able to access the internet,", + "resolve DNS entries, and download remote files.", + "", + "This is primarily useful in troubleshooting network connectivity and proxy issues.", + ] + ) + + def getPrefix(self): + """ + Returns the short name of the diagnostic for use in log output + """ + return "network" + + def run(self, logger, args=[]): + """ + Runs the diagnostic + """ + + # Parse our supplied arguments + args = self._parser.parse_args(args) + + # Determine which image platform we will build the Dockerfile for (default is the host platform unless overridden) + containerPlatform = ( + "linux" + if args.linux == True or platform.system().lower() != "windows" + else "windows" + ) + + # Verify that the user isn't trying to test Windows containers under Windows 10 when in Linux container mode (or vice versa) + try: + self._checkPlatformMistmatch(logger, containerPlatform, True) + except RuntimeError: + return False + + # Set our build arguments when testing Windows containers + buildArgs = ( + self._generateWindowsBuildArgs(logger, args.basetag, args.isolation) + if containerPlatform == "windows" + else [] + ) + + # Attempt to build the Dockerfile + logger.action( + "[network] Attempting to build an image that accesses network resources...", + False, + ) + built = self._buildDockerfile( + logger, containerPlatform, diagnosticNetwork.IMAGE_TAG, buildArgs + ) + + # Inform the user of the outcome of the diagnostic + if built == True: + logger.action( + "[network] Diagnostic succeeded! Running containers can access network resources without any issues.\n" + ) + else: + logger.error( + "[network] Diagnostic failed! Running containers cannot access network resources. See the docs for troubleshooting tips:", + True, + ) + logger.error( + "[network] https://adamrehn.github.io/ue4-docker/#building-the-ue4-build-prerequisites-image-fails-with-a-network-related-error\n", + False, + ) + + return built diff --git a/src/ue4docker/diagnostics_cmd.py b/src/ue4docker/diagnostics_cmd.py new file mode 100644 index 00000000..c112e02c --- /dev/null +++ b/src/ue4docker/diagnostics_cmd.py @@ -0,0 +1,69 @@ +from .infrastructure import Logger, PrettyPrinting +from .diagnostics import * +import sys + + +def diagnostics(): + # The diagnostics that can be run + DIAGNOSTICS = { + "all": allDiagnostics(), + "8gig": diagnostic8Gig(), + "20gig": diagnostic20Gig(), + "ipv6": diagnosticIPv6(), + "network": diagnosticNetwork(), + } + + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} diagnostics] ".format(sys.argv[0])) + + # Parse the supplied command-line arguments + stripped = list([arg for arg in sys.argv if arg.strip("-") not in ["h", "help"]]) + args = { + "help": len(sys.argv) > len(stripped), + "diagnostic": stripped[1] if len(stripped) > 1 else None, + } + + # If a diagnostic name has been specified, verify that it is valid + if args["diagnostic"] is not None and args["diagnostic"] not in DIAGNOSTICS: + logger.error( + 'Error: unrecognised diagnostic "{}".'.format(args["diagnostic"]), False + ) + sys.exit(1) + + # Determine if we are running a diagnostic + if args["help"] == False and args["diagnostic"] is not None: + # Run the diagnostic + diagnostic = DIAGNOSTICS[args["diagnostic"]] + logger.action('Running diagnostic: "{}"'.format(diagnostic.getName()), False) + passed = diagnostic.run(logger, stripped[2:]) + + # Print the result + if passed == True: + logger.action("Diagnostic result: passed", False) + else: + logger.error("Diagnostic result: failed", False) + + # Determine if we are displaying the help for a specific diagnostic + elif args["help"] == True and args["diagnostic"] is not None: + # Display the help for the diagnostic + diagnostic = DIAGNOSTICS[args["diagnostic"]] + print("{} diagnostics {}".format(sys.argv[0], args["diagnostic"])) + print(diagnostic.getName() + "\n") + print(diagnostic.getDescription()) + + else: + # Print usage syntax + print("Usage: {} diagnostics DIAGNOSTIC\n".format(sys.argv[0])) + print("Runs diagnostics to detect issues with the host system configuration\n") + print("Available diagnostics:") + PrettyPrinting.printColumns( + [ + (diagnostic, DIAGNOSTICS[diagnostic].getName()) + for diagnostic in DIAGNOSTICS + ] + ) + print( + "\nRun `{} diagnostics DIAGNOSTIC --help` for more information on a diagnostic.".format( + sys.argv[0] + ) + ) diff --git a/src/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile new file mode 100644 index 00000000..b12c58cb --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/20gig/windows/Dockerfile @@ -0,0 +1,22 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} as intermediate + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +WORKDIR C:\stuff + +# Create three 7GB files +# We do not use a single 21GiB file in order to distinguish this diagnostic from 8GiB diagnostic +RUN fsutil file createnew C:\stuff\bigfile_1.txt 7516192768 +RUN fsutil file createnew C:\stuff\bigfile_2.txt 7516192768 +RUN fsutil file createnew C:\stuff\bigfile_3.txt 7516192768 + +FROM mcr.microsoft.com/windows/servercore:${BASETAG} + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Copy more than 20GiB between docker layers +COPY --from=intermediate C:\stuff C:\stuff diff --git a/src/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile new file mode 100644 index 00000000..fb5f463d --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/8gig/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# The BusyBox version of `head` doesn't support the syntax "8G", so we specify 8GiB in bytes +RUN head -c 8589934592 file diff --git a/src/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile new file mode 100644 index 00000000..760d9920 --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/Dockerfile @@ -0,0 +1,12 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} +SHELL ["cmd", "/S", "/C"] + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Atttempt to create an 8GiB filesystem layer +ARG CREATE_RANDOM +COPY test.ps1 C:\test.ps1 +RUN powershell -ExecutionPolicy Bypass -File C:\test.ps1 diff --git a/src/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 new file mode 100644 index 00000000..436f4eb2 --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/8gig/windows/test.ps1 @@ -0,0 +1,24 @@ +# Determine if we are creating a file filled with zeroes or random bytes +if ($Env:CREATE_RANDOM -ne 1) +{ + Write-Host 'Attempting to create an 8GiB file containing zeroes...'; + fsutil.exe file createnew file 8589934592 +} +else +{ + Write-Host 'Attempting to create an 8GiB file containing random bytes...'; + $ErrorActionPreference = 'Stop'; + $writer = [System.IO.File]::OpenWrite('file'); + + # Write the file in blocks of 1GiB to avoid allocating too much memory in one hit + $random = new-object Random; + $blockSize = 1073741824; + $bytes = new-object byte[] $blockSize; + for ($i=0; $i -lt 8; $i++) + { + $random.NextBytes($bytes); + $writer.Write($bytes, 0, $blockSize); + } + + $writer.Close(); +} diff --git a/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile new file mode 100644 index 00000000..12afdb7d --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/ipv6/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Test that we can ping the IPv6 loopback address +RUN ping6 -c 5 '::1' diff --git a/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile new file mode 100644 index 00000000..d99976cf --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/network/linux/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:latest + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Test that network works +RUN apk update diff --git a/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile b/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile new file mode 100644 index 00000000..13e68909 --- /dev/null +++ b/src/ue4docker/dockerfiles/diagnostics/network/windows/Dockerfile @@ -0,0 +1,10 @@ +# escape=` +ARG BASETAG +FROM mcr.microsoft.com/windows/servercore:${BASETAG} +SHELL ["cmd", "/S", "/C"] + +# Add a sentinel label so we can easily identify intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" + +# Attempt to download the Chocolatey installation script +RUN wget 'https://chocolatey.org/install.ps1' diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile new file mode 100644 index 00000000..faeca470 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile @@ -0,0 +1,111 @@ +ARG BASEIMAGE +FROM ${BASEIMAGE} AS prerequisites + +{% if not disable_labels %} +# Add a sentinel label so we can easily identify all derived images, including intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} + +# Disable interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# Enable CUDA support for NVIDIA GPUs (even when not using a CUDA base image), since evidently some versions of UE unconditionally assume +# `libcuda.so.1` exists when using the NVIDIA proprietary drivers, and will fail to initialise the Vulkan RHI if it is missing +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},compute + +# Add the "display" driver capability for NVIDIA GPUs +# (This allows us to run the Editor from an interactive container by bind-mounting the host system's X11 socket) +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},display + +# Enable NVENC support for use by Unreal Engine plugins that depend on it (e.g. Pixel Streaming) +# (Note that adding `video` seems to implicitly enable `compute` as well, but we include separate directives here to clearly indicate the purpose of both) +ENV NVIDIA_DRIVER_CAPABILITIES=${NVIDIA_DRIVER_CAPABILITIES},video + +# Install our build prerequisites +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + git-lfs \ + gpg-agent \ + python3 \ + python3-dev \ + python3-pip \ + shared-mime-info \ + software-properties-common \ + sudo \ + tzdata \ + unzip \ + xdg-user-dirs \ + xdg-utils \ + zip && \ + rm -rf /var/lib/apt/lists/* + +# Install the X11 runtime libraries required by CEF so we can cook Unreal Engine projects that use the WebBrowserWidget plugin +# (Starting in Unreal Engine 5.0, we need these installed before creating an Installed Build to prevent cooking failures related to loading the Quixel Bridge plugin) +RUN apt-get update && apt-get install -y --no-install-recommends \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcairo2 \ + libfontconfig1 \ + libfreetype6 \ + libgbm1 \ + libglu1 \ + libnss3 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libsm6 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxi6 \ + libxkbcommon-x11-0 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + libxv1 \ + x11-xkb-utils \ + xauth \ + xfonts-base \ + xkb-data && \ + rm -rf /var/lib/apt/lists/* + +{% if enable_dso_patch %} +# Install the glibc DSO patch to improve Editor startup times +RUN add-apt-repository -y ppa:slonopotamus/glibc-dso && \ + apt-get update && \ + apt upgrade -y libc6 && \ + rm -rf /var/lib/apt/lists/* + +# Enable the glibc DSO patch +ENV GLIBC_TUNABLES=glibc.rtld.dynamic_sort=2 +{% endif %} + +# Disable the default "lecture" message the first time a user runs a command using sudo +RUN echo 'Defaults lecture="never"' >> /etc/sudoers + +# Unreal refuses to run as the root user, so create a non-root user with no password and allow them to run commands using sudo +RUN useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ + passwd -d ue4 && \ + usermod -a -G audio,video,sudo ue4 +USER ue4 + +{% if enable_ushell %} +# Install Python 3.12, which is required by ushell +USER root +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install -y --no-install-recommends python3.12 python3.12-venv && \ + rm -rf /var/lib/apt/lists/* +USER ue4 + +# Install a copy of pip for Python 3.12 +RUN curl -fsSL 'https://bootstrap.pypa.io/get-pip.py' | python3.12 +{% endif %} + +# Enable Git Large File Storage (LFS) support +RUN git lfs install diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.dockerignore diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore similarity index 100% rename from ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore rename to src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/.gitignore diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile new file mode 100644 index 00000000..16b23050 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile @@ -0,0 +1,64 @@ +# escape=` +ARG BASEIMAGE +ARG DLLSRCIMAGE + +FROM ${DLLSRCIMAGE} AS dlls + +FROM ${BASEIMAGE} AS deduplication +SHELL ["cmd", "/S", "/C"] + +{% if not disable_labels %} +# Add a sentinel label so we can easily identify all derived images, including intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} + +# Gather the system DLLs that we need from the full Windows base image +COPY --from=dlls ` + C:\Windows\System32\avicap32.dll ` + C:\Windows\System32\avifil32.dll ` + C:\Windows\System32\avrt.dll ` + C:\Windows\System32\d3d10warp.dll ` + C:\Windows\System32\D3DSCache.dll ` + C:\Windows\System32\dsound.dll ` + C:\Windows\System32\dxva2.dll ` + C:\Windows\System32\glu32.dll ` + C:\Windows\System32\InputHost.dll ` + C:\Windows\System32\ksuser.dll ` + C:\Windows\System32\mf.dll ` + C:\Windows\System32\mfcore.dll ` + C:\Windows\System32\mfplat.dll ` + C:\Windows\System32\mfplay.dll ` + C:\Windows\System32\mfreadwrite.dll ` + C:\Windows\System32\msacm32.dll ` + C:\Windows\System32\msdmo.dll ` + C:\Windows\System32\msvfw32.dll ` + C:\Windows\System32\opengl32.dll ` + C:\Windows\System32\ResampleDMO.dll ` + C:\Windows\System32\ResourcePolicyClient.dll ` + C:\Windows\System32\XInput1_4.dll ` + C:\GatheredDLLs\ + +# Remove any DLL files that already exist in the target base image, to avoid permission errors when attempting to overwrite existing files with a COPY directive +COPY remove-duplicate-dlls.ps1 C:\remove-duplicate-dlls.ps1 +RUN powershell -ExecutionPolicy Bypass -File C:\remove-duplicate-dlls.ps1 + +# Copy the DLL files into a clean image +FROM ${BASEIMAGE} AS prerequisites +SHELL ["cmd", "/S", "/C"] +COPY --from=deduplication C:\GatheredDlls\ C:\Windows\System32\ + +{% if not disable_labels %} +# Add a sentinel label so we can easily identify all derived images, including intermediate images +LABEL com.adamrehn.ue4-docker.sentinel="1" +{% endif %} + +# Enable long path support +RUN reg add HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f + +# Install Chocolatey +RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "$env:chocolateyVersion = '1.4.0'; Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" + +# Install the rest of our build prerequisites and clean up afterwards to minimise image size +COPY install-prerequisites.ps1 C:\ +ARG VISUAL_STUDIO_BUILD_NUMBER +RUN powershell -ExecutionPolicy Bypass -File C:\install-prerequisites.ps1 %VISUAL_STUDIO_BUILD_NUMBER% diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 new file mode 100644 index 00000000..db157acb --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.ps1 @@ -0,0 +1,122 @@ +$ErrorActionPreference = "stop" + +function RunProcessChecked +{ + param ([string] $Cmd, [string[]] $Argv) + + Write-Output "Executing command: $Cmd $Argv" + + $process = Start-Process -NoNewWindow -PassThru -Wait -FilePath $Cmd -ArgumentList $Argv + if ($process.ExitCode -ne 0) + { + throw "Exit code: $($process.ExitCode)" + } +} + +# TODO: Why `Update-SessionEnvironment` doesn't Just Work without this? +# Taken from https://stackoverflow.com/a/46760714 +# Make `Update-SessionEnvironment` available right away, by defining the $env:ChocolateyInstall +# variable and importing the Chocolatey profile module. +$env:ChocolateyInstall = Convert-Path "$( (Get-Command choco).Path )\..\.." +Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1" + +# Install the chocolatey packages we need +RunProcessChecked "choco" @("install", "--no-progress", "-y", "git.install", "--params", @' +"'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'`" +'@) + +# pdbcopy.exe from Windows SDK is needed for creating an Installed Build of the Engine +RunProcessChecked "choco" @("install", "--no-progress", "-y", "choco-cleaner", "python", "vcredist-all", "windowsdriverkit10") + +# Reload our environment variables from the registry so the `git` command works +Update-SessionEnvironment + +# Forcibly disable the git credential manager +RunProcessChecked "git" @("config", "--system", "--unset", "credential.helper") + +# Gather the required DirectX runtime files, since Windows Server Core does not include them +Invoke-WebRequest -Uri "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" -OutFile "$env:TEMP\directx_redist.exe" +RunProcessChecked "$env:TEMP\directx_redist.exe" @("/Q", "/T:$env:TEMP") +RunProcessChecked "expand" @("$env:TEMP\APR2007_xinput_x64.cab", "-F:xinput1_3.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_D3DCompiler_43_x64.cab", "-F:D3DCompiler_43.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Feb2010_X3DAudio_x64.cab", "-F:X3DAudio1_7.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_XAudio_x64.cab", "-F:XAPOFX1_5.dll", "C:\Windows\System32\") +RunProcessChecked "expand" @("$env:TEMP\Jun2010_XAudio_x64.cab", "-F:XAudio2_7.dll", "C:\Windows\System32\") + +# Retrieve the DirectX shader compiler files needed for DirectX Raytracing (DXR) +Invoke-WebRequest -Uri "https://github.com/microsoft/DirectXShaderCompiler/releases/download/v1.6.2104/dxc_2021_04-20.zip" -OutFile "$env:TEMP\dxc.zip" +Expand-Archive -Path "$env:TEMP\dxc.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "$env:TEMP\bin\x64\dxcompiler.dll" C:\Windows\System32\ +Copy-Item -Path "$env:TEMP\bin\x64\dxil.dll" C:\Windows\System32\ + +# Gather the Vulkan runtime library +Invoke-WebRequest -Uri "https://sdk.lunarg.com/sdk/download/1.4.304.0/windows/VulkanRT-1.4.304.0-Components.zip?u=" -OutFile "$env:TEMP\vulkan-runtime-components.zip" +Expand-Archive -Path "$env:TEMP\vulkan-runtime-components.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "*\x64\vulkan-1.dll" -Destination C:\Windows\System32\ + +$visual_studio_build = $args[0] + +if ($visual_studio_build -eq "15") +{ + $windows_sdk_version = 18362 +} +else +{ + $windows_sdk_version = 20348 +} + +# NOTE: We use the Visual Studio 2022 installer even for Visual Studio 2019 and 2017 here because the old (2017) installer now breaks +Invoke-WebRequest -Uri "https://aka.ms/vs/17/release/vs_buildtools.exe" -OutFile "$env:TEMP\vs_buildtools.exe" + +# NOTE: Microsoft.NetCore.Component.SDK only exists for VS2019+. And it is actually *needed* only for UE5 +# NOTE: .NET 4.5 is required for some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 +# NOTE: Microsoft.NetCore.Component.Runtime.3.1 is required by the AutomationTool tool and does not come installed with VS2022 so it needs targetting here. +$vs_args = @( + "--quiet", + "--wait", + "--norestart", + "--nocache", + "--installPath", "C:\BuildTools", + "--channelUri", "https://aka.ms/vs/$visual_studio_build/release/channel", + "--installChannelUri", "https://aka.ms/vs/$visual_studio_build/release/channel", + "--channelId", "VisualStudio.$visual_studio_build.Release", + "--productId", "Microsoft.VisualStudio.Product.BuildTools", + "--locale", "en-US", + "--add", "Microsoft.VisualStudio.Workload.VCTools", + "--add", "Microsoft.VisualStudio.Workload.MSBuildTools", + "--add", "Microsoft.VisualStudio.Component.NuGet", + "--add", "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", + "--add", "Microsoft.VisualStudio.Component.Windows10SDK.$windows_sdk_version", + "--add", "Microsoft.Net.Component.4.5.TargetingPack", + "--add", "Microsoft.Net.Component.4.6.2.TargetingPack", + "--add", "Microsoft.Net.ComponentGroup.DevelopmentPrerequisites", + "--add", "Microsoft.NetCore.Component.SDK", + "--add", "Microsoft.NetCore.Component.Runtime.3.1" +) + +# Install the Visual Studio Build Tools workloads and components we need +RunProcessChecked "$env:TEMP\vs_buildtools.exe" $vs_args + +# NOTE: Install the .Net 4.5 Framework Pack via NuGet as it is no longer available via Visual Studio, but still needed +# NOTE: some programs even in UE5, for example https://github.com/EpicGames/UnrealEngine/blob/5.0.1-release/Engine/Source/Programs/UnrealSwarm/SwarmCoordinator/SwarmCoordinator.csproj#L26 +Invoke-WebRequest -Uri "https://www.nuget.org/api/v2/package/Microsoft.NETFramework.ReferenceAssemblies.net45/1.0.3" -OutFile "$env:TEMP\DotNet45.zip" +Expand-Archive -Path "$env:TEMP\DotNet45.zip" -DestinationPath "$env:TEMP" +Copy-Item -Path "$env:TEMP\build\.NETFramework\v4.5\*" -Destination "C:\Program Files (x86)\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.5\" -Recurse -Force + +# Clean up any temp files generated during prerequisite installation +Remove-Item -LiteralPath "$env:TEMP" -Recurse -Force +New-Item -Type directory -Path "$env:TEMP" + +# This shaves off ~300MB as of 2021-08-31 +RunProcessChecked "choco-cleaner" @("--dummy") + +# Something that gets installed in ue4-build-prerequisites creates a bogus NuGet config file +# Just remove it, so a proper one will be generated on next NuGet run +# See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-852136034 +if (Test-Path "$env:APPDATA\NuGet") +{ + Remove-Item -LiteralPath "$env:APPDATA\NuGet" -Recurse -Force +} + +# Display a human-readable completion message +Write-Output "Finished installing build prerequisites and cleaning up temporary files." diff --git a/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 new file mode 100644 index 00000000..47a1ad00 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-build-prerequisites/windows/remove-duplicate-dlls.ps1 @@ -0,0 +1,11 @@ +$dlls = (Get-ChildItem "C:\GatheredDLLs\*.dll") +foreach ($dll in $dlls) +{ + $filename = $dll.Name + $existing = (Get-ChildItem "C:\Windows\System32\${filename}" -ErrorAction SilentlyContinue) + if ($existing) + { + [Console]::Error.WriteLine("${filename} already exists in System32 in the target base image, excluding it from the list of DLL files to copy.") + Remove-Item $dll + } +} diff --git a/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile new file mode 100644 index 00000000..b8da8539 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-full/linux/Dockerfile @@ -0,0 +1,58 @@ +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +ARG CONAN_VERSION="conan>=1.59.0,<2" +{% if combine %} +FROM source as conan +{% else %} +ARG NAMESPACE +ARG TAG +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan +{% endif %} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION + +# Install ue4cli and conan-ue4cli +USER root +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" +USER ue4 + +# Extract the third-party library details from UBT +RUN ue4 setroot /home/ue4/UnrealEngine +RUN ue4 conan generate + +# Copy the generated Conan packages into a new image with our Installed Build +{% if combine %} +FROM minimal as full +{% else %} +FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +{% endif %} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION + +# Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers +USER root +RUN apt-get update && apt-get install -y --no-install-recommends cmake +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install "$CONAN_VERSION" "$UE4CLI_VERSION" "$CONAN_UE4CLI_VERSION" ue4-ci-helpers +USER ue4 + +# Explicitly set the configuration directory for ue4cli +# (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) +ENV UE4CLI_CONFIG_DIR=/home/ue4/.config/ue4cli + +# Copy the Conan configuration settings and package cache from the previous build stage +COPY --from=conan --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan + +# Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) +RUN ue4 setroot /home/ue4/UnrealEngine +RUN ue4 conan generate --profile-only + +# Enable PulseAudio support +USER root +RUN apt-get install -y --no-install-recommends pulseaudio-utils +COPY pulseaudio-client.conf /etc/pulse/client.conf +USER ue4 diff --git a/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf b/src/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf similarity index 100% rename from ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf rename to src/ue4docker/dockerfiles/ue4-full/linux/pulseaudio-client.conf diff --git a/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile new file mode 100644 index 00000000..2ce2b834 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-full/windows/Dockerfile @@ -0,0 +1,53 @@ +# escape=` +ARG UE4CLI_VERSION="ue4cli>=0.0.45" +ARG CONAN_UE4CLI_VERSION="conan-ue4cli>=0.0.27" +ARG CONAN_VERSION="conan>=1.59.0,<2" +{% if combine %} +FROM source as conan +{% else %} +ARG NAMESPACE +ARG TAG +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS conan +{% endif %} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION + +# Install ue4cli and conan-ue4cli +RUN pip install setuptools wheel --no-warn-script-location +RUN pip install "%CONAN_VERSION%" "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" --no-warn-script-location + +# Build UBT, and extract the third-party library details from UBT +# (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) +RUN GenerateProjectFiles.bat +RUN ue4 setroot C:\UnrealEngine +RUN ue4 conan generate && ue4 conan generate --remove-only + +# Copy the generated Conan packages into a new image with our Installed Build +{% if combine %} +FROM minimal as full +{% else %} +FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} +{% endif %} +ARG UE4CLI_VERSION +ARG CONAN_UE4CLI_VERSION +ARG CONAN_VERSION + +# Install ue4cli conan-ue4cli, and ue4-ci-helpers +RUN pip install setuptools wheel --no-warn-script-location +RUN pip install "%CONAN_VERSION%" "%UE4CLI_VERSION%" "%CONAN_UE4CLI_VERSION%" ue4-ci-helpers --no-warn-script-location + +# Explicitly set the configuration directory for ue4cli +# (This prevents things from breaking when using CI/CD systems that override the $HOME environment variable) +ENV UE4CLI_CONFIG_DIR=C:\Users\ContainerAdministrator\AppData\Roaming\ue4cli + +# Copy the Conan configuration settings and package cache from the previous build stage +COPY --from=conan C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan + +# Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) +RUN ue4 setroot C:\UnrealEngine +RUN ue4 conan generate --profile-only + +# Install CMake and add it to the system PATH +RUN choco install --no-progress -y cmake --installargs "ADD_CMAKE_TO_PATH=System" diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore b/src/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore rename to src/ue4docker/dockerfiles/ue4-minimal/linux/.dockerignore diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile new file mode 100644 index 00000000..9fec768f --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile @@ -0,0 +1,124 @@ +{% if combine %} +FROM source as builder +{% else %} +ARG NAMESPACE +ARG TAG +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +{% endif %} + +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN rm -rf /home/ue4/UnrealEngine/.git + +# Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the +# target .NET Framework version, whereas InstalledEngineBuild.xml just uses the system default, +# which can result in errors when running the built UBT due to the wrong version being targeted +RUN if [ -f ./Engine/Build/BatchFiles/BuildUBT.sh ]; then \ + ./Engine/Build/BatchFiles/BuildUBT.sh; \ + else \ + ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -SkipBuild -buildubt; \ + fi + +# Create an Installed Build of the Engine +WORKDIR /home/ue4/UnrealEngine +RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph \ + -target="Make Installed Build Linux" \ + -script=Engine/Build/InstalledEngineBuild.xml \ + -set:HostPlatformOnly=true \ + -set:WithClient=true \ + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} \ + -set:WithServer=true \ + {{ buildgraph_args }} && \ + rm -R -f /home/ue4/UnrealEngine/LocalBuilds/InstalledDDC + +# Ensure UnrealVersionSelector is built, since the prebuilt binaries may not be up-to-date +RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealVersionSelector Linux Shipping + +# Copy InstalledBuild.txt from the Installed Build and run UnrealVersionSelector to populate Install.ini with any custom Build ID specified in the BuildGraph flags +# (Note that the `-unattended` flag used below requires Unreal Engine 4.22 or newer, so this will break under older versions) +# (Note also that custom Build IDs are supported by Unreal Engine 5.3.1 and newer, and older versions will just use a GUID as the Build ID) +RUN cp /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Build/InstalledBuild.txt /home/ue4/UnrealEngine/Engine/Build/InstalledBuild.txt && \ + ./Engine/Binaries/Linux/UnrealVersionSelector-Linux-Shipping -register -unattended + +{% if enable_ushell %} +# Ensure ushell is copied to the Installed Build +RUN rm -rf ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ + cp -r ./Engine/Extras/ushell ./LocalBuilds/Engine/Linux/Engine/Extras/ushell && \ + bash -c 'set -e; shopt -s globstar; cd /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux/Engine/Extras/ushell && chmod +x ./**/*.sh' +{% endif %} + +# Split out both optional components (DDC, debug symbols, template projects) and large subdirectories so they can be copied +# into the final container image as separate filesystem layers, avoiding creating a single monolithic layer with everything +COPY split-components.py /tmp/split-components.py +RUN python3 /tmp/split-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine/Components + +# Copy the Installed Build into a clean image, discarding the source build +{% if combine %} +FROM prerequisites as minimal +{% else %} +ARG NAMESPACE +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} + +# Copy the Installed Build files from the builder image +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Binaries /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Content /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Extras /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Intermediate /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Plugins /home/ue4/UnrealEngine +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/Source /home/ue4/UnrealEngine +{% if excluded_components.ddc == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DDC /home/ue4/UnrealEngine +{% endif %} +{% if excluded_components.debug == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/DebugSymbols /home/ue4/UnrealEngine +{% endif %} +{% if excluded_components.templates == false %} +COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/Components/TemplatesAndSamples /home/ue4/UnrealEngine +{% endif %} + +# Copy Install.ini from the builder image, so it can be used by tools that read the list of engine installations (e.g. ushell) +COPY --from=builder --chown=ue4:ue4 /home/ue4/.config/Epic/UnrealEngine/Install.ini /home/ue4/.config/Epic/UnrealEngine/Install.ini +WORKDIR /home/ue4/UnrealEngine + +{% if not disable_labels %} +# Add labels to the built image to identify which components (if any) were excluded from the build that it contains +LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %} +{% endif %} + +{% if enable_ushell %} +# Add ushell to the system PATH and alias `ushell` to `ushell.sh` +ENV PATH="$PATH:/home/ue4/UnrealEngine/Engine/Extras/ushell" +RUN echo 'alias ushell="ushell.sh"' >> /home/ue4/.bashrc + +# Perform first-run setup for ushell +RUN bash -c 'set -e; source /home/ue4/UnrealEngine/Engine/Extras/ushell/ushell.sh && exit 0' +{% endif %} + +# Perform first-run setup for Mono, UnrealBuildTool and AutomationTool, which makes it possible to build Unreal projects and plugins as users other than `ue4` +# (Note that this will only work with 4.26.0 and newer, older Engine versions will always require write access to `/home/ue4/UnrealEngine`) +# See the comments on this issue for details, including the need to ensure $HOME is set correctly: +COPY print-editor-target.py /tmp/print-editor-target.py +RUN EDITOR_TARGET=$(python3 /tmp/print-editor-target.py /home/ue4/UnrealEngine) && \ + ./Engine/Build/BatchFiles/Linux/Build.sh "$EDITOR_TARGET" Linux Development -SkipBuild && \ + mkdir -p ./Engine/Programs/AutomationTool/Saved && \ + chmod a+rw ./Engine/Programs/AutomationTool/Saved + +# Enable Vulkan support for NVIDIA GPUs +USER root +RUN apt-get update && apt-get install -y --no-install-recommends libvulkan1 && \ + rm -rf /var/lib/apt/lists/* && \ + VULKAN_API_VERSION=`dpkg -s libvulkan1 | grep -oP 'Version: [0-9|\.]+' | grep -oP '[0-9|\.]+'` && \ + mkdir -p /etc/vulkan/icd.d/ && \ + echo \ + "{\ + \"file_format_version\" : \"1.0.0\",\ + \"ICD\": {\ + \"library_path\": \"libGLX_nvidia.so.0\",\ + \"api_version\" : \"${VULKAN_API_VERSION}\"\ + }\ + }" > /etc/vulkan/icd.d/nvidia_icd.json +USER ue4 diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py new file mode 100644 index 00000000..d1619bf3 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/print-editor-target.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python3 +from pathlib import Path +import json, sys + +# Parse the Unreal Engine version information +engineRoot = Path(sys.argv[1]) +versionFile = engineRoot / "Engine" / "Build" / "Build.version" +versionDetails = json.loads(versionFile.read_text("utf-8")) + +# Determine the name of the Editor target based on the version +target = "UE4Editor" if versionDetails["MajorVersion"] == 4 else "UnrealEditor" +print(target) diff --git a/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py new file mode 100644 index 00000000..9c10a6b5 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/linux/split-components.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import glob, os, shutil, sys +from os.path import basename, dirname, exists, join + + +# Logs a message to stderr +def log(message): + print(message, file=sys.stderr) + sys.stderr.flush() + + +# Extracts the files and directories for the specified component and moves them to a separate output directory +def extractComponent(inputDir, outputDir, component, description, items): + # Print progress output + log("\nExtracting {}...".format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + # Verify that the item exists + if not exists(item): + log("Skipping non-existent item: {}".format(item)) + continue + + # Print progress output + log("Moving: {}".format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move(item, join(parent, basename(item))) + + +# Retrieve the path to the root directory of the Installed Build +rootDir = sys.argv[1] + +# Retrieve the path to the root output directory for extracted components and ensure it exists +outputDir = sys.argv[2] +os.makedirs(outputDir, exist_ok=True) + +# Extract the DDC +ddc = [join(rootDir, "Engine", "DerivedDataCache", "Compressed.ddp")] +extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) + +# Extract debug symbols +symbolFiles = glob.glob(join(rootDir, "**", "*.debug"), recursive=True) + glob.glob( + join(rootDir, "**", "*.sym"), recursive=True +) +extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) + +# Extract template projects and samples +subdirs = [join(rootDir, subdir) for subdir in ["FeaturePacks", "Samples", "Templates"]] +extractComponent( + rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs +) + +# Extract the larger non-optional subdirectories of the Engine directory +for subdir in ["Binaries", "Content", "Extras", "Intermediate", "Plugins", "Source"]: + extractComponent( + rootDir, + outputDir, + subdir, + f"{subdir} subdirectory", + [join(rootDir, "Engine", subdir)], + ) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-minimal/windows/.dockerignore diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile new file mode 100644 index 00000000..1c38ecdb --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile @@ -0,0 +1,63 @@ +# escape=` +{% if combine %} +FROM source as builder +{% else %} +ARG NAMESPACE +ARG TAG +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder +{% endif %} + +# Remove the .git directory to disable UBT `git status` calls and speed up the build process +RUN if exist C:\UnrealEngine\.git rmdir /s /q C:\UnrealEngine\.git + +# Create an Installed Build of the Engine +WORKDIR C:\UnrealEngine +RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph ` + -target="Make Installed Build Win64" ` + -script=Engine/Build/InstalledEngineBuild.xml ` + -set:HostPlatformOnly=true ` + -set:WithClient=true ` + -set:WithDDC={% if excluded_components.ddc == true %}false{% else %}true{% endif %} ` + -set:WithServer=true ` + {{ buildgraph_args }} && ` + (if exist C:\UnrealEngine\LocalBuilds\InstalledDDC rmdir /s /q C:\UnrealEngine\LocalBuilds\InstalledDDC) && ` + rmdir /s /q C:\UnrealEngine\Engine + +# Split out components (DDC, debug symbols, template projects) so they can be copied into the final container image as separate filesystem layers +COPY split-components.py C:\split-components.py +RUN python C:\split-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine\Components + +# Copy the Installed Build into a clean image, discarding the source tree +{% if combine %} +FROM prerequisites as minimal +{% else %} +ARG NAMESPACE +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} + +# Copy the Installed Build files from the builder image +COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Binaries C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Content C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Extras C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Intermediate C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Plugins C:\UnrealEngine +COPY --from=builder C:\UnrealEngine\Components\Source C:\UnrealEngine +{% if excluded_components.ddc == false %} +COPY --from=builder C:\UnrealEngine\Components\DDC C:\UnrealEngine +{% endif %} +{% if excluded_components.debug == false %} +COPY --from=builder C:\UnrealEngine\Components\DebugSymbols C:\UnrealEngine +{% endif %} +{% if excluded_components.templates == false %} +COPY --from=builder C:\UnrealEngine\Components\TemplatesAndSamples C:\UnrealEngine +{% endif %} +WORKDIR C:\UnrealEngine + +{% if not disable_labels %} +# Add labels to the built image to identify which components (if any) were excluded from the build that it contains +LABEL com.adamrehn.ue4-docker.excluded.ddc={% if excluded_components.ddc == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.debug={% if excluded_components.debug == true %}1{% else %}0{% endif %} +LABEL com.adamrehn.ue4-docker.excluded.templates={% if excluded_components.templates == true %}1{% else %}0{% endif %} +{% endif %} diff --git a/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py new file mode 100644 index 00000000..7c3384ba --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-minimal/windows/split-components.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import glob, os, shutil, sys +from os.path import basename, dirname, exists, join + + +# Logs a message to stderr +def log(message): + print(message, file=sys.stderr) + sys.stderr.flush() + + +# Extracts the files and directories for the specified component and moves them to a separate output directory +def extractComponent(inputDir, outputDir, component, description, items): + # Print progress output + log("\nExtracting {}...".format(description)) + + # Create the output directory for the component if it doesn't already exist + componentDir = join(outputDir, component) + os.makedirs(outputDir, exist_ok=True) + + # Move each file and directory for the component to the output directory + for item in items: + # Verify that the item exists + if not exists(item): + log("Skipping non-existent item: {}".format(item)) + continue + + # Print progress output + log("Moving: {}".format(item)) + + # Ensure the parent directory for the item exists in the output directory + parent = dirname(item).replace(inputDir, componentDir) + os.makedirs(parent, exist_ok=True) + + # Perform the move + shutil.move(item, join(parent, basename(item))) + + +# Retrieve the path to the root directory of the Installed Build +rootDir = sys.argv[1] + +# Retrieve the path to the root output directory for extracted components and ensure it exists +outputDir = sys.argv[2] +os.makedirs(outputDir, exist_ok=True) + +# Extract the DDC +ddc = [join(rootDir, "Engine", "DerivedDataCache", "Compressed.ddp")] +extractComponent(rootDir, outputDir, "DDC", "Derived Data Cache (DDC)", ddc) + +# Extract debug symbols +symbolFiles = glob.glob(join(rootDir, "**", "*U*Editor*.pdb"), recursive=True) +extractComponent(rootDir, outputDir, "DebugSymbols", "debug symbols", symbolFiles) + +# Extract template projects and samples +subdirs = [join(rootDir, subdir) for subdir in ["FeaturePacks", "Samples", "Templates"]] +extractComponent( + rootDir, outputDir, "TemplatesAndSamples", "template projects and samples", subdirs +) + +# Extract the larger non-optional subdirectories of the Engine directory +for subdir in ["Binaries", "Content", "Extras", "Intermediate", "Plugins", "Source"]: + extractComponent( + rootDir, + outputDir, + subdir, + f"{subdir} subdirectory", + [join(rootDir, "Engine", subdir)], + ) diff --git a/ue4docker/dockerfiles/ue4-source/linux/.dockerignore b/src/ue4docker/dockerfiles/ue4-source/linux/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/.dockerignore rename to src/ue4docker/dockerfiles/ue4-source/linux/.dockerignore diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile new file mode 100644 index 00000000..1e5a1837 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/linux/Dockerfile @@ -0,0 +1,120 @@ +{% if combine %} +FROM prerequisites as source +{% else %} +ARG NAMESPACE +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} + +{% if source_mode == "copy" %} + +# Copy the Unreal Engine source code from the host system +ARG SOURCE_LOCATION +COPY --chown=ue4:ue4 ${SOURCE_LOCATION} /home/ue4/UnrealEngine + +{% else %} + +# The git repository that we will clone +ARG GIT_REPO="" + +# The git branch/tag/commit that we will checkout +ARG GIT_BRANCH="" + +{% if credential_mode == "secrets" %} + +# Install our git credential helper that retrieves credentials from build secrets +COPY --chown=ue4:ue4 git-credential-helper-secrets.sh /tmp/git-credential-helper-secrets.sh +ENV GIT_ASKPASS=/tmp/git-credential-helper-secrets.sh +RUN chmod +x /tmp/git-credential-helper-secrets.sh + +# Clone the UE4 git repository using the build secret credentials +# (Note that we include the changelist override value here to ensure any cached source code is invalidated if +# the override is modified between runs, which is useful when testing preview versions of the Unreal Engine) +ARG CHANGELIST +RUN --mount=type=secret,id=username,uid=1000,required \ + --mount=type=secret,id=password,uid=1000,required \ + CHANGELIST="$CHANGELIST" \ + mkdir /home/ue4/UnrealEngine && \ + cd /home/ue4/UnrealEngine && \ + git init && \ + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && \ + {% endfor %} + {% endif %} + git remote add origin "$GIT_REPO" && \ + git fetch --progress --depth 1 origin "$GIT_BRANCH" && \ + git checkout FETCH_HEAD + +{% else %} + +# Retrieve the address for the host that will supply git credentials +ARG HOST_ADDRESS_ARG="" +ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} + +# Retrieve the security token for communicating with the credential supplier +ARG HOST_TOKEN_ARG="" +ENV HOST_TOKEN=${HOST_TOKEN_ARG} + +# Install our git credential helper that forwards requests to the credential HTTP endpoint on the host +COPY --chown=ue4:ue4 git-credential-helper-endpoint.sh /tmp/git-credential-helper-endpoint.sh +ENV GIT_ASKPASS=/tmp/git-credential-helper-endpoint.sh +RUN chmod +x /tmp/git-credential-helper-endpoint.sh + +# Clone the UE4 git repository using the endpoint-supplied credentials +RUN mkdir /home/ue4/UnrealEngine && \ + cd /home/ue4/UnrealEngine && \ + git init && \ + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && \ + {% endfor %} + {% endif %} + git remote add origin "$GIT_REPO" && \ + git fetch --progress --depth 1 origin "$GIT_BRANCH" && \ + git checkout FETCH_HEAD + +{% endif %} + +{% endif %} + +{% if not disable_all_patches %} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 +{% endif %} + +{% if (not disable_all_patches) and (not disable_release_patches) %} +# Apply our bugfix patches to broken Engine releases +# (Make sure we do this before the post-clone setup steps are run) +COPY --chown=ue4:ue4 patch-broken-releases.py /tmp/patch-broken-releases.py +RUN python3 /tmp/patch-broken-releases.py /home/ue4/UnrealEngine $VERBOSE_OUTPUT +{% endif %} + +# Run post-clone setup steps, ensuring our package lists are up to date since Setup.sh doesn't call `apt-get update` +{% if credential_mode == "secrets" %} + +# Ensure Setup.sh uses the same cache path when building either UE4 or UE5 +ENV UE_GITDEPS=/home/ue4/gitdeps +ENV UE4_GITDEPS=/home/ue4/gitdeps +RUN mkdir "$UE_GITDEPS" + +# When running with BuildKit, we use a cache mount to cache the dependency data across multiple build invocations +WORKDIR /home/ue4/UnrealEngine +RUN --mount=type=cache,target=/home/ue4/gitdeps,uid=1000,gid=1000 sudo apt-get update && \ + ./Setup.sh {{ gitdependencies_args }} && \ + sudo rm -rf /var/lib/apt/lists/* + +{% else %} + +# When running without BuildKit, we use the `-no-cache` flag to disable caching of dependency data in `.git/ue4-gitdeps`, saving disk space +WORKDIR /home/ue4/UnrealEngine +RUN sudo apt-get update && \ + ./Setup.sh -no-cache {{ gitdependencies_args }} && \ + sudo rm -rf /var/lib/apt/lists/* + +{% endif %} + +# Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST +COPY set-changelist.py /tmp/set-changelist.py +RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version $CHANGELIST diff --git a/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper.sh b/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh similarity index 100% rename from ue4docker/dockerfiles/ue4-source/linux/git-credential-helper.sh rename to src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-endpoint.sh diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh b/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh new file mode 100644 index 00000000..3063419f --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/linux/git-credential-helper-secrets.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +if [[ "$1" == "Password for"* ]]; then + cat /run/secrets/password +else + cat /run/secrets/username +fi diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py new file mode 100644 index 00000000..5dc2bf55 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/linux/patch-broken-releases.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import json +import sys +from os.path import join + + +def readFile(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +def writeFile(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +engineRoot = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" +versionDetails = json.loads( + readFile(join(engineRoot, "Engine", "Build", "Build.version")) +) + +if ( + versionDetails["MajorVersion"] == 5 + and versionDetails["MinorVersion"] == 0 + and versionDetails["PatchVersion"] == 0 +): + buildFile = join(engineRoot, "Engine", "Build", "InstalledEngineFilters.xml") + buildXml = readFile(buildFile) + + # Add missing SetupDotnet.sh in Unreal Engine 5.0.0-early-access-1 + # See https://github.com/adamrehn/ue4-docker/issues/171#issuecomment-853918412 + # and https://github.com/EpicGames/UnrealEngine/commit/a18824057e6cd490750a10b59af29ca10b3d67d9 + dotnet = "Engine/Binaries/ThirdParty/DotNet/Linux/..." + setup_dotnet = "Engine/Build/BatchFiles/Linux/SetupDotnet.sh" + if dotnet in buildXml and setup_dotnet not in buildXml: + buildXml = buildXml.replace(dotnet, f"{dotnet}\n{setup_dotnet}") + + writeFile(buildFile, buildXml) + + if verboseOutput: + print("PATCHED {}:\n\n{}".format(buildFile, buildXml), file=sys.stderr) + else: + print("PATCHED {}".format(buildFile), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/linux/set-changelist.py b/src/ue4docker/dockerfiles/ue4-source/linux/set-changelist.py new file mode 100644 index 00000000..ea54442c --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/linux/set-changelist.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +from os.path import dirname +from subprocess import run, PIPE +import json, re, sys + + +def readFile(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +def writeFile(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +# Determine whether a changelist override value was specified +changelistOverride = None +if len(sys.argv) > 2: + # If the override was "auto" then attempt to retrieve the CL number from the git commit message + if sys.argv[2] == "auto": + # Retrieve the commit message from git + engineRoot = dirname(dirname(dirname(sys.argv[1]))) + commitMessage = run( + ["git", "log", "-n", "1", "--format=%s%n%b"], + cwd=engineRoot, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + ).stdout.strip() + + # If the commit is a tagged engine release then it won't have a CL number, and using "auto" is user error + if re.fullmatch("[0-9\\.]+ release", commitMessage) is not None: + print( + "Error: you are attempting to automatically retrieve the CL number for a tagged Unreal Engine release.\n" + "For hotfix releases of the Unreal Engine, a CL override is not required and should not be specified.\n" + "For supported .0 releases of the Unreal Engine, ue4-docker ships with known CL numbers, so an override should not be necessary.", + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to extract the CL number from the commit message + match = re.search("\\[CL ([0-9]+) by .+ in .+ branch\\]", commitMessage) + if match is not None: + changelistOverride = int(match.group(1)) + else: + print( + "Error: failed to find a CL number in the git commit message! This was the commit message:\n\n" + + commitMessage, + file=sys.stderr, + ) + sys.exit(1) + + else: + changelistOverride = int(sys.argv[2]) + +# Update the `Changelist` field to reflect the override if it was supplied, or else the `CompatibleChangelist` field in our version file +versionFile = sys.argv[1] +details = json.loads(readFile(versionFile)) +details["Changelist"] = ( + changelistOverride + if changelistOverride is not None + else details["CompatibleChangelist"] +) +details["IsPromotedBuild"] = 1 +patchedJson = json.dumps(details, indent=4) +writeFile(versionFile, patchedJson) +print("PATCHED BUILD.VERSION:\n{}".format(patchedJson), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/windows/.dockerignore b/src/ue4docker/dockerfiles/ue4-source/windows/.dockerignore similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/.dockerignore rename to src/ue4docker/dockerfiles/ue4-source/windows/.dockerignore diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile new file mode 100644 index 00000000..a2c13b8a --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/windows/Dockerfile @@ -0,0 +1,81 @@ +# escape=` +{% if combine %} +FROM prerequisites as source +{% else %} +ARG NAMESPACE +ARG PREREQS_TAG +FROM ${NAMESPACE}/ue4-build-prerequisites:${PREREQS_TAG} +{% endif %} + +{% if source_mode == "copy" %} + +# Copy the Unreal Engine source code from the host system +ARG SOURCE_LOCATION +COPY ${SOURCE_LOCATION} C:\UnrealEngine + +{% else %} + +# The git repository that we will clone +ARG GIT_REPO="" + +# The git branch/tag/commit that we will checkout +ARG GIT_BRANCH="" + +# Retrieve the address for the host that will supply git credentials +ARG HOST_ADDRESS_ARG="" +ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} + +# Retrieve the security token for communicating with the credential supplier +ARG HOST_TOKEN_ARG="" +ENV HOST_TOKEN=${HOST_TOKEN_ARG} + +# Install our git credential helper that forwards requests to the host +COPY git-credential-helper.bat C:\git-credential-helper.bat +ENV GIT_ASKPASS=C:\git-credential-helper.bat +# See https://github.com/git-ecosystem/git-credential-manager/blob/main/docs/environment.md#GCM_INTERACTIVE +ENV GCM_INTERACTIVE=false + +# Clone the UE4 git repository using the host-supplied credentials +WORKDIR C:\ +RUN mkdir C:\UnrealEngine && ` + cd C:\UnrealEngine && ` + git init && ` + {% if git_config %} + {% for key, value in git_config.items() %} + git config {{ key }} {{ value }} && ` + {% endfor %} + {% endif %} + git remote add origin %GIT_REPO% && ` + git fetch --progress --depth 1 origin %GIT_BRANCH% && ` + git checkout FETCH_HEAD + +{% endif %} + +{% if (not disable_all_patches) and (not disable_windows_setup_patch) %} +# Since the UE4 prerequisites installer appears to break when newer versions +# of the VC++ runtime are present, patch out the prereqs call in Setup.bat +COPY patch-setup-win.py C:\patch-setup-win.py +RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat %VERBOSE_OUTPUT% +{% endif %} + +{% if not disable_all_patches %} +# Enable verbose output for steps that patch files? +ARG VERBOSE_OUTPUT=0 +{% endif %} + +{% if (not disable_all_patches) and (not disable_release_patches) %} +# Apply our bugfix patches to broken Engine releases +# (Make sure we do this before the post-clone setup steps are run) +COPY patch-broken-releases.py C:\patch-broken-releases.py +RUN python C:\patch-broken-releases.py C:\UnrealEngine %VERBOSE_OUTPUT% +{% endif %} + +# Run post-clone setup steps +# (Note that the `-no-cache` flag disables caching of dependency data in `.git/ue4-gitdeps`, saving disk space) +WORKDIR C:\UnrealEngine +RUN Setup.bat -no-cache {{ gitdependencies_args }} + +# Set the changelist number in Build.version to ensure our Build ID is generated correctly +ARG CHANGELIST +COPY set-changelist.py C:\set-changelist.py +RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version %CHANGELIST% diff --git a/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat b/src/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat similarity index 100% rename from ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat rename to src/ue4docker/dockerfiles/ue4-source/windows/git-credential-helper.bat diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py new file mode 100644 index 00000000..577e8525 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/windows/patch-broken-releases.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +import json +import sys +from os.path import join + + +def readFile(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +def writeFile(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +engineRoot = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" +versionDetails = json.loads( + readFile(join(engineRoot, "Engine", "Build", "Build.version")) +) + +if ( + versionDetails["MajorVersion"] == 5 + and versionDetails["MinorVersion"] == 1 + and versionDetails["PatchVersion"] == 0 +): + # Hack InstalledEngineFilters.xml with the changes from CL 23300641 + # (See: ) + buildFile = join(engineRoot, "Engine", "Build", "InstalledEngineFilters.xml") + buildXml = readFile(buildFile) + if "HoloLens.Automation.json" not in buildXml: + buildXml = buildXml.replace( + '', + '\n' + + " Engine\\Saved\\CsTools\\Engine\\Intermediate\\ScriptModules\\HoloLens.Automation.json\n", + ) + + writeFile(buildFile, buildXml) + + if verboseOutput: + print("PATCHED {}:\n\n{}".format(buildFile, buildFile), file=sys.stderr) + else: + print("PATCHED {}".format(buildFile), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py b/src/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py new file mode 100644 index 00000000..5ba8d924 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +import os, sys + + +def readFile(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +def writeFile(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +# Comment out the call to the UE4 prereqs installer in Setup.bat +PREREQ_CALL = "start /wait Engine\\Extras\\Redist\\en-us\\UE4PrereqSetup_x64.exe" +setupScript = sys.argv[1] +verboseOutput = len(sys.argv) > 2 and sys.argv[2] == "1" +code = readFile(setupScript) +code = code.replace( + "echo Installing prerequisites...", "echo (Skipping installation of prerequisites)" +) +code = code.replace(PREREQ_CALL, "@rem " + PREREQ_CALL) + +# Also comment out the version selector call, since we don't need shell integration +SELECTOR_CALL = ( + ".\\Engine\\Binaries\\Win64\\UnrealVersionSelector-Win64-Shipping.exe /register" +) +code = code.replace(SELECTOR_CALL, "@rem " + SELECTOR_CALL) + +# Add output so we can see when script execution is complete, and ensure `pause` is not called on error +code = code.replace("rem Done!", "echo Done!\r\nexit /b 0") +code = code.replace("pause", "@rem pause") +writeFile(setupScript, code) + +# Print the patched code to stderr for debug purposes +if verboseOutput == True: + print("PATCHED {}:\n\n{}".format(setupScript, code), file=sys.stderr) +else: + print("PATCHED {}".format(setupScript), file=sys.stderr) diff --git a/src/ue4docker/dockerfiles/ue4-source/windows/set-changelist.py b/src/ue4docker/dockerfiles/ue4-source/windows/set-changelist.py new file mode 100644 index 00000000..1a879e09 --- /dev/null +++ b/src/ue4docker/dockerfiles/ue4-source/windows/set-changelist.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +from os.path import dirname +from subprocess import run, PIPE +import json, re, sys + + +def readFile(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +def writeFile(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +# Determine whether a changelist override value was specified +changelistOverride = None +if len(sys.argv) > 2 and sys.argv[2] != "%CHANGELIST%": + # If the override was "auto" then attempt to retrieve the CL number from the git commit message + if sys.argv[2] == "auto": + # Retrieve the commit message from git + engineRoot = dirname(dirname(dirname(sys.argv[1]))) + commitMessage = run( + ["git", "log", "-n", "1", "--format=%s%n%b"], + cwd=engineRoot, + stdout=PIPE, + stderr=PIPE, + universal_newlines=True, + ).stdout.strip() + + # If the commit is a tagged engine release then it won't have a CL number, and using "auto" is user error + if re.fullmatch("[0-9\\.]+ release", commitMessage) is not None: + print( + "Error: you are attempting to automatically retrieve the CL number for a tagged Unreal Engine release.\n" + "For hotfix releases of the Unreal Engine, a CL override is not required and should not be specified.\n" + "For supported .0 releases of the Unreal Engine, ue4-docker ships with known CL numbers, so an override should not be necessary.", + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to extract the CL number from the commit message + match = re.search("\\[CL ([0-9]+) by .+ in .+ branch\\]", commitMessage) + if match is not None: + changelistOverride = int(match.group(1)) + else: + print( + "Error: failed to find a CL number in the git commit message! This was the commit message:\n\n" + + commitMessage, + file=sys.stderr, + ) + sys.exit(1) + + else: + changelistOverride = int(sys.argv[2]) + +# Update the `Changelist` field to reflect the override if it was supplied, or else the `CompatibleChangelist` field in our version file +versionFile = sys.argv[1] +details = json.loads(readFile(versionFile)) +details["Changelist"] = ( + changelistOverride + if changelistOverride is not None + else details["CompatibleChangelist"] +) +details["IsPromotedBuild"] = 1 +patchedJson = json.dumps(details, indent=4) +writeFile(versionFile, patchedJson) +print("PATCHED BUILD.VERSION:\n{}".format(patchedJson), file=sys.stderr) diff --git a/src/ue4docker/export.py b/src/ue4docker/export.py new file mode 100644 index 00000000..4f46be8f --- /dev/null +++ b/src/ue4docker/export.py @@ -0,0 +1,105 @@ +from .infrastructure import DockerUtils, GlobalConfiguration, PrettyPrinting +from .exports import * +import sys + + +def _notNone(items): + return len([i for i in items if i is not None]) == len(items) + + +def _extractArg(args, index): + return args[index] if len(args) > index else None + + +def _isHelpFlag(arg): + return (arg.strip("-") in ["h", "help"]) == True + + +def _stripHelpFlags(args): + return list([a for a in args if _isHelpFlag(a) == False]) + + +def export(): + # The components that can be exported + COMPONENTS = { + "installed": { + "function": exportInstalledBuild, + "description": "Exports an Installed Build of the Engine", + "image": GlobalConfiguration.resolveTag("ue4-full"), + "help": "Copies the Installed Build from a container to the host system.\nOnly supported under Linux for UE 4.21.0 and newer.", + }, + "packages": { + "function": exportPackages, + "description": "Exports conan-ue4cli wrapper packages", + "image": GlobalConfiguration.resolveTag("ue4-full"), + "help": "Runs a temporary conan server inside a container and uses it to export the\ngenerated conan-ue4cli wrapper packages.\n\n" + + 'Currently the only supported destination value is "cache", which exports\nthe packages to the Conan local cache on the host system.', + }, + } + + # Parse the supplied command-line arguments + stripped = _stripHelpFlags(sys.argv) + args = { + "help": len(stripped) < len(sys.argv), + "component": _extractArg(stripped, 1), + "tag": _extractArg(stripped, 2), + "destination": _extractArg(stripped, 3), + } + + # If a component name has been specified, verify that it is valid + if args["component"] is not None and args["component"] not in COMPONENTS: + print( + 'Error: unrecognised component "{}".'.format(args["component"]), + file=sys.stderr, + ) + sys.exit(1) + + # Determine if we are performing an export + if args["help"] == False and _notNone( + [args["component"], args["tag"], args["destination"]] + ): + # Determine if the user specified an image and a tag or just a tag + tag = args["tag"] + details = COMPONENTS[args["component"]] + requiredImage = "{}:{}".format(details["image"], tag) if ":" not in tag else tag + + # Verify that the required container image exists + if DockerUtils.exists(requiredImage) == False: + print( + 'Error: the specified container image "{}" does not exist.'.format( + requiredImage + ), + file=sys.stderr, + ) + sys.exit(1) + + # Attempt to perform the export + details["function"](requiredImage, args["destination"], stripped[4:]) + print("Export complete.") + + # Determine if we are displaying the help for a specific component + elif args["help"] == True and args["component"] is not None: + # Display the help for the component + component = sys.argv[1] + details = COMPONENTS[component] + print("{} export {}".format(sys.argv[0], component)) + print(details["description"] + "\n") + print("Exports from image: {}:TAG\n".format(details["image"])) + print(details["help"]) + + else: + # Print usage syntax + print("Usage: {} export COMPONENT TAG DESTINATION\n".format(sys.argv[0])) + print("Exports components from built container images to the host system\n") + print("Components:") + PrettyPrinting.printColumns( + [ + (component, COMPONENTS[component]["description"]) + for component in COMPONENTS + ] + ) + print( + "\nRun `{} export COMPONENT --help` for more information on a component.".format( + sys.argv[0] + ) + ) diff --git a/ue4docker/exports/__init__.py b/src/ue4docker/exports/__init__.py similarity index 100% rename from ue4docker/exports/__init__.py rename to src/ue4docker/exports/__init__.py diff --git a/src/ue4docker/exports/export_installed.py b/src/ue4docker/exports/export_installed.py new file mode 100644 index 00000000..a5484081 --- /dev/null +++ b/src/ue4docker/exports/export_installed.py @@ -0,0 +1,87 @@ +import tempfile + +from docker.models.containers import Container + +from ..infrastructure import DockerUtils, SubprocessUtils +import json, os, platform, shutil, subprocess, sys + + +def exportInstalledBuild(image, destination, extraArgs): + # Verify that the destination directory does not already exist + if os.path.exists(destination) == True: + print("Error: the destination directory already exists.", file=sys.stderr) + sys.exit(1) + + # Create a container from which we will copy files + container = DockerUtils.create(image) + + exit_code = 1 + try: + exit_code = doExportInstalledBuild(container, destination, extraArgs) + except Exception as e: + print("Error: failed to export Installed Build.", file=sys.stderr) + raise e + finally: + # Remove the container, irrespective of whether or not the export succeeded + container.remove() + + sys.exit(exit_code) + + +def doExportInstalledBuild(container: Container, destination: str, extraArgs) -> int: + if platform.system() == "Windows": + engineRoot = "C:/UnrealEngine" + else: + engineRoot = "/home/ue4/UnrealEngine" + + with tempfile.TemporaryDirectory() as tmpdir: + versionFilePath = os.path.join(tmpdir, "Build.version") + # Verify that the Installed Build in the specified image is at least 4.21.0 + subprocess.run( + [ + "docker", + "cp", + f"{container.name}:{engineRoot}/Engine/Build/Build.version", + versionFilePath, + ], + check=True, + ) + try: + with open(versionFilePath, "r") as versionFile: + version = json.load(versionFile) + if version["MajorVersion"] == 4 and version["MinorVersion"] < 21: + raise Exception() + except: + print( + "Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.", + file=sys.stderr, + ) + return 1 + + # Attempt to perform the export + print("Exporting to {}...".format(destination)) + subprocess.run( + ["docker", "cp", f"{container.name}:{engineRoot}", destination], check=True + ) + + # If the export succeeded, regenerate the linker symlinks on the host system + if platform.system() != "Windows": + print("Performing linker symlink fixup...") + subprocess.run( + [ + sys.executable, + os.path.join( + os.path.dirname(os.path.dirname(__file__)), + "dockerfiles", + "ue4-source", + "linux", + "linker-fixup.py", + ), + os.path.join( + destination, + "Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64", + ), + shutil.which("ld"), + ], + check=True, + ) diff --git a/src/ue4docker/exports/export_packages.py b/src/ue4docker/exports/export_packages.py new file mode 100644 index 00000000..b54a3df0 --- /dev/null +++ b/src/ue4docker/exports/export_packages.py @@ -0,0 +1,187 @@ +from ..infrastructure import DockerUtils, FilesystemUtils, Logger, SubprocessUtils +import docker, os, subprocess, sys, tempfile + +# The name we use for our temporary Conan remote +REMOTE_NAME = "_ue4docker_export_temp" + +# Our conan_server config file data +CONAN_SERVER_CONFIG = """ +[server] +jwt_secret: jwt_secret +jwt_expire_minutes: 120 +ssl_enabled: False +port: 9300 +public_port: 9300 +host_name: {} +authorize_timeout: 1800 +disk_storage_path: {} +disk_authorize_timeout: 1800 +updown_secret: updown_secret + +[write_permissions] +*/*@*/*: * + +[read_permissions] +*/*@*/*: * + +[users] +user: password +""" + + +def exportPackages(image, destination, extraArgs): + # Create our logger to generate coloured output on stderr + logger = Logger() + + # Verify that the destination is "cache" + if destination.lower() != "cache": + logger.error('Error: the only supported package export destination is "cache".') + sys.exit(1) + + # Verify that Conan is installed on the host + try: + SubprocessUtils.run(["conan", "--version"]) + except: + logger.error( + "Error: Conan must be installed on the host system to export packages." + ) + sys.exit(1) + + # Determine if the container image is a Windows image or a Linux image + imageOS = DockerUtils.listImages(image)[0].attrs["Os"] + + # Use the appropriate commands and paths for the container platform + cmdsAndPaths = { + "linux": { + "rootCommand": ["bash", "-c", "sleep infinity"], + "mkdirCommand": ["mkdir"], + "copyCommand": ["cp", "-f"], + "dataDir": "/home/ue4/.conan_server/data", + "configDir": "/home/ue4/.conan_server/", + "bindMount": "/hostdir/", + }, + "windows": { + "rootCommand": ["timeout", "/t", "99999", "/nobreak"], + "mkdirCommand": ["cmd", "/S", "/C", "mkdir"], + "copyCommand": ["xcopy", "/y"], + "dataDir": "C:\\Users\\ContainerAdministrator\\.conan_server\\data", + "configDir": "C:\\Users\\ContainerAdministrator\\.conan_server\\", + "bindMount": "C:\\hostdir\\", + }, + }[imageOS] + + # Create an auto-deleting temporary directory to hold our server config file + with tempfile.TemporaryDirectory() as tempDir: + # Progress output + print("Starting conan_server in a container...") + + # Start a container from which we will export packages, bind-mounting our temp directory + container = DockerUtils.start( + image, + cmdsAndPaths["rootCommand"], + ports={"9300/tcp": 9300}, + mounts=[docker.types.Mount(cmdsAndPaths["bindMount"], tempDir, "bind")], + stdin_open=imageOS == "windows", + tty=imageOS == "windows", + remove=True, + ) + + # Reload the container attributes from the Docker daemon to ensure the networking fields are populated + container.reload() + + # Under Linux we can simply access the container from the host over the loopback address, but this doesn't work under Windows + # (See ) + externalAddress = ( + "127.0.0.1" + if imageOS == "linux" + else container.attrs["NetworkSettings"]["Networks"]["nat"]["IPAddress"] + ) + + # Generate our server config file in the temp directory + FilesystemUtils.writeFile( + os.path.join(tempDir, "server.conf"), + CONAN_SERVER_CONFIG.format(externalAddress, cmdsAndPaths["dataDir"]), + ) + + # Keep track of the `conan_server` log output so we can display it in case of an error + serverOutput = None + + try: + # Copy the server config file to the expected location inside the container + DockerUtils.execMultiple( + container, + [ + cmdsAndPaths["mkdirCommand"] + [cmdsAndPaths["configDir"]], + cmdsAndPaths["copyCommand"] + + [ + cmdsAndPaths["bindMount"] + "server.conf", + cmdsAndPaths["configDir"], + ], + ], + ) + + # Start `conan_server` + serverOutput = DockerUtils.exec(container, ["conan_server"], stream=True) + + # Progress output + print("Uploading packages to the server...") + + # Upload all of the packages in the container's local cache to the server + DockerUtils.execMultiple( + container, + [ + ["conan", "remote", "add", "localhost", "http://127.0.0.1:9300"], + ["conan", "user", "user", "-r", "localhost", "-p", "password"], + ["conan", "upload", "*/4.*", "--all", "--confirm", "-r=localhost"], + ], + ) + + # Configure the server as a temporary remote on the host system + SubprocessUtils.run( + [ + "conan", + "remote", + "add", + REMOTE_NAME, + "http://{}:9300".format(externalAddress), + ] + ) + SubprocessUtils.run( + ["conan", "user", "user", "-r", REMOTE_NAME, "-p", "password"] + ) + + # Retrieve the list of packages that were uploaded to the server + packages = SubprocessUtils.extractLines( + SubprocessUtils.capture( + ["conan", "search", "-r", REMOTE_NAME, "*"] + ).stdout + ) + packages = [ + package for package in packages if "/" in package and "@" in package + ] + + # Download each package in turn + for package in packages: + print( + "Downloading package {} to host system local cache...".format( + package + ) + ) + SubprocessUtils.run(["conan", "download", "-r", REMOTE_NAME, package]) + + # Once we reach this point, everything has worked and we don't need to output any logs + serverOutput = None + + finally: + # Stop the container, irrespective of whether or not the export succeeded + print("Stopping conan_server...") + container.stop() + + # If something went wrong then output the logs from `conan_server` to assist in diagnosing the failure + if serverOutput is not None: + print("Log output from conan_server:") + for chunk in serverOutput: + logger.error(chunk.decode("utf-8")) + + # Remove the temporary remote if it was created successfully + SubprocessUtils.run(["conan", "remote", "remove", REMOTE_NAME], check=False) diff --git a/src/ue4docker/info.py b/src/ue4docker/info.py new file mode 100644 index 00000000..26ee0af1 --- /dev/null +++ b/src/ue4docker/info.py @@ -0,0 +1,108 @@ +import humanfriendly, platform, psutil, shutil, sys +from .version import __version__ +from .infrastructure import * + + +def _osName(dockerInfo): + if platform.system() == "Windows": + return WindowsUtils.systemString() + elif platform.system() == "Darwin": + return DarwinUtils.systemString() + else: + return "Linux ({}, {}{})".format( + dockerInfo["OperatingSystem"], + dockerInfo["KernelVersion"], + ", running under WSL" if WindowsUtils.isWSL() else "", + ) + + +def _formatSize(size): + return humanfriendly.format_size(size, binary=True) + + +def info(): + # Verify that Docker is installed + if DockerUtils.installed() == False: + print( + "Error: could not detect Docker version. Please ensure Docker is installed.", + file=sys.stderr, + ) + sys.exit(1) + + # Gather our information about the Docker daemon + dockerInfo = DockerUtils.info() + nvidiaDocker = platform.system() == "Linux" and "nvidia" in dockerInfo["Runtimes"] + maxSize = DockerUtils.maxsize() + rootDir = dockerInfo["DockerRootDir"] + + # If we are communicating with a Linux Docker daemon under Windows or macOS then we can't query the available disk space + canQueryDisk = dockerInfo["OSType"].lower() == platform.system().lower() + + # Gather our information about the host system + diskSpace = ( + _formatSize(shutil.disk_usage(rootDir).free) + if canQueryDisk == True + else "Unknown (typically means the Docker daemon is running in a Moby VM, e.g. Docker Desktop)" + ) + memPhysical = psutil.virtual_memory().total + memVirtual = psutil.swap_memory().total + cpuPhysical = psutil.cpu_count(False) + cpuLogical = psutil.cpu_count() + cpuModel = platform.processor() + + # Attempt to query PyPI to determine the latest version of ue4-docker + # (We ignore any errors here to ensure the `ue4-docker info` command still works without network access) + try: + latestVersion = GlobalConfiguration.getLatestVersion() + except: + latestVersion = None + + # Prepare our report items + items = [ + ( + "ue4-docker version", + "{}{}".format( + __version__, + ( + "" + if latestVersion is None + else " (latest available version is {})".format(latestVersion) + ), + ), + ), + ("Operating system", _osName(dockerInfo)), + ("Docker daemon version", dockerInfo["ServerVersion"]), + ("NVIDIA Docker supported", "Yes" if nvidiaDocker == True else "No"), + ( + "Maximum image size", + "{:.0f}GB".format(maxSize) if maxSize != -1 else "No limit detected", + ), + ("Available disk space", diskSpace), + ( + "Total system memory", + "{} physical, {} virtual".format( + _formatSize(memPhysical), _formatSize(memVirtual) + ), + ), + ( + "CPU", + "{} physical, {} logical ({})".format(cpuPhysical, cpuLogical, cpuModel), + ), + ] + + # Determine the longest item name so we can format our list in nice columns + longestName = max([len(i[0]) for i in items]) + minSpaces = 4 + + # Print our report + for item in items: + print( + "{}:{}{}".format( + item[0], " " * ((longestName + minSpaces) - len(item[0])), item[1] + ) + ) + + # Warn the user if they're using an older version of Docker that can't build or run UE 5.4 Linux images without config changes + if DockerUtils.isVersionWithoutIPV6Loopback(): + logger = Logger(prefix="") + logger.warning(DockerUtils.getIPV6WarningMessage()) diff --git a/src/ue4docker/infrastructure/BuildConfiguration.py b/src/ue4docker/infrastructure/BuildConfiguration.py new file mode 100644 index 00000000..200f1cf6 --- /dev/null +++ b/src/ue4docker/infrastructure/BuildConfiguration.py @@ -0,0 +1,760 @@ +import json +import platform +import random +from typing import Optional + +import humanfriendly +from packaging.version import Version, InvalidVersion + +from .DockerUtils import DockerUtils +from .WindowsUtils import WindowsUtils + +# The default Unreal Engine git repository +DEFAULT_GIT_REPO = "https://github.com/EpicGames/UnrealEngine.git" + +# The base images for Linux containers +LINUX_BASE_IMAGES = { + "opengl": "nvidia/opengl:1.0-glvnd-devel-{ubuntu}", + "cuda": "nvidia/cuda:{cuda}-devel-{ubuntu}", +} + +# The default ubuntu base to use +DEFAULT_LINUX_VERSION = "ubuntu22.04" + +# The default CUDA version to use when `--cuda` is specified without a value +DEFAULT_CUDA_VERSION = "12.2.0" + +# The default memory limit (in GB) under Windows +DEFAULT_MEMORY_LIMIT = 10.0 + +# The Perforce changelist numbers for each supported .0 release of the Unreal Engine +UNREAL_ENGINE_RELEASE_CHANGELISTS = { + "4.27.0": 17155196, + "5.0.0": 19505902, + "5.1.0": 23058290, + "5.2.0": 25360045, + "5.3.0": 27405482, + "5.4.0": 33043543, + "5.5.0": 37670630, + "5.6.0": 43139311, +} + + +class VisualStudio(object): + def __init__( + self, + name: str, + build_number: str, + supported_since: Version, + unsupported_since: Optional[Version], + pass_version_to_buildgraph: bool, + ): + self.name = name + self.build_number = build_number + self.supported_since = supported_since + self.unsupported_since = unsupported_since + self.pass_version_to_buildgraph = pass_version_to_buildgraph + + def __str__(self) -> str: + return self.name + + +DefaultVisualStudio = "2017" + +VisualStudios = { + "2017": VisualStudio( + name="2017", + build_number="15", + # We do not support versions older than 4.27 + supported_since=Version("4.27"), + unsupported_since=Version("5.0"), + pass_version_to_buildgraph=False, + ), + "2019": VisualStudio( + name="2019", + build_number="16", + supported_since=Version("4.27"), + unsupported_since=Version("5.4"), + pass_version_to_buildgraph=True, + ), + "2022": VisualStudio( + name="2022", + build_number="17", + supported_since=Version("5.0"), + unsupported_since=None, + pass_version_to_buildgraph=True, + ), +} + + +class ExcludedComponent(object): + """ + The different components that we support excluding from the built images + """ + + # Engine Derived Data Cache (DDC) + DDC = "ddc" + + # Engine debug symbols + Debug = "debug" + + # Template projects and samples + Templates = "templates" + + @staticmethod + def description(component): + """ + Returns a human-readable description of the specified component + """ + return { + ExcludedComponent.DDC: "Derived Data Cache (DDC)", + ExcludedComponent.Debug: "Debug symbols", + ExcludedComponent.Templates: "Template projects and samples", + }.get(component, "[Unknown component]") + + +class BuildConfiguration(object): + @staticmethod + def addArguments(parser): + """ + Registers our supported command-line arguments with the supplied argument parser + """ + parser.add_argument( + "release", + nargs="?", # aka "required = False", but that doesn't work in positionals + help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch (deprecated, use --ue-version instead)', + ) + parser.add_argument( + "--ue-version", + default=None, + help='UE4 release to build, in semver format (e.g. 4.27.0) or "custom" for a custom repo and branch', + ) + parser.add_argument( + "--linux", + action="store_true", + help="Build Linux container images under Windows", + ) + parser.add_argument( + "--rebuild", + action="store_true", + help="Rebuild images even if they already exist", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print `docker build` commands instead of running them", + ) + parser.add_argument( + "--no-minimal", + action="store_true", + help="Don't build the ue4-minimal image (deprecated, use --target instead)", + ) + parser.add_argument( + "--no-full", + action="store_true", + help="Don't build the ue4-full image (deprecated, use --target instead)", + ) + parser.add_argument( + "--no-cache", action="store_true", help="Disable Docker build cache" + ) + parser.add_argument( + "--target", + action="append", + help="Add a target to the build list. Valid targets are `build-prerequisites`, `source`, `engine`, `minimal`, `full`, and `all`. May be specified multiple times or comma-separated. Defaults to `all`.", + ) + parser.add_argument( + "--random-memory", + action="store_true", + help="Use a random memory limit for Windows containers", + ) + parser.add_argument( + "--docker-build-args", + action="append", + default=[], + help="Specify additional options for 'docker build' commands", + ) + parser.add_argument( + "--exclude", + action="append", + default=[], + choices=[ + ExcludedComponent.DDC, + ExcludedComponent.Debug, + ExcludedComponent.Templates, + ], + help="Exclude the specified component (can be specified multiple times to exclude multiple components)", + ) + parser.add_argument( + "--opt", + action="append", + default=[], + help="Set an advanced configuration option (can be specified multiple times to specify multiple options)", + ) + parser.add_argument( + "--cuda", + default=None, + metavar="VERSION", + help="Add CUDA support as well as OpenGL support when building Linux containers", + ) + parser.add_argument( + "--visual-studio", + default=DefaultVisualStudio, + choices=VisualStudios.keys(), + help="Specify Visual Studio Build Tools version to use for Windows containers", + ) + parser.add_argument( + "-username", + default=None, + help="Specify the username to use when cloning the git repository", + ) + parser.add_argument( + "-password", + default=None, + help="Specify access token or password to use when cloning the git repository", + ) + parser.add_argument( + "-repo", + default=None, + help='Set the custom git repository to clone when "custom" is specified as the release value', + ) + parser.add_argument( + "-branch", + default=None, + help='Set the custom branch/tag to clone when "custom" is specified as the release value', + ) + parser.add_argument( + "-isolation", + default=None, + help="Set the isolation mode to use for Windows containers (process or hyperv)", + ) + parser.add_argument( + "-basetag", + default=None if platform.system() == "Windows" else DEFAULT_LINUX_VERSION, + help="Operating system base image tag to use. For Linux this is the version of Ubuntu (default is ubuntu22.04). " + "For Windows this is the Windows Server Core base image tag (default is the host OS version)", + ) + parser.add_argument( + "-suffix", default="", help="Add a suffix to the tags of the built images" + ) + parser.add_argument( + "-m", + default=None, + help="Override the default memory limit under Windows (also overrides --random-memory)", + ) + parser.add_argument( + "-ue4cli", + default=None, + help="Override the default version of ue4cli installed in the ue4-full image", + ) + parser.add_argument( + "-conan-ue4cli", + default=None, + help="Override the default version of conan-ue4cli installed in the ue4-full image", + ) + parser.add_argument( + "-layout", + default=None, + help="Copy generated Dockerfiles to the specified directory and don't build the images", + ) + parser.add_argument( + "--combine", + action="store_true", + help="Combine generated Dockerfiles into a single multi-stage build Dockerfile", + ) + parser.add_argument( + "--monitor", + action="store_true", + help="Monitor resource usage during builds (useful for debugging)", + ) + parser.add_argument( + "-interval", + type=float, + default=20.0, + help="Sampling interval in seconds when resource monitoring has been enabled using --monitor (default is 20 seconds)", + ) + parser.add_argument( + "--ignore-blacklist", + action="store_true", + help="Run builds even on blacklisted versions of Windows (advanced use only)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output during builds (useful for debugging)", + ) + parser.add_argument( + "-changelist", + type=int, + default=None, + help="Set a specific changelist number in the Unreal Engine's Build.version file", + ) + parser.add_argument( + "--prerequisites-dockerfile", + default=None, + help="Specifies path to custom ue4-build-prerequisites dockerfile", + ) + + def __init__(self, parser, argv, logger): + """ + Creates a new build configuration based on the supplied arguments object + """ + + # If the user has specified `--cuda` without a version value, treat the value as an empty string + argv = [arg + "=" if arg == "--cuda" else arg for arg in argv] + + # Parse the supplied command-line arguments + self.args = parser.parse_args(argv) + self.changelist = self.args.changelist + + # Figure out what targets we have; this is needed to find out if we need --ue-version. + using_target_specifier_old = self.args.no_minimal or self.args.no_full + using_target_specifier_new = self.args.target is not None + + # If we specified nothing, it's the same as specifying `minimal` + if not using_target_specifier_old and not using_target_specifier_new: + self.args.target = ["minimal"] + elif using_target_specifier_old and not using_target_specifier_new: + # Convert these to the new style + logger.warning( + "Using deprecated `--no-*` target specifiers; recommend changing to `--target`", + False, + ) + + # no-minimal implies no-full + if self.args.no_minimal: + self.args.no_full = True + + # Change into target descriptors + self.args.target = [] + + if not self.args.no_full: + self.args.target += ["full"] + + if not self.args.no_minimal: + self.args.target += ["minimal"] + + # disabling these was never supported + self.args.target += ["source"] + self.args.target += ["build-prerequisites"] + + elif using_target_specifier_new and not using_target_specifier_old: + # these can be token-delimited, so let's just split them apart and then remerge them into one list + split = [item.split(",") for item in self.args.target] + self.args.target = [item for sublist in split for item in sublist] + + elif using_target_specifier_old and using_target_specifier_new: + # uhoh + raise RuntimeError( + "specified both `--target` and the old `--no-*` options; please use only `--target`!" + ) + + # Now that we have our options in `self.args.target`, evaluate our dependencies + # In a theoretical ideal world this should be code-driven; if you find yourself adding a lot more code to this, consider a redesign! + active_targets = set(self.args.target) + + # build-prereq -> source -> engine + # build-prereq -> source -> minimal -> full + + # We initialize these with all the options, with the intent that you should be accessing them directly and not checking for existence + # This is to avoid typos giving false-negatives; KeyError is reliable and tells you what you did wrong + self.buildTargets = { + "build-prerequisites": False, + "source": False, + "minimal": False, + "full": False, + } + + for target in active_targets: + if target != "all" and target not in self.buildTargets: + valid_options = sorted(self.buildTargets.keys()) + raise RuntimeError( + f"unknown build target '{target}', valid options are: all {' '.join(valid_options)}" + ) + + if "full" in active_targets or "all" in active_targets: + self.buildTargets["full"] = True + active_targets.add("minimal") + + if "minimal" in active_targets or "all" in active_targets: + self.buildTargets["minimal"] = True + active_targets.add("source") + + if "source" in active_targets or "all" in active_targets: + self.buildTargets["source"] = True + active_targets.add("build-prerequisites") + + if "build-prerequisites" in active_targets or "all" in active_targets: + self.buildTargets["build-prerequisites"] = True + + if not self.buildTargets["build-prerequisites"]: + raise RuntimeError( + "we're not building anything; this shouldn't even be possible, but is definitely not useful" + ) + + # See if the user specified both the old positional version option and the new ue-version option + if self.args.release is not None and self.args.ue_version is not None: + raise RuntimeError( + "specified both `--ue-version` and the old positional version option; please use only `--ue-version`!" + ) + + # For the sake of a simpler pull request, we use self.args.release as the canonical place for this data. + # If support for the old positional version option is removed, this should be fixed. + if self.args.ue_version is not None: + self.args.release = self.args.ue_version + + # We care about the version number only if we're building source + if self.buildTargets["source"]: + if self.args.release is None: + raise RuntimeError("missing `--ue-version` when building source") + + # Determine if we are building a custom version of UE4 rather than an official release + self.args.release = self.args.release.lower() + if self.args.release == "custom" or self.args.release.startswith("custom:"): + # Both a custom repository and a custom branch/tag must be specified + if self.args.repo is None or self.args.branch is None: + raise RuntimeError( + "both a repository and branch/tag must be specified when building a custom version of the Engine" + ) + + # Use the specified repository and branch/tag + customName = ( + self.args.release.split(":", 2)[1].strip() + if ":" in self.args.release + else "" + ) + self.release = customName if len(customName) > 0 else "custom" + self.repository = self.args.repo + self.branch = self.args.branch + self.custom = True + + else: + # Validate the specified version string + try: + ue4Version = Version(self.args.release) + if ue4Version.major not in [4, 5] or ue4Version.pre is not None: + raise Exception(f"unsupported engine version: {ue4Version}") + self.release = ( + f"{ue4Version.major}.{ue4Version.minor}.{ue4Version.micro}" + ) + except InvalidVersion: + raise RuntimeError( + 'invalid Unreal Engine release number "{}", full semver format required (e.g. "4.27.0")'.format( + self.args.release + ) + ) + + # Use the default repository and the release tag for the specified version + self.repository = DEFAULT_GIT_REPO + self.branch = "{}-release".format(self.release) + self.custom = False + + # If the user specified a .0 release of the Unreal Engine and did not specify a changelist override then + # use the official changelist number for that release to ensure consistency with Epic Games Launcher builds + # (This is necessary because .0 releases do not include a `CompatibleChangelist` value in Build.version) + if ( + self.changelist is None + and self.release in UNREAL_ENGINE_RELEASE_CHANGELISTS + ): + self.changelist = UNREAL_ENGINE_RELEASE_CHANGELISTS[self.release] + else: + # defaults needed by other parts of the codebase + self.custom = False + self.release = None + self.repository = None + self.branch = None + + # Store our common configuration settings + self.containerPlatform = ( + "windows" + if platform.system() == "Windows" and self.args.linux == False + else "linux" + ) + self.dryRun = self.args.dry_run + self.rebuild = self.args.rebuild + self.suffix = self.args.suffix + self.platformArgs = ["--no-cache"] if self.args.no_cache == True else [] + self.excludedComponents = set(self.args.exclude) + self.baseImage = None + self.prereqsTag = None + self.ignoreBlacklist = self.args.ignore_blacklist + self.verbose = self.args.verbose + self.layoutDir = self.args.layout + self.combine = self.args.combine + + # If the user specified custom version strings for ue4cli and/or conan-ue4cli, process them + self.ue4cliVersion = self._processPackageVersion("ue4cli", self.args.ue4cli) + self.conanUe4cliVersion = self._processPackageVersion( + "conan-ue4cli", self.args.conan_ue4cli + ) + + # Process any specified advanced configuration options (which we use directly as context values for the Jinja templating system) + self.opts = {} + for o in self.args.opt: + if "=" in o: + key, value = o.split("=", 1) + self.opts[key.replace("-", "_")] = self._processTemplateValue(value) + else: + self.opts[o.replace("-", "_")] = True + + # If we are generating Dockerfiles then generate them for all images that have not been explicitly excluded + if self.layoutDir is not None: + self.rebuild = True + + # If we are generating Dockerfiles and combining them then set the corresponding Jinja context value + if self.layoutDir is not None and self.combine == True: + self.opts["combine"] = True + + # If the user requested an option that is only compatible with generated Dockerfiles then ensure `-layout` was specified + if self.layoutDir is None and self.opts.get("source_mode", "git") != "git": + raise RuntimeError( + "the `-layout` flag must be used when specifying a non-default value for the `source_mode` option" + ) + if self.layoutDir is None and self.combine == True: + raise RuntimeError( + "the `-layout` flag must be used when specifying the `--combine` flag" + ) + + # We care about source_mode and credential_mode only if we're building source + if self.buildTargets["source"]: + # Verify that the value for `source_mode` is valid if specified + validSourceModes = ["git", "copy"] + if self.opts.get("source_mode", "git") not in validSourceModes: + raise RuntimeError( + "invalid value specified for the `source_mode` option, valid values are {}".format( + validSourceModes + ) + ) + + if "credential_mode" not in self.opts: + # On Linux, default to secrets mode that causes fewer issues with firewalls + self.opts["credential_mode"] = ( + "secrets" if self.containerPlatform == "linux" else "endpoint" + ) + + # Verify that the value for `credential_mode` is valid if specified + validCredentialModes = ( + ["endpoint", "secrets"] + if self.containerPlatform == "linux" + else ["endpoint"] + ) + if self.opts["credential_mode"] not in validCredentialModes: + raise RuntimeError( + "invalid value specified for the `credential_mode` option, valid values are {} when building {} containers".format( + validCredentialModes, self.containerPlatform.title() + ) + ) + + # Generate Jinja context values for keeping or excluding components + self.opts["excluded_components"] = { + "ddc": ExcludedComponent.DDC in self.excludedComponents, + "debug": ExcludedComponent.Debug in self.excludedComponents, + "templates": ExcludedComponent.Templates in self.excludedComponents, + } + + if "gitdependencies_args" not in self.opts: + self.opts["gitdependencies_args"] = ( + "--exclude=Android --exclude=Mac --exclude=Linux" + if self.containerPlatform == "windows" + else "--exclude=Android --exclude=Mac --exclude=Win32 --exclude=Win64" + ) + + # Warn user that they are in danger of Docker 20GB COPY bug + # Unfortunately, we don't have a cheap way to check whether user environment is affected + # See https://github.com/adamrehn/ue4-docker/issues/99 + if self.containerPlatform == "windows": + warn20GiB = False + if ExcludedComponent.Debug not in self.excludedComponents: + logger.warning("Warning: You didn't pass --exclude debug", False) + warn20GiB = True + if self.release and not self.custom and Version(self.release).major >= 5: + logger.warning("Warning: You're building Unreal Engine 5", False) + warn20GiB = True + + if warn20GiB: + logger.warning("Warning: You might hit Docker 20GiB COPY bug", False) + logger.warning( + "Warning: Make sure that `ue4-docker diagnostics 20gig` passes", + False, + ) + logger.warning( + "Warning: See https://github.com/adamrehn/ue4-docker/issues/99#issuecomment-1079702817 for details and workarounds", + False, + ) + + # If we're building Windows containers, generate our Windows-specific configuration settings + if self.containerPlatform == "windows": + self._generateWindowsConfig() + + # If we're building Linux containers, generate our Linux-specific configuration settings + if self.containerPlatform == "linux": + self._generateLinuxConfig() + + # If the user-specified suffix passed validation, prefix it with a dash + self.suffix = "-{}".format(self.suffix) if self.suffix != "" else "" + + def describeExcludedComponents(self): + """ + Returns a list of strings describing the components that will be excluded (if any.) + """ + return sorted( + [ + ExcludedComponent.description(component) + for component in self.excludedComponents + ] + ) + + def _generateWindowsConfig(self): + self.visualStudio = VisualStudios.get(self.args.visual_studio) + if self.visualStudio is None: + raise RuntimeError( + f"unknown Visual Studio version: {self.args.visual_studio}" + ) + + if self.release is not None and not self.custom: + # Check whether specified Unreal Engine release is compatible with specified Visual Studio + if ( + self.visualStudio.supported_since is not None + and Version(self.release) < self.visualStudio.supported_since + ): + raise RuntimeError( + f"specified version of Unreal Engine is too old for Visual Studio {self.visualStudio.name}" + ) + + if ( + self.visualStudio.unsupported_since is not None + and Version(self.release) >= self.visualStudio.unsupported_since + ): + raise RuntimeError( + "Visual Studio {} is too old for specified version of Unreal Engine".format( + self.visualStudio + ) + ) + + # See https://github.com/EpicGames/UnrealEngine/commit/72585138472785e2ee58aab9950a7260275ee2ac + # Note: We must not pass VS2019 arg for older UE4 versions that didn't have VS2019 variable in their build graph xml. + # Otherwise, UAT errors out with "Unknown argument: VS2019". + if self.visualStudio.pass_version_to_buildgraph: + self.opts["buildgraph_args"] = ( + self.opts.get("buildgraph_args", "") + + f" -set:VS{self.visualStudio.name}=true" + ) + + # Determine base tag for the Windows release of the host system + self.hostBasetag = WindowsUtils.getHostBaseTag() + + # Store the tag for the base Windows Server Core image + self.basetag = ( + self.args.basetag if self.args.basetag is not None else self.hostBasetag + ) + + if self.basetag is None: + raise RuntimeError( + "unable to determine Windows Server Core base image tag from host system. Specify it explicitly using -basetag command-line flag" + ) + + self.baseImage = "mcr.microsoft.com/windows/servercore:" + self.basetag + self.dllSrcImage = WindowsUtils.getDllSrcImage(self.basetag) + self.prereqsTag = self.basetag + "-vs" + self.visualStudio.name + + # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect + if self.args.isolation is not None: + self.isolation = self.args.isolation + else: + # If we are able to use process isolation mode then use it, otherwise use Hyper-V isolation mode + differentKernels = self.basetag != self.hostBasetag + dockerSupportsProcess = Version( + DockerUtils.version()["Version"] + ) >= Version("18.09.0") + if not differentKernels and dockerSupportsProcess: + self.isolation = "process" + else: + self.isolation = "hyperv" + + # Set the isolation mode Docker flag + self.platformArgs.append("--isolation=" + self.isolation) + + # If the user has explicitly specified a memory limit then use it, otherwise auto-detect + self.memLimit = None + if self.args.m is not None: + try: + self.memLimit = humanfriendly.parse_size(self.args.m) / ( + 1000 * 1000 * 1000 + ) + except: + raise RuntimeError('invalid memory limit "{}"'.format(self.args.m)) + else: + # Only specify a memory limit when using Hyper-V isolation mode, in order to override the 1GB default limit + # (Process isolation mode does not impose any memory limits by default) + if self.isolation == "hyperv": + self.memLimit = ( + DEFAULT_MEMORY_LIMIT + if self.args.random_memory == False + else random.uniform( + DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT + 2.0 + ) + ) + + # Set the memory limit Docker flag + if self.memLimit is not None: + self.platformArgs.extend(["-m", "{:.2f}GB".format(self.memLimit)]) + + def _generateLinuxConfig(self): + # Verify that any user-specified tag suffix does not collide with our base tags + if self.suffix.startswith("opengl") or self.suffix.startswith("cuda"): + raise RuntimeError('tag suffix cannot begin with "opengl" or "cuda".') + + # Determine if we are building CUDA-enabled container images + self.cuda = None + if self.args.cuda is not None: + # Verify that the specified CUDA version is valid + self.cuda = self.args.cuda if self.args.cuda != "" else DEFAULT_CUDA_VERSION + # Use the appropriate base image for the specified CUDA version + self.baseImage = LINUX_BASE_IMAGES["cuda"] + self.prereqsTag = "cuda{cuda}-{ubuntu}" + else: + self.baseImage = LINUX_BASE_IMAGES["opengl"] + self.prereqsTag = "opengl-{ubuntu}" + + self.baseImage = self.baseImage.format( + cuda=self.args.cuda, ubuntu=self.args.basetag + ) + self.prereqsTag = self.prereqsTag.format( + cuda=self.args.cuda, ubuntu=self.args.basetag + ) + + def _processPackageVersion(self, package, version): + # Leave the version value unmodified if a blank version was specified or a fully-qualified version was specified + # (e.g. package==X.X.X, package>=X.X.X, git+https://url/for/package/repo.git, etc.) + if version is None or "/" in version or version.lower().startswith(package): + return version + + # If a version specifier (e.g. ==X.X.X, >=X.X.X, etc.) was specified, prefix it with the package name + if "=" in version: + return package + version + + # If a raw version number was specified, prefix the package name and a strict equality specifier + return "{}=={}".format(package, version) + + def _processTemplateValue(self, value): + # If the value is a boolean (either raw or represented by zero or one) then parse it + if value.lower() in ["true", "1"]: + return True + elif value.lower() in ["false", "0"]: + return False + + # If the value is a JSON object or array then attempt to parse it + if (value.startswith("{") and value.endswith("}")) or ( + value.startswith("[") and value.endswith("]") + ): + try: + return json.loads(value) + except: + print( + 'Warning: could not parse option value "{}" as JSON, treating value as a string'.format( + value + ) + ) + + # Treat all other values as strings + return value diff --git a/src/ue4docker/infrastructure/ContainerUtils.py b/src/ue4docker/infrastructure/ContainerUtils.py new file mode 100644 index 00000000..14676c25 --- /dev/null +++ b/src/ue4docker/infrastructure/ContainerUtils.py @@ -0,0 +1,138 @@ +import contextlib +import io +import logging +import os +import shutil +import sys +import tempfile + +import docker +from docker.models.containers import Container + + +class ContainerUtils(object): + """ + Provides functionality related to Docker containers + """ + + @staticmethod + @contextlib.contextmanager + def automatically_stop(container: Container, timeout: int = 1): + """ + Context manager to automatically stop a container returned by `ContainerUtils.start_for_exec()` + """ + try: + yield container + finally: + logging.info("Stopping Docker container {}...".format(container.short_id)) + container.stop(timeout=timeout) + + @staticmethod + def copy_from_host( + container: Container, host_path: str, container_path: str + ) -> None: + """ + Copies a file or directory from the host system to a container returned by `ContainerUtils.start_for_exec()`. + + `host_path` is the absolute path to the file or directory on the host system. + + `container_path` is the absolute path to the directory in the container where the copied file(s) will be placed. + """ + + # If the host path denotes a file rather than a directory, copy it to a temporary directory + # (If the host path is a directory then we create a no-op context manager to use in our `with` statement below) + tempDir = contextlib.suppress() + if os.path.isfile(host_path): + tempDir = tempfile.TemporaryDirectory() + shutil.copy2( + host_path, os.path.join(tempDir.name, os.path.basename(host_path)) + ) + host_path = tempDir.name + + # Automatically delete the temporary directory if we created one + with tempDir: + # Create a temporary file to hold the .tar archive data + with tempfile.NamedTemporaryFile( + suffix=".tar", delete=False + ) as tempArchive: + # Add the data from the host system to the temporary archive + tempArchive.close() + archiveName = os.path.splitext(tempArchive.name)[0] + shutil.make_archive(archiveName, "tar", host_path) + + # Copy the data from the temporary archive to the container + with open(tempArchive.name, "rb") as archive: + container.put_archive(container_path, archive.read()) + + # Remove the temporary archive + os.unlink(tempArchive.name) + + @staticmethod + def exec(container: Container, command: [str], capture: bool = False, **kwargs): + """ + Executes a command in a container returned by `ContainerUtils.start_for_exec()` and streams or captures the output + """ + + # Determine if we are capturing the output or printing it + stdoutDest = io.StringIO() if capture else sys.stdout + stderrDest = io.StringIO() if capture else sys.stderr + + # Attempt to start the command + details = container.client.api.exec_create(container.id, command, **kwargs) + output = container.client.api.exec_start(details["Id"], stream=True, demux=True) + + # Stream the output + for chunk in output: + # Isolate the stdout and stderr chunks + stdout, stderr = chunk + + # Capture/print the stderr data if we have any + if stderr is not None: + print(stderr.decode("utf-8"), end="", flush=True, file=stderrDest) + + # Capture/print the stdout data if we have any + if stdout is not None: + print(stdout.decode("utf-8"), end="", flush=True, file=stdoutDest) + + # Determine if the command succeeded + capturedOutput = ( + (stdoutDest.getvalue(), stderrDest.getvalue()) if capture else None + ) + result = container.client.api.exec_inspect(details["Id"])["ExitCode"] + if result != 0: + container.stop() + raise RuntimeError( + "Failed to run command {} in container. Process returned exit code {} with output {}.".format( + command, + result, + capturedOutput if capture else "printed above", + ) + ) + + # If we captured the output then return it + return capturedOutput + + @staticmethod + def start_for_exec( + client: docker.DockerClient, image: str, platform: str, **kwargs + ) -> Container: + """ + Starts a container in a detached state using a command that will block indefinitely + and returns the container handle. The handle can then be used to execute commands + inside the container. The container will be removed automatically when it is stopped, + but it will need to be stopped manually by calling `ContainerUtils.stop()`. + """ + command = ( + ["timeout", "/t", "99999", "/nobreak"] + if platform == "windows" + else ["bash", "-c", "sleep infinity"] + ) + return client.containers.run( + image, + command, + stdin_open=platform == "windows", + tty=platform == "windows", + detach=True, + remove=True, + **kwargs, + ) diff --git a/src/ue4docker/infrastructure/CredentialEndpoint.py b/src/ue4docker/infrastructure/CredentialEndpoint.py new file mode 100644 index 00000000..6c9dcf66 --- /dev/null +++ b/src/ue4docker/infrastructure/CredentialEndpoint.py @@ -0,0 +1,97 @@ +import multiprocessing, secrets, time, urllib.parse +from .NetworkUtils import NetworkUtils +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import urlparse, parse_qs +from functools import partial + + +class CredentialRequestHandler(BaseHTTPRequestHandler): + def __init__(self, username: str, password: str, token: str, *args, **kwargs): + self.username = username + self.password = password + self.token = token + super().__init__(*args, **kwargs) + + def log_request(self, code: str = "-", size: str = "-") -> None: + # We do not want to log each and every incoming request + pass + + def do_POST(self): + query_components = parse_qs(urlparse(self.path).query) + + self.send_response(200) + self.end_headers() + + if "token" in query_components and query_components["token"][0] == self.token: + content_length = int(self.headers["Content-Length"]) + prompt = self.rfile.read(content_length).decode("utf-8") + + response = self.password if "Password for" in prompt else self.username + self.wfile.write(response.encode("utf-8")) + + +class CredentialEndpoint(object): + def __init__(self, username: str, password: str): + """ + Creates an endpoint manager for the supplied credentials + """ + + # Make sure neither our username or password are blank, since that can cause `git clone` to hang indefinitely + self.username = username if username is not None and len(username) > 0 else " " + self.password = password if password is not None and len(password) > 0 else " " + self.endpoint = None + + # Generate a security token to require when requesting credentials + self.token = secrets.token_hex(16) + + def args(self) -> [str]: + """ + Returns the Docker build arguments for creating containers that require Git credentials + """ + + # Resolve the IP address for the host system + hostAddress = NetworkUtils.hostIP() + + # Provide the host address and security token to the container + return [ + "--build-arg", + "HOST_ADDRESS_ARG=" + urllib.parse.quote_plus(hostAddress), + "--build-arg", + "HOST_TOKEN_ARG=" + urllib.parse.quote_plus(self.token), + ] + + def start(self) -> None: + """ + Starts the HTTP endpoint as a child process + """ + + # Create a child process to run the credential endpoint + self.endpoint = multiprocessing.Process( + target=CredentialEndpoint._endpoint, + args=(self.username, self.password, self.token), + ) + + # Spawn the child process and give the endpoint time to start + self.endpoint.start() + time.sleep(2) + + # Verify that the endpoint started correctly + if not self.endpoint.is_alive(): + raise RuntimeError("failed to start the credential endpoint") + + def stop(self) -> None: + """ + Stops the HTTP endpoint child process + """ + self.endpoint.terminate() + self.endpoint.join() + + @staticmethod + def _endpoint(username: str, password: str, token: str) -> None: + """ + Implements a HTTP endpoint to provide Git credentials to Docker containers + """ + handler = partial(CredentialRequestHandler, username, password, token) + + server = HTTPServer(("0.0.0.0", 9876), RequestHandlerClass=handler) + server.serve_forever() diff --git a/src/ue4docker/infrastructure/DarwinUtils.py b/src/ue4docker/infrastructure/DarwinUtils.py new file mode 100644 index 00000000..7fdf415a --- /dev/null +++ b/src/ue4docker/infrastructure/DarwinUtils.py @@ -0,0 +1,38 @@ +import platform + +from packaging.version import Version + + +class DarwinUtils(object): + @staticmethod + def minimumRequiredVersion() -> Version: + """ + Returns the minimum required version of macOS, which is 10.10.3 Yosemite + + (10.10.3 is the minimum required version for Docker for Mac, as per: + ) + """ + return Version("10.10.3") + + @staticmethod + def systemString(): + """ + Generates a human-readable version string for the macOS host system + """ + return "macOS {} (Kernel Version {})".format( + platform.mac_ver()[0], platform.uname().release + ) + + @staticmethod + def getMacOsVersion() -> Version: + """ + Returns the version number for the macOS host system + """ + return Version(platform.mac_ver()[0]) + + @staticmethod + def isSupportedMacOsVersion(): + """ + Verifies that the macOS host system meets our minimum version requirements + """ + return DarwinUtils.getMacOsVersion() >= DarwinUtils.minimumRequiredVersion() diff --git a/src/ue4docker/infrastructure/DockerUtils.py b/src/ue4docker/infrastructure/DockerUtils.py new file mode 100644 index 00000000..dd9310e5 --- /dev/null +++ b/src/ue4docker/infrastructure/DockerUtils.py @@ -0,0 +1,261 @@ +import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re, sys +from docker.models.containers import Container +from packaging.version import Version + +from .FilesystemUtils import FilesystemUtils + + +class DockerUtils(object): + @staticmethod + def installed(): + """ + Determines if Docker is installed + """ + try: + return (DockerUtils.version() is not None), None + except Exception as e: + logging.debug(str(e)) + return False, e + + @staticmethod + def version(): + """ + Retrieves the version information for the Docker daemon + """ + client = docker.from_env() + return client.version() + + @staticmethod + def info(): + """ + Retrieves the system information as produced by `docker info` + """ + client = docker.from_env() + return client.info() + + @staticmethod + def minimumVersionForIPV6(): + """ + Returns the minimum version of the Docker daemon that supports IPv6 by default + without requiring manual configuration by the user + """ + return Version("26.0.0") + + @staticmethod + def isVersionWithoutIPV6Loopback(): + """ + Determines if the version of the Docker daemon lacks support for using the + IPv6 loopback address [::1] when using its default network configuration + """ + dockerVersion = Version(DockerUtils.version()["Version"]) + return dockerVersion < DockerUtils.minimumVersionForIPV6() + + @staticmethod + def getIPV6WarningMessage(): + """ """ + return "\n".join( + [ + f"Warning: detected a Docker version older than {DockerUtils.minimumVersionForIPV6()}.", + "Older versions of Docker cannot build or run Linux images for Unreal Engine 5.4 or", + "newer unless the Docker daemon is explicitly configured to enable IPv6 support.", + "", + "To test whether IPv6 support is working, run the following diagnostic test:", + f"{sys.argv[0]} diagnostics ipv6", + "", + "For more details, see: https://github.com/adamrehn/ue4-docker/issues/357", + ] + ) + + @staticmethod + def exists(name): + """ + Determines if the specified image exists + """ + client = docker.from_env() + try: + image = client.images.get(name) + return True + except: + return False + + @staticmethod + def build(tags: [str], context: str, args: [str]) -> [str]: + """ + Returns the `docker build` command to build an image + """ + tagArgs = [["-t", tag] for tag in tags] + return ( + ["docker", "build"] + + list(itertools.chain.from_iterable(tagArgs)) + + [context] + + args + ) + + @staticmethod + def buildx(tags: [str], context: str, args: [str], secrets: [str]) -> [str]: + """ + Returns the `docker buildx` command to build an image with the BuildKit backend + """ + tagArgs = [["-t", tag] for tag in tags] + return ( + ["docker", "build"] + + list(itertools.chain.from_iterable(tagArgs)) + + [context] + + ["--progress=plain"] + + args + + list(itertools.chain.from_iterable([["--secret", s] for s in secrets])) + ) + + @staticmethod + def pull(image): + """ + Returns the `docker pull` command to pull an image from a remote registry + """ + return ["docker", "pull", image] + + @staticmethod + def start(image, command, **kwargs): + """ + Starts a container in a detached state and returns the container handle + """ + client = docker.from_env() + return client.containers.run(image, command, detach=True, **kwargs) + + @staticmethod + def create(image: str, **kwargs) -> Container: + """ + Creates a stopped container for specified image name and returns the container handle + """ + client = docker.from_env() + return client.containers.create(image, **kwargs) + + @staticmethod + def configFilePath(): + """ + Returns the path to the Docker daemon configuration file under Windows + """ + return "{}\\Docker\\config\\daemon.json".format(os.environ["ProgramData"]) + + @staticmethod + def getConfig(): + """ + Retrieves and parses the Docker daemon configuration file under Windows + """ + configPath = DockerUtils.configFilePath() + if os.path.exists(configPath) == True: + with open(configPath) as configFile: + return json.load(configFile) + + return {} + + @staticmethod + def setConfig(config): + """ + Writes new values to the Docker daemon configuration file under Windows + """ + configPath = DockerUtils.configFilePath() + with open(configPath, "w") as configFile: + configFile.write(json.dumps(config)) + + @staticmethod + def maxsize(): + """ + Determines the configured size limit (in GB) for Windows containers + """ + if platform.system() != "Windows": + return -1 + + config = DockerUtils.getConfig() + if "storage-opts" in config: + sizes = [ + opt.replace("size=", "") + for opt in config["storage-opts"] + if "size=" in opt + ] + if len(sizes) > 0: + return humanfriendly.parse_size(sizes[0]) / 1000000000 + + # The default limit on image size is 20GB + # (https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container-issues) + return 20.0 + + @staticmethod + def listImages(tagFilter=None, filters={}, all=False): + """ + Retrieves the details for each image matching the specified filters + """ + + # Retrieve the list of images matching the specified filters + client = docker.from_env() + images = client.images.list(filters=filters, all=all) + + # Apply our tag filter if one was specified + if tagFilter is not None: + images = [ + i + for i in images + if len(i.tags) > 0 and len(fnmatch.filter(i.tags, tagFilter)) > 0 + ] + + return images + + @staticmethod + def exec(container, command: [str], **kwargs): + """ + Executes a command in a container returned by `DockerUtils.start()` and returns the output + """ + result, output = container.exec_run(command, **kwargs) + if result is not None and result != 0: + container.stop() + raise RuntimeError( + "Failed to run command {} in container. Process returned exit code {} with output: {}".format( + command, result, output + ) + ) + + return output + + @staticmethod + def execMultiple(container, commands: [[str]], **kwargs): + """ + Executes multiple commands in a container returned by `DockerUtils.start()` + """ + for command in commands: + DockerUtils.exec(container, command, **kwargs) + + @staticmethod + def injectPostRunMessage(dockerfile, platform, messageLines): + """ + Injects the supplied message at the end of each RUN directive in the specified Dockerfile + """ + + # Generate the `echo` command for each line of the message + prefix = "echo." if platform == "windows" else "echo '" + suffix = "" if platform == "windows" else "'" + echoCommands = "".join( + [" && {}{}{}".format(prefix, line, suffix) for line in messageLines] + ) + + # Read the Dockerfile contents and convert all line endings to \n + contents = FilesystemUtils.readFile(dockerfile) + contents = contents.replace("\r\n", "\n") + + # Determine the escape character for the Dockerfile + escapeMatch = re.search("#[\\s]*escape[\\s]*=[\\s]*([^\n])\n", contents) + escape = escapeMatch[1] if escapeMatch is not None else "\\" + + # Identify each RUN directive in the Dockerfile + runMatches = re.finditer( + "^RUN(.+?[^{}])\n".format(re.escape(escape)), + contents, + re.DOTALL | re.MULTILINE, + ) + if runMatches is not None: + for match in runMatches: + # Append the `echo` commands to the directive + contents = contents.replace( + match[0], "RUN{}{}\n".format(match[1], echoCommands) + ) + + # Write the modified contents back to the Dockerfile + FilesystemUtils.writeFile(dockerfile, contents) diff --git a/src/ue4docker/infrastructure/FilesystemUtils.py b/src/ue4docker/infrastructure/FilesystemUtils.py new file mode 100644 index 00000000..c7a2d6ef --- /dev/null +++ b/src/ue4docker/infrastructure/FilesystemUtils.py @@ -0,0 +1,16 @@ +class FilesystemUtils(object): + @staticmethod + def readFile(filename): + """ + Reads data from a file + """ + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + @staticmethod + def writeFile(filename, data): + """ + Writes data to a file + """ + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) diff --git a/src/ue4docker/infrastructure/GlobalConfiguration.py b/src/ue4docker/infrastructure/GlobalConfiguration.py new file mode 100644 index 00000000..fbb31ac1 --- /dev/null +++ b/src/ue4docker/infrastructure/GlobalConfiguration.py @@ -0,0 +1,44 @@ +import os + +from packaging.version import Version +from urllib.request import urlopen +import json + +# The default namespace for our tagged container images +DEFAULT_TAG_NAMESPACE = "adamrehn" + + +class GlobalConfiguration(object): + """ + Manages access to the global configuration settings for ue4-docker itself + """ + + @staticmethod + def getLatestVersion(): + """ + Queries PyPI to determine the latest available release of ue4-docker + """ + with urlopen("https://pypi.org/pypi/ue4-docker/json") as url: + data = json.load(url) + releases = [Version(release) for release in data["releases"]] + return sorted(releases)[-1] + + @staticmethod + def getTagNamespace(): + """ + Returns the currently-configured namespace for container image tags + """ + return os.environ.get("UE4DOCKER_TAG_NAMESPACE", DEFAULT_TAG_NAMESPACE) + + @staticmethod + def resolveTag(tag): + """ + Resolves a Docker image tag with respect to our currently-configured namespace + """ + + # If the specified tag already includes a namespace, simply return it unmodified + return ( + tag + if "/" in tag + else "{}/{}".format(GlobalConfiguration.getTagNamespace(), tag) + ) diff --git a/src/ue4docker/infrastructure/ImageBuilder.py b/src/ue4docker/infrastructure/ImageBuilder.py new file mode 100644 index 00000000..374ede0a --- /dev/null +++ b/src/ue4docker/infrastructure/ImageBuilder.py @@ -0,0 +1,294 @@ +from typing import Dict, Optional + +from .DockerUtils import DockerUtils +from .FilesystemUtils import FilesystemUtils +from .GlobalConfiguration import GlobalConfiguration +import glob, humanfriendly, os, shutil, subprocess, tempfile, time +from os.path import basename, exists, join +from jinja2 import Environment + + +class ImageBuildParams(object): + def __init__( + self, dockerfile: str, context_dir: str, env: Optional[Dict[str, str]] = None + ): + self.dockerfile = dockerfile + self.context_dir = context_dir + self.env = env + + +class ImageBuilder(object): + def __init__( + self, + tempDir: str, + platform: str, + logger, + rebuild: bool = False, + dryRun: bool = False, + layoutDir: str = None, + templateContext: Dict[str, str] = None, + combine: bool = False, + ): + """ + Creates an ImageBuilder for the specified build parameters + """ + self.tempDir = tempDir + self.platform = platform + self.logger = logger + self.rebuild = rebuild + self.dryRun = dryRun + self.layoutDir = layoutDir + self.templateContext = templateContext if templateContext is not None else {} + self.combine = combine + + def get_built_image_context(self, name): + """ + Resolve the full path to the build context for the specified image + """ + return os.path.normpath( + os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "dockerfiles", + basename(name), + self.platform, + ) + ) + + def build_builtin_image( + self, + name: str, + tags: [str], + args: [str], + builtin_name: str = None, + secrets: Dict[str, str] = None, + ): + context_dir = self.get_built_image_context( + name if builtin_name is None else builtin_name + ) + return self.build( + name, tags, args, join(context_dir, "Dockerfile"), context_dir, secrets + ) + + def build( + self, + name: str, + tags: [str], + args: [str], + dockerfile_template: str, + context_dir: str, + secrets: Dict[str, str] = None, + ): + """ + Builds the specified image if it doesn't exist or if we're forcing a rebuild + """ + + workdir = join(self.tempDir, basename(name), self.platform) + os.makedirs(workdir, exist_ok=True) + + # Create a Jinja template environment and render the Dockerfile template + environment = Environment( + autoescape=False, trim_blocks=True, lstrip_blocks=True + ) + dockerfile = join(workdir, "Dockerfile") + + templateInstance = environment.from_string( + FilesystemUtils.readFile(dockerfile_template) + ) + rendered = templateInstance.render(self.templateContext) + + # Compress excess whitespace introduced during Jinja rendering and save the contents back to disk + # (Ensure that we still have a single trailing newline at the end of the Dockerfile) + while "\n\n\n" in rendered: + rendered = rendered.replace("\n\n\n", "\n\n") + rendered = rendered.strip("\n") + "\n" + FilesystemUtils.writeFile(dockerfile, rendered) + + # Inject our filesystem layer commit message after each RUN directive in the Dockerfile + DockerUtils.injectPostRunMessage( + dockerfile, + self.platform, + [ + "", + "RUN directive complete. Docker will now commit the filesystem layer to disk.", + "Note that for large filesystem layers this can take quite some time.", + "Performing filesystem layer commit...", + "", + ], + ) + + # When building Linux images, explicitly specify the target CPU architecture + archFlags = ["--platform", "linux/amd64"] if self.platform == "linux" else [] + + # Create a temporary directory to hold any files needed for the build + with tempfile.TemporaryDirectory() as tempDir: + # Determine whether we are building using `docker buildx` with build secrets + imageTags = self._formatTags(name, tags) + + if self.platform == "linux" and secrets is not None and len(secrets) > 0: + # Create temporary files to store the contents of each of our secrets + secretFlags = [] + for secret, contents in secrets.items(): + secretFile = join(tempDir, secret) + FilesystemUtils.writeFile(secretFile, contents) + secretFlags.append("id={},src={}".format(secret, secretFile)) + + # Generate the `docker buildx` command to use our build secrets + command = DockerUtils.buildx( + imageTags, context_dir, archFlags + args, secretFlags + ) + else: + command = DockerUtils.build(imageTags, context_dir, archFlags + args) + + command += ["--file", dockerfile] + + env = os.environ.copy() + if self.platform == "linux": + env["DOCKER_BUILDKIT"] = "1" + + # Build the image if it doesn't already exist + self._processImage( + imageTags[0], + name, + command, + "build", + "built", + ImageBuildParams(dockerfile, context_dir, env), + ) + + def pull(self, image: str) -> None: + """ + Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version + """ + self._processImage(image, None, DockerUtils.pull(image), "pull", "pulled") + + def willBuild(self, name: str, tags: [str]) -> bool: + """ + Determines if we will build the specified image, based on our build settings + """ + imageTags = self._formatTags(name, tags) + return self._willProcess(imageTags[0]) + + def _formatTags(self, name: str, tags: [str]): + """ + Generates the list of fully-qualified tags that we will use when building an image + """ + return [ + "{}:{}".format(GlobalConfiguration.resolveTag(name), tag) for tag in tags + ] + + def _willProcess(self, image: [str]) -> bool: + """ + Determines if we will build or pull the specified image, based on our build settings + """ + return self.rebuild or not DockerUtils.exists(image) + + def _processImage( + self, + image: str, + name: Optional[str], + command: [str], + actionPresentTense: str, + actionPastTense: str, + build_params: Optional[ImageBuildParams] = None, + ) -> None: + """ + Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) + """ + + # Determine if we are processing the image + if not self._willProcess(image): + self.logger.info( + 'Image "{}" exists and rebuild not requested, skipping {}.'.format( + image, actionPresentTense + ) + ) + return + + # Determine if we are running in "dry run" mode + self.logger.action( + '{}ing image "{}"...'.format(actionPresentTense.capitalize(), image) + ) + if self.dryRun: + print(command) + self.logger.action( + 'Completed dry run for image "{}".'.format(image), newline=False + ) + return + + # Determine if we're just copying the Dockerfile to an output directory + if self.layoutDir is not None: + # Determine whether we're performing a simple copy or combining generated Dockerfiles + if self.combine: + # Ensure the destination directory exists + dest = join(self.layoutDir, "combined") + self.logger.action( + 'Merging "{}" into "{}"...'.format(build_params.context_dir, dest), + newline=False, + ) + os.makedirs(dest, exist_ok=True) + + # Merge the source Dockerfile with any existing Dockerfile contents in the destination directory + # (Insert a single newline between merged file contents and ensure we have a single trailing newline) + sourceDockerfile = build_params.dockerfile + destDockerfile = join(dest, "Dockerfile") + dockerfileContents = ( + FilesystemUtils.readFile(destDockerfile) + if exists(destDockerfile) + else "" + ) + dockerfileContents = ( + dockerfileContents + + "\n" + + FilesystemUtils.readFile(sourceDockerfile) + ) + dockerfileContents = dockerfileContents.strip("\n") + "\n" + FilesystemUtils.writeFile(destDockerfile, dockerfileContents) + + # Copy any supplemental files from the source directory to the destination directory + # (Exclude any extraneous files which are not referenced in the Dockerfile contents) + for file in glob.glob(join(build_params.context_dir, "*.*")): + if basename(file) in dockerfileContents: + shutil.copy(file, join(dest, basename(file))) + + # Report our success + self.logger.action( + 'Merged Dockerfile for image "{}".'.format(image), newline=False + ) + + else: + # Copy the source directory to the destination + dest = join(self.layoutDir, basename(name)) + self.logger.action( + 'Copying "{}" to "{}"...'.format(build_params.context_dir, dest), + newline=False, + ) + shutil.copytree(build_params.context_dir, dest) + shutil.copy(build_params.dockerfile, dest) + self.logger.action( + 'Copied Dockerfile for image "{}".'.format(image), newline=False + ) + + return + + # Attempt to process the image using the supplied command + startTime = time.time() + exitCode = subprocess.call( + command, env=build_params.env if build_params else None + ) + endTime = time.time() + + # Determine if processing succeeded + if exitCode == 0: + self.logger.action( + '{} image "{}" in {}'.format( + actionPastTense.capitalize(), + image, + humanfriendly.format_timespan(endTime - startTime), + ), + newline=False, + ) + else: + raise RuntimeError( + 'failed to {} image "{}".'.format(actionPresentTense, image) + ) diff --git a/src/ue4docker/infrastructure/ImageCleaner.py b/src/ue4docker/infrastructure/ImageCleaner.py new file mode 100644 index 00000000..b29cccb6 --- /dev/null +++ b/src/ue4docker/infrastructure/ImageCleaner.py @@ -0,0 +1,27 @@ +from .DockerUtils import DockerUtils +import humanfriendly, os, subprocess, time + + +class ImageCleaner(object): + def __init__(self, logger): + self.logger = logger + + def clean(self, image, dryRun=False): + """ + Removes the specified image + """ + + # Determine if we are running in "dry run" mode + self.logger.action('Removing image "{}"...'.format(image)) + cleanCommand = ["docker", "rmi", image] + if dryRun == True: + print(cleanCommand) + else: + subprocess.call(cleanCommand) + + def cleanMultiple(self, images, dryRun=False): + """ + Removes all of the images in the supplied list + """ + for image in images: + self.clean(image, dryRun) diff --git a/src/ue4docker/infrastructure/Logger.py b/src/ue4docker/infrastructure/Logger.py new file mode 100644 index 00000000..758aafc4 --- /dev/null +++ b/src/ue4docker/infrastructure/Logger.py @@ -0,0 +1,39 @@ +from termcolor import colored +import colorama, sys + + +class Logger(object): + def __init__(self, prefix=""): + """ + Creates a logger that will print coloured output to stderr + """ + colorama.init() + self.prefix = prefix + + def action(self, output, newline=True): + """ + Prints information about an action that is being performed + """ + self._print("green", output, newline) + + def error(self, output, newline=False): + """ + Prints information about an error that has occurred + """ + self._print("red", output, newline) + + def info(self, output, newline=True): + """ + Prints information that does not pertain to an action or an error + """ + self._print("green", output, newline) + + def warning(self, output, newline=True): + """ + Prints a warning (something that is not an error as of today, but might break in future releases) + """ + self._print("yellow", output, newline) + + def _print(self, colour, output, newline): + whitespace = "\n" if newline == True else "" + print(colored(whitespace + self.prefix + output, color=colour), file=sys.stderr) diff --git a/src/ue4docker/infrastructure/NetworkUtils.py b/src/ue4docker/infrastructure/NetworkUtils.py new file mode 100644 index 00000000..210b775d --- /dev/null +++ b/src/ue4docker/infrastructure/NetworkUtils.py @@ -0,0 +1,19 @@ +import socket + + +class NetworkUtils(object): + @staticmethod + def hostIP(): + """ + Determines the IP address of the host + """ + # Code from + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + s.connect(("10.255.255.255", 1)) + IP = s.getsockname()[0] + except: + IP = "127.0.0.1" + finally: + s.close() + return IP diff --git a/src/ue4docker/infrastructure/PrettyPrinting.py b/src/ue4docker/infrastructure/PrettyPrinting.py new file mode 100644 index 00000000..2c1745d8 --- /dev/null +++ b/src/ue4docker/infrastructure/PrettyPrinting.py @@ -0,0 +1,14 @@ +class PrettyPrinting(object): + @staticmethod + def printColumns(pairs, indent=2, minSpaces=6): + """ + Prints a list of paired values in two nicely aligned columns + """ + + # Determine the length of the longest item in the left-hand column + longestName = max([len(pair[0]) for pair in pairs]) + + # Print the two columns + for pair in pairs: + whitespace = " " * ((longestName + minSpaces) - len(pair[0])) + print("{}{}{}{}".format(" " * indent, pair[0], whitespace, pair[1])) diff --git a/src/ue4docker/infrastructure/ResourceMonitor.py b/src/ue4docker/infrastructure/ResourceMonitor.py new file mode 100644 index 00000000..aa9bf270 --- /dev/null +++ b/src/ue4docker/infrastructure/ResourceMonitor.py @@ -0,0 +1,83 @@ +import datetime, humanfriendly, os, psutil, shutil, threading, time +from .DockerUtils import DockerUtils + + +class ResourceMonitor(threading.Thread): + def __init__(self, logger, interval): + """ + Creates a resource monitor with the specified configuration + """ + super().__init__() + self._logger = logger + self._interval = interval + self._lock = threading.Lock() + self._shouldStop = False + + def stop(self): + """ + Stops the resource monitor thread + """ + + # Set the flag to instruct the resource monitor loop to stop + with self._lock: + self._shouldStop = True + + # Wait for the resource monitor thread to complete + if self.is_alive() == True: + self.join() + + def run(self): + """ + The resource monitor loop itself + """ + + # Determine which filesystem the Docker daemon uses for storing its data directory + dockerInfo = DockerUtils.info() + rootDir = dockerInfo["DockerRootDir"] + + # If we cannot access the Docker data directory (e.g. when the daemon is in a Moby VM), don't report disk space + reportDisk = os.path.exists(rootDir) + + # Sample the CPU usage using an interval of 1 second the first time to prime the system + # (See: ) + psutil.cpu_percent(1.0) + + # Loop until asked to stop + while True: + # Check that the thread has not been asked to stop + with self._lock: + if self._shouldStop == True: + return + + # Format the timestamp for the current time in ISO 8601 format (albeit without the "T" separator) + isoTime = datetime.datetime.now().replace(microsecond=0).isoformat(" ") + + # We format data sizes using binary units (KiB, MiB, GiB, etc.) + formatSize = lambda size: humanfriendly.format_size( + size, binary=True, keep_width=True + ) + + # Format the current quantity of available disk space on the Docker data directory's filesystem + diskSpace = ( + formatSize(shutil.disk_usage(rootDir).free) + if reportDisk == True + else "Unknown" + ) + + # Format the current quantity of available system memory + physicalMemory = formatSize(psutil.virtual_memory().free) + virtualMemory = formatSize(psutil.swap_memory().free) + + # Format the current CPU usage levels + cpu = psutil.cpu_percent() + + # Report the current levels of our available resources + self._logger.info( + "[{}] [Available disk: {}] [Available memory: {} physical, {} virtual] [CPU usage: {:.2f}%]".format( + isoTime, diskSpace, physicalMemory, virtualMemory, cpu + ), + False, + ) + + # Sleep until the next sampling interval + time.sleep(self._interval) diff --git a/src/ue4docker/infrastructure/SubprocessUtils.py b/src/ue4docker/infrastructure/SubprocessUtils.py new file mode 100644 index 00000000..6dedf0e8 --- /dev/null +++ b/src/ue4docker/infrastructure/SubprocessUtils.py @@ -0,0 +1,51 @@ +import subprocess + + +class VerboseCalledProcessError(RuntimeError): + """' + A verbose wrapper for `subprocess.CalledProcessError` that prints stdout and stderr + """ + + def __init__(self, wrapped): + self.wrapped = wrapped + + def __str__(self): + return "{}\nstdout: {}\nstderr: {}".format( + self.wrapped, self.wrapped.output, self.wrapped.stderr + ) + + +class SubprocessUtils(object): + @staticmethod + def extractLines(output): + """ + Extracts the individual lines from the output of a child process + """ + return output.decode("utf-8").replace("\r\n", "\n").strip().split("\n") + + @staticmethod + def capture(command, check=True, **kwargs): + """ + Executes a child process and captures its output. + + If the child process fails and `check` is True then a verbose exception will be raised. + """ + try: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=check, + **kwargs, + ) + except subprocess.CalledProcessError as e: + raise VerboseCalledProcessError(e) from None + + @staticmethod + def run(command, check=True, **kwargs): + """ + Executes a child process. + + If the child process fails and `check` is True then a verbose exception will be raised. + """ + return SubprocessUtils.capture(command, check, **kwargs) diff --git a/src/ue4docker/infrastructure/WindowsUtils.py b/src/ue4docker/infrastructure/WindowsUtils.py new file mode 100644 index 00000000..7b47130a --- /dev/null +++ b/src/ue4docker/infrastructure/WindowsUtils.py @@ -0,0 +1,151 @@ +from .DockerUtils import DockerUtils +from packaging.version import Version +import os, platform, sys +from typing import Optional + +if platform.system() == "Windows": + import winreg + + +class WindowsUtils(object): + # The oldest Windows build we support + _minimumRequiredBuild = 17763 + + # This lookup table is based on the list of valid tags from + # and list of build-to-release mappings from the following pages: + # - https://docs.microsoft.com/en-us/windows/release-health/release-information + # - https://docs.microsoft.com/en-us/windows/release-health/windows11-release-information + _knownTagsByBuildNumber = { + 17763: "ltsc2019", + 18362: "1903", + 18363: "1909", + 19041: "2004", + 19042: "20H2", + 19043: "21H1", + 20348: "ltsc2022", + 22000: "ltsc2022", + } + + _knownTags = list(_knownTagsByBuildNumber.values()) + + # The list of Windows Server and Windows 10 host OS releases that are blacklisted due to critical bugs + # (See: ) + _blacklistedHosts = [18362, 18363] + + @staticmethod + def _getVersionRegKey(subkey: str) -> str: + """ + Retrieves the specified Windows version key from the registry + + @raises FileNotFoundError if registry key doesn't exist + """ + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, "SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion" + ) + value = winreg.QueryValueEx(key, subkey) + winreg.CloseKey(key) + return value[0] + + @staticmethod + def requiredSizeLimit() -> float: + """ + Returns the minimum required image size limit (in GB) for Windows containers + """ + return 800.0 + + @staticmethod + def minimumRequiredBuild() -> int: + """ + Returns the minimum required version of Windows 10 / Windows Server + """ + return WindowsUtils._minimumRequiredBuild + + @staticmethod + def systemString() -> str: + """ + Generates a verbose human-readable version string for the Windows host system + """ + return "{} (Build {}.{})".format( + WindowsUtils._getVersionRegKey("ProductName"), + WindowsUtils.getWindowsBuild(), + WindowsUtils._getVersionRegKey("UBR"), + ) + + @staticmethod + def getHostBaseTag() -> Optional[str]: + """ + Retrieves the tag for the Windows Server Core base image matching the host Windows system + """ + + hostBuild = WindowsUtils.getWindowsBuild() + + return WindowsUtils._knownTagsByBuildNumber.get(hostBuild) + + @staticmethod + def getWindowsBuild() -> int: + """ + Returns build number for the Windows host system + """ + return sys.getwindowsversion().build + + @staticmethod + def isBlacklistedWindowsHost() -> bool: + """ + Determines if host Windows version is one with bugs that make it unsuitable for use + (defaults to checking the host OS release if one is not specified) + """ + dockerVersion = Version(DockerUtils.version()["Version"]) + build = WindowsUtils.getWindowsBuild() + return build in WindowsUtils._blacklistedHosts and dockerVersion < Version( + "19.03.6" + ) + + @staticmethod + def isWindowsServer() -> bool: + """ + Determines if the Windows host system is Windows Server + """ + # TODO: Replace this with something more reliable + return "Windows Server" in WindowsUtils._getVersionRegKey("ProductName") + + @staticmethod + def isWSL() -> bool: + """ + Determines if the host system is Linux running under WSL + """ + return "WSL_DISTRO_NAME" in os.environ or "WSL_INTEROP" in os.environ + + @staticmethod + def getDllSrcImage(basetag: str) -> str: + """ + Returns Windows image that can be used as a source for DLLs missing from Windows Server Core base image + """ + # TODO: we also need to use Windows Server image when user specifies custom tags, like '10.0.20348.169' + image = { + "ltsc2022": "mcr.microsoft.com/windows/server", + }.get(basetag, "mcr.microsoft.com/windows") + + tag = { + "ltsc2019": "1809", + }.get(basetag, basetag) + + return f"{image}:{tag}" + + @staticmethod + def getKnownBaseTags() -> [str]: + """ + Returns the list of known tags for the Windows Server Core base image, in ascending chronological release order + """ + return WindowsUtils._knownTags + + @staticmethod + def isNewerBaseTag(older: str, newer: str) -> Optional[bool]: + """ + Determines if the base tag `newer` is chronologically newer than the base tag `older` + """ + try: + return WindowsUtils._knownTags.index(newer) > WindowsUtils._knownTags.index( + older + ) + except ValueError: + return None diff --git a/ue4docker/infrastructure/__init__.py b/src/ue4docker/infrastructure/__init__.py similarity index 86% rename from ue4docker/infrastructure/__init__.py rename to src/ue4docker/infrastructure/__init__.py index 600abb9f..2d1ac7fe 100644 --- a/ue4docker/infrastructure/__init__.py +++ b/src/ue4docker/infrastructure/__init__.py @@ -1,4 +1,5 @@ from .BuildConfiguration import BuildConfiguration +from .ContainerUtils import ContainerUtils from .CredentialEndpoint import CredentialEndpoint from .DarwinUtils import DarwinUtils from .DockerUtils import DockerUtils @@ -8,7 +9,7 @@ from .ImageCleaner import ImageCleaner from .Logger import Logger from .NetworkUtils import NetworkUtils -from .PackageUtils import PackageUtils from .PrettyPrinting import PrettyPrinting +from .ResourceMonitor import ResourceMonitor from .SubprocessUtils import SubprocessUtils from .WindowsUtils import WindowsUtils diff --git a/src/ue4docker/main.py b/src/ue4docker/main.py new file mode 100644 index 00000000..430a7779 --- /dev/null +++ b/src/ue4docker/main.py @@ -0,0 +1,123 @@ +from .infrastructure import ( + DarwinUtils, + DockerUtils, + Logger, + PrettyPrinting, + WindowsUtils, +) +from .build import build +from .clean import clean +from .diagnostics_cmd import diagnostics +from .export import export +from .info import info +from .setup_cmd import setup +from .test import test +from .version_cmd import version +import logging, os, platform, sys + + +def _exitWithError(err): + Logger().error(err) + sys.exit(1) + + +def main(): + # Configure verbose logging if the user requested it + # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) + if "-v" in sys.argv or "--verbose" in sys.argv: + # Enable verbose logging + logging.getLogger().setLevel(logging.DEBUG) + + # Filter out the verbose flag to avoid breaking commands that don't support it + if not (len(sys.argv) > 1 and sys.argv[1] in ["build"]): + sys.argv = list([arg for arg in sys.argv if arg not in ["-v", "--verbose"]]) + + # Verify that Docker is installed + installed, error = DockerUtils.installed() + if installed == False: + _exitWithError( + "Error: could not detect Docker daemon version. Please ensure Docker is installed.\n\nError details: {}".format( + error + ) + ) + + # Under Windows, verify that the host is a supported version + if platform.system() == "Windows": + host_build = WindowsUtils.getWindowsBuild() + min_build = WindowsUtils.minimumRequiredBuild() + if host_build < min_build: + _exitWithError( + "Error: the detected build of Windows ({}) is not supported. {} or newer is required.".format( + host_build, min_build + ) + ) + + # Under macOS, verify that the host is a supported version + if platform.system() == "Darwin" and DarwinUtils.isSupportedMacOsVersion() == False: + _exitWithError( + "Error: the detected version of macOS ({}) is not supported. macOS {} or newer is required.".format( + DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion() + ) + ) + + # Our supported commands + COMMANDS = { + "build": { + "function": build, + "description": "Builds container images for UE4", + }, + "clean": {"function": clean, "description": "Cleans built container images"}, + "diagnostics": { + "function": diagnostics, + "description": "Runs diagnostics to detect issues with the host system configuration", + }, + "export": { + "function": export, + "description": "Exports components from built container images to the host system", + }, + "info": { + "function": info, + "description": "Displays information about the host system and Docker daemon", + }, + "setup": { + "function": setup, + "description": "Automatically configures the host system where possible", + }, + "test": { + "function": test, + "description": "Runs tests to verify the correctness of built container images", + }, + "version": { + "function": version, + "description": "Prints the ue4-docker version number", + }, + } + + # Truncate argv[0] to just the command name without the full path + sys.argv[0] = os.path.basename(sys.argv[0]) + + # Determine if a command has been specified + if len(sys.argv) > 1: + # Verify that the specified command is valid + command = sys.argv[1] + if command not in COMMANDS: + print('Error: unrecognised command "{}".'.format(command), file=sys.stderr) + sys.exit(1) + + # Invoke the command + sys.argv = [sys.argv[0]] + sys.argv[2:] + COMMANDS[command]["function"]() + + else: + # Print usage syntax + print("Usage: {} COMMAND [OPTIONS]\n".format(sys.argv[0])) + print("Windows and Linux containers for Unreal Engine 4\n") + print("Commands:") + PrettyPrinting.printColumns( + [(command, COMMANDS[command]["description"]) for command in COMMANDS] + ) + print( + "\nRun `{} COMMAND --help` for more information on a command.".format( + sys.argv[0] + ) + ) diff --git a/src/ue4docker/setup_cmd.py b/src/ue4docker/setup_cmd.py new file mode 100644 index 00000000..37e8d5a1 --- /dev/null +++ b/src/ue4docker/setup_cmd.py @@ -0,0 +1,151 @@ +import docker, os, platform, requests, shutil, subprocess, sys +from .infrastructure import * + + +# Runs a command without displaying its output and returns the exit code +def _runSilent(command): + result = SubprocessUtils.capture(command, check=False) + return result.returncode + + +# Performs setup for Linux hosts +def _setupLinux(): + # Pull the latest version of the Alpine container image + alpineImage = "alpine:latest" + SubprocessUtils.capture(["docker", "pull", alpineImage]) + + # Start the credential endpoint with blank credentials + endpoint = CredentialEndpoint("", "") + endpoint.start() + + try: + # Run an Alpine container to see if we can access the host port for the credential endpoint + SubprocessUtils.capture( + [ + "docker", + "run", + "--rm", + alpineImage, + "wget", + "--timeout=1", + "--post-data=dummy", + "http://{}:9876".format(NetworkUtils.hostIP()), + ], + check=True, + ) + + # If we reach this point then the host port is accessible + print("No firewall configuration required.") + + except: + # The host port is blocked, so we need to perform firewall configuration + print("Creating firewall rule for credential endpoint...") + + # Create the firewall rule + subprocess.run( + ["iptables", "-I", "INPUT", "-p", "tcp", "--dport", "9876", "-j", "ACCEPT"], + check=True, + ) + + # Ensure the firewall rule persists after reboot + # (Requires the `iptables-persistent` service to be installed and running) + os.makedirs("/etc/iptables", exist_ok=True) + subprocess.run("iptables-save > /etc/iptables/rules.v4", shell=True, check=True) + + # Inform users of the `iptables-persistent` requirement + print( + "Firewall rule created. Note that the `iptables-persistent` service will need to" + ) + print("be installed for the rule to persist after the host system reboots.") + + finally: + # Stop the credential endpoint + endpoint.stop() + + +# Performs setup for Windows Server hosts +def _setupWindowsServer(): + # Check if we need to configure the maximum image size + requiredLimit = WindowsUtils.requiredSizeLimit() + if DockerUtils.maxsize() < requiredLimit: + # Attempt to stop the Docker daemon + print("Stopping the Docker daemon...") + subprocess.run(["sc.exe", "stop", "docker"], check=True) + + # Attempt to set the maximum image size + print("Setting maximum image size to {}GB...".format(requiredLimit)) + config = DockerUtils.getConfig() + sizeOpt = "size={}GB".format(requiredLimit) + if "storage-opts" in config: + config["storage-opts"] = list( + [ + o + for o in config["storage-opts"] + if o.lower().startswith("size=") == False + ] + ) + config["storage-opts"].append(sizeOpt) + else: + config["storage-opts"] = [sizeOpt] + DockerUtils.setConfig(config) + + # Attempt to start the Docker daemon + print("Starting the Docker daemon...") + subprocess.run(["sc.exe", "start", "docker"], check=True) + + else: + print("Maximum image size is already correctly configured.") + + # Determine if we need to configure Windows firewall + ruleName = "Open TCP port 9876 for ue4-docker credential endpoint" + ruleExists = ( + _runSilent( + [ + "netsh", + "advfirewall", + "firewall", + "show", + "rule", + "name={}".format(ruleName), + ] + ) + == 0 + ) + if ruleExists == False: + # Add a rule to ensure Windows firewall allows access to the credential helper from our containers + print("Creating firewall rule for credential endpoint...") + subprocess.run( + [ + "netsh", + "advfirewall", + "firewall", + "add", + "rule", + "name={}".format(ruleName), + "dir=in", + "action=allow", + "protocol=TCP", + "localport=9876", + ], + check=True, + ) + + else: + print("Firewall rule for credential endpoint is already configured.") + + +def setup(): + # We don't currently support auto-config for VM-based containers + if platform.system() == "Darwin" or ( + platform.system() == "Windows" and WindowsUtils.isWindowsServer() == False + ): + print( + "Manual configuration is required under Windows 10 and macOS. Automatic configuration is not available." + ) + return + + # Perform setup based on the host system type + if platform.system() == "Linux": + _setupLinux() + else: + _setupWindowsServer() diff --git a/src/ue4docker/test.py b/src/ue4docker/test.py new file mode 100644 index 00000000..1e63ad0d --- /dev/null +++ b/src/ue4docker/test.py @@ -0,0 +1,98 @@ +import ntpath +import os +import posixpath +import sys + +import docker +from docker.errors import ImageNotFound + +from .infrastructure import ( + ContainerUtils, + GlobalConfiguration, + Logger, +) + + +def test(): + # Create our logger to generate coloured output on stderr + logger = Logger(prefix="[{} test] ".format(sys.argv[0])) + + # Create our Docker API client + client = docker.from_env() + + # Check that an image tag has been specified + if len(sys.argv) > 1 and sys.argv[1].strip("-") not in ["h", "help"]: + # Verify that the specified container image exists + tag = sys.argv[1] + image_name = GlobalConfiguration.resolveTag( + "ue4-full:{}".format(tag) if ":" not in tag else tag + ) + + try: + image = client.images.get(image_name) + except ImageNotFound: + logger.error( + 'Error: the specified container image "{}" does not exist.'.format( + image_name + ) + ) + sys.exit(1) + + # Use process isolation mode when testing Windows containers, since running Hyper-V containers don't currently support manipulating the filesystem + platform = image.attrs["Os"] + isolation = "process" if platform == "windows" else None + + # Start a container to run our tests in, automatically stopping and removing the container when we finish + logger.action( + 'Starting a container using the "{}" image...'.format(image_name), False + ) + container = ContainerUtils.start_for_exec( + client, image_name, platform, isolation=isolation + ) + with ContainerUtils.automatically_stop(container): + # Create the workspace directory in the container + workspaceDir = ( + "C:\\workspace" if platform == "windows" else "/tmp/workspace" + ) + shell_prefix = ( + ["cmd", "/S", "/C"] if platform == "windows" else ["bash", "-c"] + ) + + ContainerUtils.exec( + container, + shell_prefix + ["mkdir " + workspaceDir], + ) + + # Copy our test scripts into the container + testDir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "tests") + ContainerUtils.copy_from_host(container, testDir, workspaceDir) + + # Create a harness to invoke individual tests + containerPath = ntpath if platform == "windows" else posixpath + pythonCommand = "python" if platform == "windows" else "python3" + + def runTest(script): + logger.action('Running test "{}"...'.format(script), False) + try: + ContainerUtils.exec( + container, + [pythonCommand, containerPath.join(workspaceDir, script)], + workdir=workspaceDir, + ) + logger.action('Passed test "{}"'.format(script), False) + except RuntimeError as e: + logger.error('Error: test "{}" failed!'.format(script)) + raise e from None + + # Run each of our tests in turn + runTest("build-and-package.py") + runTest("consume-external-deps.py") + + # If we've reached this point then all of the tests passed + logger.action("All tests passed.", False) + + else: + # Print usage syntax + print("Usage: {} test TAG".format(sys.argv[0])) + print("Runs tests to verify the correctness of built container images\n") + print("TAG should specify the tag of the ue4-full image to test.") diff --git a/src/ue4docker/tests/build-and-package.py b/src/ue4docker/tests/build-and-package.py new file mode 100644 index 00000000..82046684 --- /dev/null +++ b/src/ue4docker/tests/build-and-package.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import os, platform, subprocess, tempfile, ue4cli + + +# Runs a command, raising an error if it returns a nonzero exit code +def run(command, **kwargs): + print("[RUN COMMAND] {} {}".format(command, kwargs), flush=True) + return subprocess.run(command, check=True, **kwargs) + + +# Retrieve the short version string for the Engine +manager = ue4cli.UnrealManagerFactory.create() +version = manager.getEngineVersion("short") + +# Create an auto-deleting temporary directory to work in +with tempfile.TemporaryDirectory() as tempDir: + # Clone a simple C++ project and verify that we can build and package it + repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) + projectDir = os.path.join(tempDir, "BasicCxx") + run(["git", "clone", "--depth=1", repo, projectDir]) + run(["ue4", "package", "Shipping"], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == "Windows": + run(["del", "/f", "/s", "/q", os.path.join(projectDir, ".git")], shell=True) diff --git a/src/ue4docker/tests/consume-external-deps.py b/src/ue4docker/tests/consume-external-deps.py new file mode 100644 index 00000000..791c8d26 --- /dev/null +++ b/src/ue4docker/tests/consume-external-deps.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +import os, platform, subprocess, tempfile, ue4cli + + +# Reads data from a file +def read(filename): + with open(filename, "rb") as f: + return f.read().decode("utf-8") + + +# Runs a command, raising an error if it returns a nonzero exit code +def run(command, **kwargs): + print("[RUN COMMAND] {} {}".format(command, kwargs), flush=True) + return subprocess.run(command, check=True, **kwargs) + + +# Writes data to a file +def write(filename, data): + with open(filename, "wb") as f: + f.write(data.encode("utf-8")) + + +# Retrieve the short version string for the Engine +manager = ue4cli.UnrealManagerFactory.create() +version = manager.getEngineVersion("short") + +# Create an auto-deleting temporary directory to work in +with tempfile.TemporaryDirectory() as tempDir: + # Clone a simple C++ project + repo = "https://gitlab.com/ue4-test-projects/{}/BasicCxx.git".format(version) + projectDir = os.path.join(tempDir, "BasicCxx") + run(["git", "clone", "--depth=1", repo, projectDir]) + + # Generate a code module to wrap our external dependencies + sourceDir = os.path.join(projectDir, "Source") + run(["ue4", "conan", "boilerplate", "WrapperModule"], cwd=sourceDir) + + # Add the wrapper module as a dependency of the project's main source code module + rulesFile = os.path.join(sourceDir, "BasicCxx", "BasicCxx.Build.cs") + rules = read(rulesFile) + rules = rules.replace( + "PublicDependencyModuleNames.AddRange(new string[] {", + 'PublicDependencyModuleNames.AddRange(new string[] { "WrapperModule", ', + ) + write(rulesFile, rules) + + # Add some dependencies to the module's conanfile.py + moduleDir = os.path.join(sourceDir, "WrapperModule") + conanfile = os.path.join(moduleDir, "conanfile.py") + deps = read(conanfile) + deps = deps.replace("pass", 'self._requireUnreal("zlib/ue4@adamrehn/{}")') + write(conanfile, deps) + + # Verify that we can build the project with dynamically located dependencies + run(["ue4", "build"], cwd=projectDir) + run(["ue4", "clean"], cwd=projectDir) + + # Verify that we can build the project with precomputed dependency data + run(["ue4", "conan", "precompute", "host"], cwd=moduleDir) + run(["ue4", "build"], cwd=projectDir) + + # Forcibly delete the .git subdirectory under Windows to avoid permissions errors when deleting the temp directory + if platform.system() == "Windows": + run(["del", "/f", "/s", "/q", os.path.join(projectDir, ".git")], shell=True) diff --git a/src/ue4docker/version.py b/src/ue4docker/version.py new file mode 100644 index 00000000..33ec3a65 --- /dev/null +++ b/src/ue4docker/version.py @@ -0,0 +1,8 @@ +import sys + +if sys.version_info >= (3, 8): + from importlib import metadata +else: + import importlib_metadata as metadata + +__version__ = metadata.version("ue4-docker") diff --git a/ue4docker/version_cmd.py b/src/ue4docker/version_cmd.py similarity index 67% rename from ue4docker/version_cmd.py rename to src/ue4docker/version_cmd.py index 01cfc13b..e74c6800 100644 --- a/ue4docker/version_cmd.py +++ b/src/ue4docker/version_cmd.py @@ -1,4 +1,5 @@ from .version import __version__ + def version(): - print(__version__) + print(__version__) diff --git a/test-suite/credentials/.gitignore b/test-suite/credentials/.gitignore new file mode 100644 index 00000000..e45e01ad --- /dev/null +++ b/test-suite/credentials/.gitignore @@ -0,0 +1,2 @@ +username.txt +password.txt diff --git a/test-suite/test-ue-releases.py b/test-suite/test-ue-releases.py new file mode 100755 index 00000000..6ebfa410 --- /dev/null +++ b/test-suite/test-ue-releases.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import argparse, os, platform, subprocess, sys, traceback +from pathlib import Path + +try: + import colorama + from termcolor import colored +except: + print( + "Error: could not import colorama and termcolor! Make sure you install ue4-docker at least once before running the test suite." + ) + sys.exit(1) + + +class UERelease: + def __init__( + self, name: str, tag: str, repo: str, vsVersion: int, ubuntuVersion: str | None + ) -> None: + self.name = name + self.tag = tag + self.repo = repo + self.vsVersion = vsVersion + self.ubuntuVersion = ubuntuVersion + + +# Older releases have broken tags in the upstream Unreal Engine repository, so we use a fork with the updated `Commit.gitdeps.xml` files +UPSTREAM_REPO = "https://github.com/EpicGames/UnrealEngine.git" +COMMITDEPS_REPO = "https://github.com/adamrehn/UnrealEngine.git" + +# The list of Unreal Engine releases that are currently supported by ue4-docker +SUPPORTED_RELEASES = [ + UERelease("4.27", "4.27.2-fixed", COMMITDEPS_REPO, 2017, None), + UERelease("5.0", "5.0.3-fixed", COMMITDEPS_REPO, 2019, "20.04"), + UERelease("5.1", "5.1.1-fixed", COMMITDEPS_REPO, 2019, None), + UERelease("5.2", "5.2.1-release", UPSTREAM_REPO, 2022, None), + UERelease("5.3", "5.3.2-release", UPSTREAM_REPO, 2022, None), + UERelease("5.4", "5.4.4-release", UPSTREAM_REPO, 2022, None), + UERelease("5.5", "5.5.1-release", UPSTREAM_REPO, 2022, None), +] + + +# Logs a message with the specified colour, making it bold to distinguish it from `ue4-docker build` log output +def log(message: str, colour: str): + print(colored(message, color=colour, attrs=["bold"]), file=sys.stderr, flush=True) + + +# Logs a command and runs it +def run(dryRun: bool, command: str, **kwargs: dict) -> subprocess.CompletedProcess: + log(command, colour="green") + if not dryRun: + return subprocess.run(command, **{"check": True, **kwargs}) + else: + return subprocess.CompletedProcess(command, 0) + + +# Runs our tests for the specified Unreal Engine release +def testRelease( + release: UERelease, username: str, token: str, keepImages: bool, dryRun: bool +) -> None: + + # Pass the supplied credentials to the build process via environment variables + environment = { + **os.environ, + "UE4DOCKER_USERNAME": username, + "UE4DOCKER_PASSWORD": token, + } + + # Generate the command to build the ue4-minimal image (and its dependencies) for the specified Unreal Engine release + command = [ + sys.executable, + "-m", + "ue4docker", + "build", + "--ue-version", + f"custom:{release.name}", + "-repo", + release.repo, + "-branch", + release.tag, + "--target", + "minimal", + ] + + # Apply any platform-specific flags + if platform.system() == "Windows": + command += ["--visual-studio", release.vsVersion] + elif release.ubuntuVersion is not None: + command += ["-basetag", f"ubuntu{release.ubuntuVersion}"] + + # Attempt to run the build + run( + dryRun, + command, + env=environment, + ) + + # Unless requested otherwise, remove the built images to free up disk space + if not keepImages: + run( + dryRun, + [ + sys.executable, + "-m", + "ue4docker", + "clean", + "-tag", + release.name, + "--all", + "--prune", + "--dry-run", + ], + ) + + +if __name__ == "__main__": + + try: + # Initialise coloured log output under Windows + colorama.init() + + # Resolve the paths to our input directories + testDir = Path(__file__).parent + credentialsDir = testDir / "credentials" + repoRoot = testDir.parent + + # Parse our command-line arguments + parser = argparse.ArgumentParser() + parser.add_argument( + "--releases", + default=None, + help="Run tests for the specified Unreal Engine releases (comma-delimited, defaults to all supported releases)", + ) + parser.add_argument( + "--keep-images", + action="store_true", + help="Don't remove images after they are built (uses more disk space)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print commands instead of running them", + ) + args = parser.parse_args() + + # Parse and validate the specified list of releases + if args.releases is not None: + testQueue = [] + versions = args.releases.split(",") + for version in versions: + found = [r for r in SUPPORTED_RELEASES if r.name == version] + if len(found) == 1: + testQueue.append(found[0]) + else: + raise RuntimeError(f'unsupported Unreal Engine release "{version}"') + else: + testQueue = SUPPORTED_RELEASES + + # Read the GitHub username from the credentials directory + usernameFile = credentialsDir / "username.txt" + if usernameFile.exists(): + username = usernameFile.read_text("utf-8").strip() + else: + raise RuntimeError(f"place GitHub username in the file {str(usernameFile)}") + + # Read the GitHub Personal Access Token (PAT) from the credentials directory + tokenFile = credentialsDir / "password.txt" + if tokenFile.exists(): + token = tokenFile.read_text("utf-8").strip() + else: + raise RuntimeError( + f"place GitHub Personal Access Token (PAT) in the file {str(tokenFile)}" + ) + + # Ensure any local changes to ue4-docker are installed + run( + args.dry_run, + [sys.executable, "-m", "pip", "install", "--user", str(repoRoot)], + ) + + # Run the tests for each of the selected Unreal Engine releases + for release in testQueue: + testRelease(release, username, token, args.keep_images, args.dry_run) + + except Exception as e: + log(traceback.format_exc(), colour="red") diff --git a/ue4docker/__main__.py b/ue4docker/__main__.py deleted file mode 100644 index a6f4407d..00000000 --- a/ue4docker/__main__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .main import main -import os, sys - -if __name__ == '__main__': - - # Rewrite sys.argv[0] so our help prompts display the correct base command - interpreter = sys.executable if sys.executable not in [None, ''] else 'python3' - sys.argv[0] = '{} -m ue4docker'.format(os.path.basename(interpreter)) - main() diff --git a/ue4docker/build.py b/ue4docker/build.py deleted file mode 100644 index 90ca1eca..00000000 --- a/ue4docker/build.py +++ /dev/null @@ -1,218 +0,0 @@ -import argparse, getpass, humanfriendly, os, shutil, sys, tempfile, time -from .infrastructure import * -from os.path import join - -def _getCredential(args, name, envVar, promptFunc): - - # Check if the credential was specified via the command-line - if getattr(args, name, None) is not None: - print('Using {} specified via `-{}` command-line argument.'.format(name, name)) - return getattr(args, name) - - # Check if the credential was specified via an environment variable - if envVar in os.environ: - print('Using {} specified via {} environment variable.'.format(name, envVar)) - return os.environ[envVar] - - # Fall back to prompting the user for the value - return promptFunc() - -def _getUsername(args): - return _getCredential(args, 'username', 'UE4DOCKER_USERNAME', lambda: input("Username: ")) - -def _getPassword(args): - return _getCredential(args, 'password', 'UE4DOCKER_PASSWORD', lambda: getpass.getpass("Password: ")) - - -def build(): - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} build] '.format(sys.argv[0])) - - # Register our supported command-line arguments - parser = argparse.ArgumentParser(prog='{} build'.format(sys.argv[0])) - BuildConfiguration.addArguments(parser) - - # If no command-line arguments were supplied, display the help message and exit - if len(sys.argv) < 2: - parser.print_help() - sys.exit(0) - - # Parse the supplied command-line arguments - try: - config = BuildConfiguration(parser, sys.argv[1:]) - except RuntimeError as e: - logger.error('Error: {}'.format(e)) - sys.exit(1) - - # Verify that Docker is installed - if DockerUtils.installed() == False: - logger.error('Error: could not detect Docker version. Please ensure Docker is installed.') - sys.exit(1) - - # Create an auto-deleting temporary directory to hold our build context - with tempfile.TemporaryDirectory() as tempDir: - - # Copy our Dockerfiles to the temporary directory - contextOrig = join(os.path.dirname(os.path.abspath(__file__)), 'dockerfiles') - contextRoot = join(tempDir, 'dockerfiles') - shutil.copytree(contextOrig, contextRoot) - - # Create the builder instance to build the Docker images - builder = ImageBuilder(contextRoot, config.containerPlatform, logger, config.rebuild, config.dryRun) - - # Resolve our main set of tags for the generated images - mainTags = ['{}{}-{}'.format(config.release, config.suffix, config.prereqsTag), config.release + config.suffix] - - # Determine if we are building a custom version of UE4 - if config.custom == True: - logger.info('CUSTOM ENGINE BUILD:', False) - logger.info('Custom name: ' + config.release, False) - logger.info('Repository: ' + config.repository, False) - logger.info('Branch/tag: ' + config.branch + '\n', False) - - # Determine if we are building Windows or Linux containers - if config.containerPlatform == 'windows': - - # Provide the user with feedback so they are aware of the Windows-specific values being used - logger.info('WINDOWS CONTAINER SETTINGS', False) - logger.info('Isolation mode: {}'.format(config.isolation), False) - logger.info('Base OS image tag: {} (host OS is {})'.format(config.basetag, WindowsUtils.systemStringShort()), False) - logger.info('Memory limit: {}'.format('No limit' if config.memLimit is None else '{:.2f}GB'.format(config.memLimit)), False) - logger.info('Detected max image size: {:.0f}GB'.format(DockerUtils.maxsize()), False) - logger.info('Directory to copy DLLs from: {}\n'.format(config.dlldir), False) - - # Verify that the user is not attempting to build images with a newer kernel version than the host OS - if WindowsUtils.isNewerBaseTag(config.hostBasetag, config.basetag): - logger.error('Error: cannot build container images with a newer kernel version than that of the host OS!') - sys.exit(1) - - # Check if the user is building a different kernel version to the host OS but is still copying DLLs from System32 - differentKernels = WindowsUtils.isInsiderPreview() or config.basetag != config.hostBasetag - if config.pullPrerequisites == False and differentKernels == True and config.dlldir == config.defaultDllDir: - logger.error('Error: building images with a different kernel version than the host,', False) - logger.error('but a custom DLL directory has not specified via the `-dlldir=DIR` arg.', False) - logger.error('The DLL files will be the incorrect version and the container OS will', False) - logger.error('refuse to load them, preventing the built Engine from running correctly.', False) - sys.exit(1) - - # Attempt to copy the required DLL files from the host system if we are building the prerequisites image - if config.pullPrerequisites == False: - for dll in WindowsUtils.requiredHostDlls(config.basetag): - shutil.copy2(join(config.dlldir, dll), join(builder.context('ue4-build-prerequisites'), dll)) - - # Ensure the Docker daemon is configured correctly - requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit: - logger.error('SETUP REQUIRED:') - logger.error('The max image size for Windows containers must be set to at least {}GB.'.format(requiredLimit)) - logger.error('See the Microsoft documentation for configuration instructions:') - logger.error('https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container#step-4-expand-maximum-container-disk-size') - logger.error('Under Windows Server, the command `{} setup` can be used to automatically configure the system.'.format(sys.argv[0])) - sys.exit(1) - - elif config.containerPlatform == 'linux': - - # Determine if we are building CUDA-enabled container images - capabilities = 'CUDA {} + OpenGL'.format(config.cuda) if config.cuda is not None else 'OpenGL' - logger.info('LINUX CONTAINER SETTINGS', False) - logger.info('Building GPU-enabled images compatible with NVIDIA Docker ({} support).\n'.format(capabilities), False) - - # Report which Engine components are being excluded (if any) - logger.info('GENERAL SETTINGS', False) - if len(config.excludedComponents) > 0: - logger.info('Excluding the following Engine components:', False) - for component in config.describeExcludedComponents(): - logger.info('- {}'.format(component), False) - else: - logger.info('Not excluding any Engine components.', False) - - # Determine if we need to prompt for credentials - if config.dryRun == True: - - # Don't bother prompting the user for any credentials during a dry run - logger.info('Performing a dry run, `docker build` commands will be printed and not executed.', False) - username = '' - password = '' - - elif builder.willBuild('ue4-source', mainTags) == False: - - # Don't bother prompting the user for any credentials if we're not building the ue4-source image - logger.info('Not building the ue4-source image, no Git credentials required.', False) - username = '' - password = '' - - else: - - # Retrieve the Git username and password from the user when building the ue4-source image - print('\nRetrieving the Git credentials that will be used to clone the UE4 repo') - username = _getUsername(config.args) - password = _getPassword(config.args) - print() - - # Start the HTTP credential endpoint as a child process and wait for it to start - endpoint = CredentialEndpoint(username, password) - endpoint.start() - - try: - - # Keep track of our starting time - startTime = time.time() - - # Compute the build options for the UE4 build prerequisites image - # (This is the only image that does not use any user-supplied tag suffix, since the tag always reflects any customisations) - prereqsArgs = ['--build-arg', 'BASEIMAGE=' + config.baseImage] - if config.containerPlatform == 'windows': - prereqsArgs = prereqsArgs + ['--build-arg', 'HOST_VERSION=' + WindowsUtils.getWindowsBuild()] - - # Build or pull the UE4 build prerequisites image - if config.pullPrerequisites == True: - builder.pull('adamrehn/ue4-build-prerequisites:{}'.format(config.prereqsTag)) - else: - builder.build('adamrehn/ue4-build-prerequisites', [config.prereqsTag], config.platformArgs + prereqsArgs) - - # Build the UE4 source image - prereqConsumerArgs = ['--build-arg', 'PREREQS_TAG={}'.format(config.prereqsTag)] - ue4SourceArgs = prereqConsumerArgs + [ - '--build-arg', 'GIT_REPO={}'.format(config.repository), - '--build-arg', 'GIT_BRANCH={}'.format(config.branch) - ] - builder.build('ue4-source', mainTags, config.platformArgs + ue4SourceArgs + endpoint.args()) - - # Build the UE4 Engine source build image, unless requested otherwise by the user - ue4BuildArgs = prereqConsumerArgs + [ - '--build-arg', 'TAG={}'.format(mainTags[1]), - '--build-arg', 'NAMESPACE={}'.format(GlobalConfiguration.getTagNamespace()) - ] - if config.noEngine == False: - builder.build('ue4-engine', mainTags, config.platformArgs + ue4BuildArgs) - else: - logger.info('User specified `--no-engine`, skipping ue4-engine image build.') - - # Build the minimal UE4 CI image, unless requested otherwise by the user - buildUe4Minimal = config.noMinimal == False - if buildUe4Minimal == True: - builder.build('ue4-minimal', mainTags, config.platformArgs + config.exclusionFlags + ue4BuildArgs) - else: - logger.info('User specified `--no-minimal`, skipping ue4-minimal image build.') - - # Build the full UE4 CI image, unless requested otherwise by the user - buildUe4Full = buildUe4Minimal == True and config.noFull == False - if buildUe4Full == True: - builder.build('ue4-full', mainTags, config.platformArgs + ue4BuildArgs) - else: - logger.info('Not building ue4-minimal or user specified `--no-full`, skipping ue4-full image build.') - - # Report the total execution time - endTime = time.time() - logger.action('Total execution time: {}'.format(humanfriendly.format_timespan(endTime - startTime))) - - # Stop the HTTP server - endpoint.stop() - - except Exception as e: - - # One of the images failed to build - logger.error('Error: {}'.format(e)) - endpoint.stop() - sys.exit(1) diff --git a/ue4docker/clean.py b/ue4docker/clean.py deleted file mode 100644 index dcd19529..00000000 --- a/ue4docker/clean.py +++ /dev/null @@ -1,63 +0,0 @@ -import argparse, subprocess, sys -from .infrastructure import * - -def _isIntermediateImage(image): - sentinel = 'com.adamrehn.ue4-docker.sentinel' - labels = image.attrs['ContainerConfig']['Labels'] - return labels is not None and sentinel in labels - -def _cleanMatching(cleaner, filter, tag, dryRun): - tagSuffix = ':{}'.format(tag) if tag is not None else '*' - matching = DockerUtils.listImages(tagFilter = filter + tagSuffix) - cleaner.cleanMultiple([image.tags[0] for image in matching], dryRun) - -def clean(): - - # Create our logger to generate coloured output on stderr - logger = Logger(prefix='[{} clean] '.format(sys.argv[0])) - - # Our supported command-line arguments - parser = argparse.ArgumentParser( - prog='{} clean'.format(sys.argv[0]), - description = - 'Cleans built container images. ' + - 'By default, only dangling intermediate images leftover from ue4-docker multi-stage builds are removed.' - ) - parser.add_argument('-tag', default=None, help='Only clean images with the specified tag') - parser.add_argument('--source', action='store_true', help='Clean ue4-source images') - parser.add_argument('--engine', action='store_true', help='Clean ue4-engine images') - parser.add_argument('--all', action='store_true', help='Clean all ue4-docker images') - parser.add_argument('--dry-run', action='store_true', help='Print docker commands instead of running them') - parser.add_argument('--prune', action='store_true', help='Run `docker system prune` after cleaning') - - # Parse the supplied command-line arguments - args = parser.parse_args() - - # Create our image cleaner - cleaner = ImageCleaner(logger) - - # Remove any intermediate images leftover from our multi-stage builds - dangling = DockerUtils.listImages(filters = {'dangling': True}) - dangling = [image.id for image in dangling if _isIntermediateImage(image)] - cleaner.cleanMultiple(dangling, args.dry_run) - - # If requested, remove ue4-source images - if args.source == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-source'), args.tag, args.dry_run) - - # If requested, remove ue4-engine images - if args.engine == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-engine'), args.tag, args.dry_run) - - # If requested, remove everything - if args.all == True: - _cleanMatching(cleaner, GlobalConfiguration.resolveTag('ue4-*'), args.tag, args.dry_run) - - # If requested, run `docker system prune` - if args.prune == True: - logger.action('Running `docker system prune`...') - pruneCommand = ['docker', 'system', 'prune', '-f'] - if args.dry_run == True: - print(pruneCommand) - else: - subprocess.call(pruneCommand) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile deleted file mode 100644 index 3a5254e6..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/linux/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} - -# Add a sentinel label so we can easily identify all derived images, including intermediate images -LABEL com.adamrehn.ue4-docker.sentinel="1" - -# Disable interactive prompts during package installation -ENV DEBIAN_FRONTEND=noninteractive - -# Install our build prerequisites -RUN \ - apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - ca-certificates \ - curl \ - git \ - python3 \ - python3-pip \ - shared-mime-info \ - tzdata \ - unzip \ - xdg-user-dirs \ - zip && \ - rm -rf /var/lib/apt/lists/* - -# Since UE4 refuses to build or run as the root user under Linux, create a non-root user -RUN \ - useradd --create-home --home /home/ue4 --shell /bin/bash --uid 1000 ue4 && \ - usermod -a -G audio,video ue4 -USER ue4 diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile deleted file mode 100644 index fc62d1e9..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/Dockerfile +++ /dev/null @@ -1,57 +0,0 @@ -# escape=` -ARG BASEIMAGE -FROM ${BASEIMAGE} AS dlls -SHELL ["cmd", "/S", "/C"] - -# Include our sentinel so `ue4-docker clean` can find this intermediate image -LABEL com.adamrehn.ue4-docker.sentinel="1" - -# Create a directory in which to gather the DLL files we need -RUN mkdir C:\GatheredDlls - -# Install 7-Zip, curl, and Python using Chocolatey -# (Note that these need to be separate RUN directives for `choco` to work) -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" -RUN choco install -y 7zip curl python - -# Copy the required DirectSound/DirectDraw and OpenGL DLL files from the host system (since these ship with Windows and don't have installers) -COPY *.dll C:\GatheredDlls\ - -# Verify that the DLL files copied from the host can be loaded by the container OS -ARG HOST_VERSION -RUN pip install pypiwin32 -COPY copy.py verify-host-dlls.py C:\ -RUN C:\copy.py "C:\GatheredDlls\*.dll" C:\Windows\System32\ -RUN python C:\verify-host-dlls.py %HOST_VERSION% C:\GatheredDlls - -# Gather the required DirectX runtime files, since Windows Server Core does not include them -RUN curl --progress -L "https://download.microsoft.com/download/8/4/A/84A35BF1-DAFE-4AE8-82AF-AD2AE20B6B14/directx_Jun2010_redist.exe" --output %TEMP%\directx_redist.exe -RUN start /wait %TEMP%\directx_redist.exe /Q /T:%TEMP% && ` - expand %TEMP%\APR2007_xinput_x64.cab -F:xinput1_3.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_D3DCompiler_43_x64.cab -F:D3DCompiler_43.dll C:\GatheredDlls\ && ` - expand %TEMP%\Feb2010_X3DAudio_x64.cab -F:X3DAudio1_7.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAPOFX1_5.dll C:\GatheredDlls\ && ` - expand %TEMP%\Jun2010_XAudio_x64.cab -F:XAudio2_7.dll C:\GatheredDlls\ - -# Gather the Vulkan runtime library -RUN curl --progress -L "https://sdk.lunarg.com/sdk/download/1.1.73.0/windows/VulkanSDK-1.1.73.0-Installer.exe?Human=true" --output %TEMP%\VulkanSDK.exe -RUN 7z e %TEMP%\VulkanSDK.exe -oC:\GatheredDlls -y RunTimeInstaller\x64\vulkan-1.dll - -# Gather pdbcopy.exe (needed for creating an Installed Build of the Engine) -RUN choco install -y windbg -RUN C:\copy.py "C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\pdbcopy.exe" C:\GatheredDlls\ - -# Copy our gathered DLLs (and pdbcopy.exe) into a clean image to reduce image size -FROM ${BASEIMAGE} -SHELL ["cmd", "/S", "/C"] -COPY --from=dlls C:\GatheredDlls\ C:\Windows\System32\ - -# Add a sentinel label so we can easily identify all derived images, including intermediate images -LABEL com.adamrehn.ue4-docker.sentinel="1" - -# Install Chocolatey -RUN powershell -NoProfile -ExecutionPolicy Bypass -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" - -# Install the rest of our build prerequisites and clean up afterwards to minimise image size -COPY buildtools-exitcode.py copy.py copy-pdbcopy.py install-prerequisites.bat C:\ -RUN C:\install-prerequisites.bat diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py deleted file mode 100644 index 70ef15eb..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/buildtools-exitcode.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -import sys - -# Propagate any exit codes from the VS Build Tools installer except for 3010 -code = int(sys.argv[1]) -code = 0 if code == 3010 else code -sys.exit(code) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py deleted file mode 100644 index f1c2d812..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy-pdbcopy.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -import os, shutil - -# Copy pdbcopy.exe to the expected location for both newer Engine versions, -# as well as UE4 versions prior to the UE-51362 fix (https://issues.unrealengine.com/issue/UE-51362) -pdbcopyExe = 'C:\\Windows\\System32\\pdbcopy.exe' -destDirTemplate = 'C:\\Program Files (x86)\\MSBuild\\Microsoft\\VisualStudio\\v{}\\AppxPackage' -destDirs = [ - 'C:\\Program Files (x86)\\Windows Kits\\10\\Debuggers\\x64', - destDirTemplate.format('12.0'), - destDirTemplate.format('14.0') -] -for destDir in destDirs: - destFile = os.path.join(destDir, 'pdbcopy.exe') - os.makedirs(destDir, exist_ok=True) - shutil.copy2(pdbcopyExe, destFile) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py deleted file mode 100644 index 0f61a829..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/copy.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -# This script is used to avoid issues with `xcopy.exe` under Windows Server 2016 (https://github.com/moby/moby/issues/38425) -import glob, os, shutil, sys - -# If the destination is an existing directory then expand wildcards in the source -destination = sys.argv[2] -if os.path.isdir(destination) == True: - sources = glob.glob(sys.argv[1]) -else: - sources = [sys.argv[1]] - -# Copy each of our source files/directories -for source in sources: - if os.path.isdir(source): - dest = os.path.join(destination, os.path.basename(source)) - shutil.copytree(source, dest) - else: - shutil.copy2(source, destination) - print('Copied {} to {}.'.format(source, destination), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat deleted file mode 100644 index 19cdb415..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/install-prerequisites.bat +++ /dev/null @@ -1,44 +0,0 @@ -@rem Install the chocolatey packages we need -choco install -y git --params "'/GitOnlyOnPath /NoAutoCrlf /WindowsTerminal /NoShellIntegration /NoCredentialManager'" || goto :error -choco install -y curl python vcredist-all || goto :error - -@rem Reload our environment variables from the registry so the `git` command works -call refreshenv -@echo on - -@rem Forcibly disable the git credential manager -git config --system credential.helper "" || goto :error - -@rem Install the Visual Studio 2017 Build Tools workloads and components we need, excluding components with known issues in containers -@rem (Note that we use the Visual Studio 2019 installer here because the old installer now breaks, but explicitly install the VS2017 Build Tools) -curl --progress -L "https://aka.ms/vs/16/release/vs_buildtools.exe" --output %TEMP%\vs_buildtools.exe || goto :error -%TEMP%\vs_buildtools.exe --quiet --wait --norestart --nocache ^ - --installPath C:\BuildTools ^ - --channelUri "https://aka.ms/vs/15/release/channel" ^ - --installChannelUri "https://aka.ms/vs/15/release/channel" ^ - --channelId VisualStudio.15.Release ^ - --productId Microsoft.VisualStudio.Product.BuildTools ^ - --add Microsoft.VisualStudio.Workload.VCTools;includeRecommended ^ - --add Microsoft.VisualStudio.Workload.ManagedDesktopBuildTools;includeRecommended ^ - --add Microsoft.VisualStudio.Workload.UniversalBuildTools ^ - --add Microsoft.VisualStudio.Workload.NetCoreBuildTools ^ - --add Microsoft.VisualStudio.Workload.MSBuildTools ^ - --add Microsoft.VisualStudio.Component.NuGet -python C:\buildtools-exitcode.py %ERRORLEVEL% || goto :error - -@rem Copy pdbcopy.exe to the expected location(s) -python C:\copy-pdbcopy.py || goto :error - -@rem Clean up any temp files generated during prerequisite installation -rmdir /S /Q \\?\%TEMP% -mkdir %TEMP% - -@rem Display a human-readable completion message -@echo off -@echo Finished installing build prerequisites and cleaning up temporary files. -goto :EOF - -@rem If any of our essential commands fail, propagate the error code -:error -@echo off -exit /b %ERRORLEVEL% diff --git a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py b/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py deleted file mode 100644 index a565f7c3..00000000 --- a/ue4docker/dockerfiles/ue4-build-prerequisites/windows/verify-host-dlls.py +++ /dev/null @@ -1,54 +0,0 @@ -import glob, os, platform, sys, win32api, winreg - -# Adapted from the code in this SO answer: -def getDllVersion(dllPath): - info = win32api.GetFileVersionInfo(dllPath, '\\') - return '{:d}.{:d}.{:d}.{:d}'.format( - info['FileVersionMS'] // 65536, - info['FileVersionMS'] % 65536, - info['FileVersionLS'] // 65536, - info['FileVersionLS'] % 65536 - ) - -def getVersionRegKey(subkey): - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') - value = winreg.QueryValueEx(key, subkey) - winreg.CloseKey(key) - return value[0] - -def getOsVersion(): - version = platform.win32_ver()[1] - build = getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) - -# Print the host and container OS build numbers -print('Host OS build number: {}'.format(sys.argv[1])) -print('Container OS build number: {}'.format(getOsVersion())) -sys.stdout.flush() - -# Verify each DLL file in the directory specified by our command-line argument -dlls = glob.glob(os.path.join(sys.argv[2], '*.dll')) -for dll in dlls: - - # Attempt to retrieve the version number of the DLL - dllName = os.path.basename(dll) - try: - dllVersion = getDllVersion(dll) - except: - print('\nError: could not read the version string from the DLL file "{}".'.format(dllName), file=sys.stderr) - print('Please ensure the DLLs copied from the host are valid DLL files.', file=sys.stderr) - sys.exit(1) - - # Print the DLL details - print('Found host DLL file "{}" with version string "{}".'.format(dllName, dllVersion)) - sys.stdout.flush() - - # Determine if the container OS will load the DLL - try: - handle = win32api.LoadLibrary(dll) - win32api.FreeLibrary(handle) - except: - print('\nError: the container OS cannot load the DLL file "{}".'.format(dllName), file=sys.stderr) - print('This typically indicates that the DLL is from a newer version of Windows.', file=sys.stderr) - print('Please ensure the DLLs copied from the host match the container OS version.', file=sys.stderr) - sys.exit(1) diff --git a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile b/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile deleted file mode 100644 index fe1f894b..00000000 --- a/ue4docker/dockerfiles/ue4-engine/linux/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} - -# Build UBT and build the Engine -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -WaitMutex -RUN ./Engine/Build/BatchFiles/Linux/Build.sh ShaderCompileWorker Linux Development -WaitMutex -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UnrealPak Linux Development -WaitMutex diff --git a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile b/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile deleted file mode 100644 index 0c1830c3..00000000 --- a/ue4docker/dockerfiles/ue4-engine/windows/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# escape=` -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} - -# Build UBT and build the Engine -RUN GenerateProjectFiles.bat -RUN .\Engine\Build\BatchFiles\Build.bat UE4Editor Win64 Development -WaitMutex -RUN .\Engine\Build\BatchFiles\Build.bat ShaderCompileWorker Win64 Development -WaitMutex -RUN .\Engine\Build\BatchFiles\Build.bat UnrealPak Win64 Development -WaitMutex diff --git a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile b/ue4docker/dockerfiles/ue4-full/linux/Dockerfile deleted file mode 100644 index 8f0d36c9..00000000 --- a/ue4docker/dockerfiles/ue4-full/linux/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder - -# Install ue4cli and conan-ue4cli -USER root -RUN pip3 install setuptools wheel -RUN pip3 install ue4cli conan-ue4cli -USER ue4 - -# Extract the third-party library details from UBT -RUN ue4 setroot /home/ue4/UnrealEngine -RUN ue4 conan generate - -# Copy the generated Conan packages into a new image with our Installed Build -FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} - -# Clone the UE4Capture repository -RUN git clone "https://github.com/adamrehn/UE4Capture.git" /home/ue4/UE4Capture - -# Install CMake, ue4cli, conan-ue4cli, and ue4-ci-helpers -USER root -RUN apt-get update && apt-get install -y --no-install-recommends cmake -RUN pip3 install setuptools wheel -RUN pip3 install ue4cli conan-ue4cli ue4-ci-helpers -USER ue4 - -# Copy the Conan configuration settings and package cache from the builder image -COPY --from=builder --chown=ue4:ue4 /home/ue4/.conan /home/ue4/.conan - -# Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) -RUN ue4 setroot /home/ue4/UnrealEngine -RUN ue4 conan generate --profile-only - -# Build the Conan packages for the UE4Capture dependencies -RUN ue4 conan build MediaIPC-ue4 - -# Patch the problematic UE4 header files under 4.19.x (this call is a no-op under newer Engine versions) -RUN python3 /home/ue4/UE4Capture/scripts/patch-headers.py - -# Enable PulseAudio support -USER root -RUN apt-get install -y --no-install-recommends pulseaudio-utils -COPY pulseaudio-client.conf /etc/pulse/client.conf - -# Enable X11 support -USER root -RUN apt-get install -y --no-install-recommends \ - libfontconfig1 \ - libfreetype6 \ - libglu1 \ - libsm6 \ - libxcomposite1 \ - libxcursor1 \ - libxi6 \ - libxrandr2 \ - libxrender1 \ - libxss1 \ - libxv1 \ - x11-xkb-utils \ - xauth \ - xfonts-base \ - xkb-data -USER ue4 diff --git a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile b/ue4docker/dockerfiles/ue4-full/windows/Dockerfile deleted file mode 100644 index 44cee90b..00000000 --- a/ue4docker/dockerfiles/ue4-full/windows/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -# escape=` -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder - -# Install ue4cli and conan-ue4cli -RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli conan-ue4cli --no-warn-script-location - -# Build UBT, and extract the third-party library details from UBT -# (Remove the profile base packages to avoid a bug where Windows locks the files and breaks subsequent profile generation) -RUN GenerateProjectFiles.bat -RUN ue4 setroot C:\UnrealEngine -RUN ue4 conan generate && ue4 conan generate --remove-only - -# Copy the generated Conan packages into a new image with our Installed Build -FROM ${NAMESPACE}/ue4-minimal:${TAG}-${PREREQS_TAG} - -# Install ue4cli conan-ue4cli, and ue4-ci-helpers -RUN pip install setuptools wheel --no-warn-script-location -RUN pip install ue4cli conan-ue4cli ue4-ci-helpers --no-warn-script-location - -# Copy the Conan configuration settings and package cache from the builder image -COPY --from=builder C:\Users\ContainerAdministrator\.conan C:\Users\ContainerAdministrator\.conan - -# Install conan-ue4cli (just generate the profile, since we've already copied the generated packages) -RUN ue4 setroot C:\UnrealEngine -RUN ue4 conan generate --profile-only - -# Install CMake and add it to the system PATH -RUN choco install -y cmake --installargs "ADD_CMAKE_TO_PATH=System" diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile deleted file mode 100644 index 7bbfc4f9..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/Dockerfile +++ /dev/null @@ -1,53 +0,0 @@ -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder - -# Set the changelist number in Build.version to ensure our Build ID is generated correctly -COPY set-changelist.py /tmp/set-changelist.py -RUN python3 /tmp/set-changelist.py /home/ue4/UnrealEngine/Engine/Build/Build.version - -# Increase the output verbosity of the DDC generation step -COPY verbose-ddc.py /tmp/verbose-ddc.py -RUN python3 /tmp/verbose-ddc.py /home/ue4/UnrealEngine/Engine/Build/InstalledEngineBuild.xml - -# Ensure UBT is built before we create the Installed Build, since Build.sh explicitly sets the -# target .NET framework version, whereas InstalledEngineBuild.xml just uses the system default, -# which can result in errors when running the built UBT due to the wrong version being targeted -RUN ./Engine/Build/BatchFiles/Linux/Build.sh UE4Editor Linux Development -Clean - -# Create an Installed Build of the Engine -# (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -WORKDIR /home/ue4/UnrealEngine -COPY exclude-components.py /tmp/exclude-components.py -RUN ./Engine/Build/BatchFiles/RunUAT.sh BuildGraph -target="Make Installed Build Linux" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && \ - python3 /tmp/exclude-components.py /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux $EXCLUDE_DEBUG $EXCLUDE_TEMPLATES - -# Some versions of the Engine fail to include UnrealPak in the Installed Build, so copy it manually -RUN cp ./Engine/Binaries/Linux/UnrealPak ./LocalBuilds/Engine/Linux/Engine/Binaries/Linux/UnrealPak - -# Ensure the bundled toolchain included in 4.20.0 and newer is copied to the Installed Build -COPY --chown=ue4:ue4 copy-toolchain.py /tmp/copy-toolchain.py -RUN python3 /tmp/copy-toolchain.py /home/ue4/UnrealEngine - -# Copy the Installed Build into a clean image, discarding the source build -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} - -# Copy the Installed Build files from the builder image -COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/LocalBuilds/Engine/Linux /home/ue4/UnrealEngine -COPY --from=builder --chown=ue4:ue4 /home/ue4/UnrealEngine/root_commands.sh /tmp/root_commands.sh -WORKDIR /home/ue4/UnrealEngine - -# Run the post-setup commands that were previously extracted from `Setup.sh` -USER root -RUN /tmp/root_commands.sh -USER ue4 - -# Add labels to the built image to identify which components (if any) were excluded from the build that it contains -# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} -LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py b/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py deleted file mode 100644 index 04c4add4..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/copy-toolchain.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 -from os.path import basename, dirname, exists, join, relpath -import glob, shutil, sys - -# Determine the root directory for the source build and the Installed Build -sourceRoot = sys.argv[1] -installedRoot = join(sourceRoot, 'LocalBuilds', 'Engine', 'Linux') - -# Locate the bundled toolchain and copy it to the Installed Build -sdkGlob = join(sourceRoot, 'Engine', 'Extras', 'ThirdPartyNotUE', 'SDKs', 'HostLinux', 'Linux_x64', '*', 'x86_64-unknown-linux-gnu') -for bundled in glob.glob(sdkGlob): - - # Extract the root path for the toolchain - toolchain = dirname(bundled) - - # Print progress output - print('Copying bundled toolchain "{}" to Installed Build...'.format(basename(toolchain)), file=sys.stderr) - sys.stderr.flush() - - # Perform the copy - dest = join(installedRoot, relpath(toolchain, sourceRoot)) - if exists(dest) == True: - print('Destination toolchain already exists: {}'.format(dest), file=sys.stderr, flush=True) - else: - shutil.copytree(toolchain, dest) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py deleted file mode 100644 index be1ec31e..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/exclude-components.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -import glob, json, os, shutil, sys -from os.path import join - -# Logs a message to stderr -def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() - -# Reads the contents of a file -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -rootDir = sys.argv[1] -version = json.loads(readFile(join(rootDir, 'Engine', 'Build', 'Build.version'))) - -# Determine if we are excluding debug symbols -truncateDebug = len(sys.argv) > 2 and sys.argv[2] == '1' -if truncateDebug == True: - - # Remove all *.debug and *.sym files - log('User opted to exclude debug symbols, removing all *.debug and *.sym files.') - log('Scanning for debug symbols in directory {}...'.format(rootDir)) - symbolFiles = glob.glob(join(rootDir, '**', '*.debug'), recursive=True) + glob.glob(join(rootDir, '**', '*.sym'), recursive=True) - for symbolFile in symbolFiles: - log('Removing debug symbol file {}...'.format(symbolFile)) - try: - os.unlink(symbolFile, 0) - except: - log(' Warning: failed to remove debug symbol file {}.'.format(symbolFile)) - -# Determine if we are excluding the Engine's template projects and samples -excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' -if excludeTemplates == True: - log('User opted to exclude templates and samples.') - for subdir in ['FeaturePacks', 'Samples', 'Templates']: - log('Removing {} directory...'.format(subdir)) - try: - shutil.rmtree(join(rootDir, subdir)) - except: - log(' Warning: failed to remove {} directory...'.format(subdir)) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py deleted file mode 100644 index 7535f234..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/set-changelist.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import json, os, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file -versionFile = sys.argv[1] -details = json.loads(readFile(versionFile)) -details['Changelist'] = details['CompatibleChangelist'] -details['IsPromotedBuild'] = 1 -writeFile(versionFile, json.dumps(details, indent=4)) diff --git a/ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py b/ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py deleted file mode 100644 index 622087d5..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/linux/verbose-ddc.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Add verbose output flags to the `BuildDerivedDataCache` command -buildXml = sys.argv[1] -code = readFile(buildXml) -code = code.replace( - 'Command Name="BuildDerivedDataCache" Arguments="', - 'Command Name="BuildDerivedDataCache" Arguments="-Verbose -AllowStdOutLogVerbosity ' -) -writeFile(buildXml, code) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile b/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile deleted file mode 100644 index 07374545..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/windows/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# escape=` -ARG NAMESPACE -ARG TAG -ARG PREREQS_TAG -FROM ${NAMESPACE}/ue4-source:${TAG}-${PREREQS_TAG} AS builder - -# Set the changelist number in Build.version to ensure our Build ID is generated correctly -COPY set-changelist.py C:\set-changelist.py -RUN python C:\set-changelist.py C:\UnrealEngine\Engine\Build\Build.version - -# Patch out problematic entries in InstalledEngineFilters.xml introduced in UE4.20.0 -COPY patch-filters-xml.py C:\patch-filters-xml.py -RUN python C:\patch-filters-xml.py C:\UnrealEngine\Engine\Build\InstalledEngineFilters.xml - -# Create an Installed Build of the Engine -# (We can optionally remove debug symbols and/or template projects in order to reduce the final container image size) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -WORKDIR C:\UnrealEngine -COPY exclude-components.py C:\exclude-components.py -RUN .\Engine\Build\BatchFiles\RunUAT.bat BuildGraph -target="Make Installed Build Win64" -script=Engine/Build/InstalledEngineBuild.xml -set:HostPlatformOnly=true && ` - python C:\exclude-components.py C:\UnrealEngine\LocalBuilds\Engine\Windows %EXCLUDE_DEBUG% %EXCLUDE_TEMPLATES% - -# Copy our legacy toolchain installation script into the Installed Build -# (This ensures we only need one COPY directive below) -RUN C:\copy.py C:\legacy-toolchain-fix.py C:\UnrealEngine\LocalBuilds\Engine\Windows\ - -# Copy the Installed Build into a clean image, discarding the source tree -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} -COPY --from=builder C:\UnrealEngine\LocalBuilds\Engine\Windows C:\UnrealEngine -WORKDIR C:\UnrealEngine - -# Install legacy toolchain components if we're building UE4.19 -# (This call is a no-op under newer Engine versions) -RUN python C:\UnrealEngine\legacy-toolchain-fix.py - -# Add labels to the built image to identify which components (if any) were excluded from the build that it contains -# (Note that we need to redeclare the relevant ARG directives here because they are scoped to each individual stage in a multi-stage build) -ARG EXCLUDE_DEBUG -ARG EXCLUDE_TEMPLATES -LABEL com.adamrehn.ue4-docker.excluded.debug=${EXCLUDE_DEBUG} -LABEL com.adamrehn.ue4-docker.excluded.templates=${EXCLUDE_TEMPLATES} diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py b/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py deleted file mode 100644 index a40c49d9..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/windows/exclude-components.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -import glob, json, os, shutil, sys -from os.path import join - -# Logs a message to stderr -def log(message): - print(message, file=sys.stderr) - sys.stderr.flush() - -# Reads the contents of a file -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -rootDir = sys.argv[1] -version = json.loads(readFile(join(rootDir, 'Engine', 'Build', 'Build.version'))) - -# Determine if we are excluding debug symbols -truncateDebug = len(sys.argv) > 2 and sys.argv[2] == '1' -if truncateDebug == True: - - # Truncate all PDB files to save space whilst avoiding the issues that would be caused by the files being missing - log('User opted to exclude debug symbols, truncating all PDB files.') - log('Scanning for PDB files in directory {}...'.format(rootDir)) - pdbFiles = glob.glob(join(rootDir, '**', '*.pdb'), recursive=True) - for pdbFile in pdbFiles: - log('Truncating PDB file {}...'.format(pdbFile)) - try: - os.truncate(pdbFile, 0) - except: - log(' Warning: failed to truncate PDB file {}.'.format(pdbFile)) - - # Under UE4.19, we need to delete the PDB files for AutomationTool entirely, since truncated files cause issues - if version['MinorVersion'] < 20: - pdbFiles = glob.glob(join(rootDir, 'Engine', 'Source', 'Programs', 'AutomationTool', '**', '*.pdb'), recursive=True) - for pdbFile in pdbFiles: - log('Removing PDB file {}...'.format(pdbFile)) - try: - os.unlink(pdbFile) - except: - log(' Warning: failed to remove PDB file {}.'.format(pdbFile)) - -# Determine if we are excluding the Engine's template projects and samples -excludeTemplates = len(sys.argv) > 3 and sys.argv[3] == '1' -if excludeTemplates == True: - log('User opted to exclude templates and samples.') - for subdir in ['FeaturePacks', 'Samples', 'Templates']: - log('Removing {} directory...'.format(subdir)) - try: - shutil.rmtree(join(rootDir, subdir)) - except: - log(' Warning: failed to remove {} directory...'.format(subdir)) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py b/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py deleted file mode 100644 index af7de086..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/windows/patch-filters-xml.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Remove the dependency on Linux cross-compilation debug tools introduced in UE4.20.0 -filtersXml = sys.argv[1] -code = readFile(filtersXml) -code = code.replace('Engine/Binaries/Linux/dump_syms.exe', '') -code = code.replace('Engine/Binaries/Linux/BreakpadSymbolEncoder.exe', '') -writeFile(filtersXml, code) diff --git a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py b/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py deleted file mode 100644 index 7535f234..00000000 --- a/ue4docker/dockerfiles/ue4-minimal/windows/set-changelist.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import json, os, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Update the `Changelist` field to reflect the `CompatibleChangelist` field in our version file -versionFile = sys.argv[1] -details = json.loads(readFile(versionFile)) -details['Changelist'] = details['CompatibleChangelist'] -details['IsPromotedBuild'] = 1 -writeFile(versionFile, json.dumps(details, indent=4)) diff --git a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile b/ue4docker/dockerfiles/ue4-source/linux/Dockerfile deleted file mode 100644 index 48ec3229..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -ARG PREREQS_TAG -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} - -# The git repository that we will clone -ARG GIT_REPO="" - -# The git branch/tag that we will checkout -ARG GIT_BRANCH="" - -# Retrieve the address for the host that will supply git credentials -ARG HOST_ADDRESS_ARG="" -ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} - -# Retrieve the security token for communicating with the credential supplier -ARG HOST_TOKEN_ARG="" -ENV HOST_TOKEN=${HOST_TOKEN_ARG} - -# Install our git credential helper that forwards requests to the host -COPY --chown=ue4:ue4 git-credential-helper.sh /tmp/git-credential-helper.sh -ENV GIT_ASKPASS=/tmp/git-credential-helper.sh -RUN chmod +x /tmp/git-credential-helper.sh - -# Clone the UE4 git repository using the host-supplied credentials -RUN git clone --progress --depth=1 -b $GIT_BRANCH $GIT_REPO /home/ue4/UnrealEngine - -# Ensure our package lists are up to date, since Setup.sh doesn't call `apt-get update` -USER root -RUN apt-get update -USER ue4 - -# Patch out all instances of `sudo` in Setup.sh, plus any commands that refuse to run as root -COPY --chown=ue4:ue4 patch-setup-linux.py /tmp/patch-setup-linux.py -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Setup.sh -RUN python3 /tmp/patch-setup-linux.py /home/ue4/UnrealEngine/Engine/Build/BatchFiles/Linux/Setup.sh - -# Create a script to hold the list of post-clone setup commands that require root -WORKDIR /home/ue4/UnrealEngine -RUN echo "#!/usr/bin/env bash" >> ./root_commands.sh -RUN echo "set -x" >> ./root_commands.sh -RUN echo "apt-get update" >> ./root_commands.sh -RUN chmod a+x ./root_commands.sh - -# Preinstall mono for 4.19 so the GitDependencies.exe call triggered by Setup.sh works correctly -# (this is a no-op for newer Engine versions) -USER root -COPY --chown=ue4:ue4 preinstall-mono.py /tmp/preinstall-mono.py -RUN python3 /tmp/preinstall-mono.py -USER ue4 - -# Extract the list of post-clone setup commands that require root and add them to the script, -# running everything else as the non-root user to avoid creating files owned by root -RUN ./Setup.sh - -# Run the extracted root commands we gathered in our script -USER root -RUN ./root_commands.sh -USER ue4 - -# Make sure the root commands script cleans up the package lists when it is run in the ue4-minimal image -RUN echo "rm -rf /var/lib/apt/lists/*" >> ./root_commands.sh - -# The linker bundled with UE4.20.0 onwards chokes on system libraries built with newer compilers, -# so redirect the bundled clang to use the system linker instead -COPY --chown=ue4:ue4 linker-fixup.py /tmp/linker-fixup.py -RUN python3 /tmp/linker-fixup.py /home/ue4/UnrealEngine/Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64 `which ld` diff --git a/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py b/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py deleted file mode 100644 index ff75f591..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/linker-fixup.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python3 -import glob, os, shutil, sys - -# Retrieve the Linux SDK root directory and system ld location from our command-line arguments -sdkRoot = sys.argv[1] -systemLd = sys.argv[2] - -# Locate the bundled version(s) of ld and replace them with symlinks to the system ld -for bundled in glob.glob(os.path.join(sdkRoot, '*', 'x86_64-unknown-linux-gnu', 'bin', 'x86_64-unknown-linux-gnu-ld')): - os.unlink(bundled) - os.symlink(systemLd, bundled) - print('{} => {}'.format(bundled, systemLd), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py b/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py deleted file mode 100644 index c0368dce..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/patch-setup-linux.py +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env python3 -import os, re, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Extract all commands requiring `sudo` in Setup.sh and place them in root_commands.sh -setupScript = sys.argv[1] -code = readFile(setupScript) -code = re.sub('(\\s)sudo ([^\\n]+)\\n', '\\1echo \\2 >> /home/ue4/UnrealEngine/root_commands.sh\\n', code) -writeFile(setupScript, code) - -# Print the patched code to stderr for debug purposes -print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) diff --git a/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py b/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py deleted file mode 100644 index b98b8c3b..00000000 --- a/ue4docker/dockerfiles/ue4-source/linux/preinstall-mono.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 -import json, os, subprocess, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Determine if we are building UE 4.19 -versionDetails = json.loads(readFile('/home/ue4/UnrealEngine/Engine/Build/Build.version')) -if versionDetails['MinorVersion'] == 19: - - # Our required packages, extracted from Setup.sh - packages = [ - 'mono-xbuild', - 'mono-dmcs', - 'libmono-microsoft-build-tasks-v4.0-4.0-cil', - 'libmono-system-data-datasetextensions4.0-cil', - 'libmono-system-web-extensions4.0-cil', - 'libmono-system-management4.0-cil', - 'libmono-system-xml-linq4.0-cil', - 'libmono-corlib4.5-cil', - 'libmono-windowsbase4.0-cil', - 'libmono-system-io-compression4.0-cil', - 'libmono-system-io-compression-filesystem4.0-cil', - 'libmono-system-runtime4.0-cil', - 'mono-devel' - ] - - # Preinstall the packages - result = subprocess.call(['apt-get', 'install', '-y'] + packages) - if result != 0: - sys.exit(result) - - # Add the package installation commands to root_commands.sh - # (This ensures the packages will subsequently be installed by the ue4-minimal image) - with open('/home/ue4/UnrealEngine/root_commands.sh', 'a') as script: - for package in packages: - script.write('apt-get install -y {}\n'.format(package)) diff --git a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile b/ue4docker/dockerfiles/ue4-source/windows/Dockerfile deleted file mode 100644 index 0c4dfb34..00000000 --- a/ue4docker/dockerfiles/ue4-source/windows/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# escape=` -ARG PREREQS_TAG -FROM adamrehn/ue4-build-prerequisites:${PREREQS_TAG} - -# The git repository that we will clone -ARG GIT_REPO="" - -# The git branch/tag that we will checkout -ARG GIT_BRANCH="" - -# Retrieve the address for the host that will supply git credentials -ARG HOST_ADDRESS_ARG="" -ENV HOST_ADDRESS=${HOST_ADDRESS_ARG} - -# Retrieve the security token for communicating with the credential supplier -ARG HOST_TOKEN_ARG="" -ENV HOST_TOKEN=${HOST_TOKEN_ARG} - -# Install our git credential helper that forwards requests to the host -COPY git-credential-helper.bat C:\git-credential-helper.bat -ENV GIT_ASKPASS=C:\git-credential-helper.bat - -# Clone the UE4 git repository using the host-supplied credentials -WORKDIR C:\ -RUN git clone --progress --depth=1 -b %GIT_BRANCH% %GIT_REPO% C:\UnrealEngine - -# Install legacy toolchain components if we're building UE4.19 -# (This call is a no-op under newer Engine versions) -COPY legacy-toolchain-fix.py C:\legacy-toolchain-fix.py -RUN python C:\legacy-toolchain-fix.py - -# Since the UE4 prerequisites installer appears to break when newer versions -# of the VC++ runtime are present, patch out the prereqs call in Setup.bat -COPY patch-setup-win.py C:\patch-setup-win.py -RUN python C:\patch-setup-win.py C:\UnrealEngine\Setup.bat - -# Run post-clone setup steps -WORKDIR C:\UnrealEngine -RUN Setup.bat diff --git a/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py b/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py deleted file mode 100644 index 4edfbd21..00000000 --- a/ue4docker/dockerfiles/ue4-source/windows/legacy-toolchain-fix.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import json, os, subprocess, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -# Parse the UE4 version information -version = json.loads(readFile('C:\\UnrealEngine\\Engine\\Build\\Build.version')) - -# UE4.19 has problems detecting the VS2017 Build Tools and doesn't use the Windows 10 SDK -# by default, so we need to install the VS2015 Build Tools and the Windows 8.1 SDK -if version['MinorVersion'] < 20: - print('Installing VS2015 Build Tools and Windows 8.1 SDK for UE4.19 compatibility...') - sys.stdout.flush() - run = lambda cmd: subprocess.run(cmd, check=True) - installerFile = '{}\\vs_buildtools.exe'.format(os.environ['TEMP']) - run(['curl', '--progress', '-L', 'https://aka.ms/vs/16/release/vs_buildtools.exe', '--output', installerFile]) - run([ - installerFile, - '--quiet', '--wait', '--norestart', '--nocache', - '--installPath', 'C:\BuildTools', - '--channelUri', 'https://aka.ms/vs/15/release/channel', - '--installChannelUri', 'https://aka.ms/vs/15/release/channel', - '--channelId', 'VisualStudio.15.Release', - '--productId', 'Microsoft.VisualStudio.Product.BuildTools', - '--add', 'Microsoft.VisualStudio.Component.VC.140', - '--add', 'Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81' - ]) diff --git a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py b/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py deleted file mode 100644 index ecb6b88f..00000000 --- a/ue4docker/dockerfiles/ue4-source/windows/patch-setup-win.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python3 -import os, sys - -def readFile(filename): - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - -def writeFile(filename, data): - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) - -# Comment out the call to the UE4 prereqs installer in Setup.bat -PREREQ_CALL = 'start /wait Engine\\Extras\\Redist\\en-us\\UE4PrereqSetup_x64.exe' -setupScript = sys.argv[1] -code = readFile(setupScript) -code = code.replace('echo Installing prerequisites...', 'echo (Skipping installation of prerequisites)') -code = code.replace(PREREQ_CALL, '@rem ' + PREREQ_CALL) - -# Also comment out the version selector call, since we don't need shell integration -SELECTOR_CALL = '.\\Engine\\Binaries\\Win64\\UnrealVersionSelector-Win64-Shipping.exe /register' -code = code.replace(SELECTOR_CALL, '@rem ' + SELECTOR_CALL) - -# Add output so we can see when script execution is complete, and ensure `pause` is not called on error -code = code.replace('rem Done!', 'echo Done!\r\nexit /b 0') -code = code.replace('pause', '@rem pause') -writeFile(setupScript, code) - -# Print the patched code to stderr for debug purposes -print('PATCHED {}:\n\n{}'.format(setupScript, code), file=sys.stderr) diff --git a/ue4docker/export.py b/ue4docker/export.py deleted file mode 100644 index c34e6ee4..00000000 --- a/ue4docker/export.py +++ /dev/null @@ -1,89 +0,0 @@ -from .infrastructure import DockerUtils, GlobalConfiguration, PrettyPrinting -from .exports import * -import sys - -def _notNone(items): - return len([i for i in items if i is not None]) == len(items) - -def _extractArg(args, index): - return args[index] if len(args) > index else None - -def _isHelpFlag(arg): - return (arg.strip('-') in ['h', 'help']) == True - -def _stripHelpFlags(args): - return list([a for a in args if _isHelpFlag(a) == False]) - -def export(): - - # The components that can be exported - COMPONENTS = { - 'installed': { - 'function': exportInstalledBuild, - 'description': 'Exports an Installed Build of the Engine', - 'image': GlobalConfiguration.resolveTag('ue4-full'), - 'help': 'Copies the Installed Build from a container to the host system.\nOnly supported under Linux for UE 4.21.0 and newer.' - }, - 'packages': { - 'function': exportPackages, - 'description': 'Exports conan-ue4cli wrapper packages', - 'image': GlobalConfiguration.resolveTag('ue4-full'), - 'help': - 'Runs a temporary conan server inside a container and uses it to export the\ngenerated conan-ue4cli wrapper packages.\n\n' + - 'Currently the only supported destination value is "cache", which exports\nthe packages to the Conan local cache on the host system.' - } - } - - # Parse the supplied command-line arguments - stripped = _stripHelpFlags(sys.argv) - args = { - 'help': len(stripped) < len(sys.argv), - 'component': _extractArg(stripped, 1), - 'tag': _extractArg(stripped, 2), - 'destination': _extractArg(stripped, 3) - } - - # If a component name has been specified, verify that it is valid - if args['component'] is not None and args['component'] not in COMPONENTS: - print('Error: unrecognised component "{}".'.format(args['component']), file=sys.stderr) - sys.exit(1) - - # Determine if we are performing an export - if args['help'] == False and _notNone([args['component'], args['tag'], args['destination']]): - - # Determine if the user specified an image and a tag or just a tag - tag = args['tag'] - details = COMPONENTS[ args['component'] ] - requiredImage = '{}:{}'.format(details['image'], tag) if ':' not in tag else tag - - # Verify that the required container image exists - if DockerUtils.exists(requiredImage) == False: - print('Error: the specified container image "{}" does not exist.'.format(requiredImage), file=sys.stderr) - sys.exit(1) - - # Attempt to perform the export - details['function'](requiredImage, args['destination'], stripped[4:]) - print('Export complete.') - - # Determine if we are displaying the help for a specific component - elif args['help'] == True and args['component'] is not None: - - # Display the help for the component - component = sys.argv[1] - details = COMPONENTS[component] - print('{} export {}'.format(sys.argv[0], component)) - print(details['description'] + '\n') - print('Exports from image: {}:TAG\n'.format(details['image'])) - print(details['help']) - - else: - - # Print usage syntax - print('Usage: {} export COMPONENT TAG DESTINATION\n'.format(sys.argv[0])) - print('Exports components from built container images to the host system\n') - print('Components:') - PrettyPrinting.printColumns([ - (component, COMPONENTS[component]['description']) - for component in COMPONENTS - ]) - print('\nRun `{} export COMPONENT --help` for more information on a component.'.format(sys.argv[0])) diff --git a/ue4docker/exports/export_installed.py b/ue4docker/exports/export_installed.py deleted file mode 100644 index b072eb0f..00000000 --- a/ue4docker/exports/export_installed.py +++ /dev/null @@ -1,53 +0,0 @@ -from ..infrastructure import DockerUtils, PackageUtils, SubprocessUtils -import os, platform, shutil, subprocess, sys - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - -def exportInstalledBuild(image, destination, extraArgs): - - # Verify that we are running under Linux - if platform.system() != 'Linux': - print('Error: Installed Builds can only be exported under Linux.', file=sys.stderr) - sys.exit(1) - - # Verify that the destination directory does not already exist - if os.path.exists(destination) == True: - print('Error: the destination directory already exists.', file=sys.stderr) - sys.exit(1) - - # Verify that the Installed Build in the specified image is at least 4.21.0 - versionResult = SubprocessUtils.capture(['docker', 'run', '--rm', '-ti', image, 'ue4', 'version']) - try: - version = semver.parse(SubprocessUtils.extractLines(versionResult.stdout)[-1]) - if version['minor'] < 21: - raise Exception() - except: - print('Error: Installed Builds can only be exported for Unreal Engine 4.21.0 and newer.', file=sys.stderr) - sys.exit(1) - - # Start a container from which we will copy files - container = DockerUtils.start(image, 'bash') - - # Attempt to perform the export - print('Exporting to {}...'.format(destination)) - containerPath = '{}:/home/ue4/UnrealEngine'.format(container.name) - exportResult = subprocess.call(['docker', 'cp', containerPath, destination]) - - # Stop the container, irrespective of whether or not the export succeeded - container.stop() - - # If the export succeeded, regenerate the linker symlinks on the host system - if exportResult == 0: - print('Performing linker symlink fixup...') - subprocess.call([ - sys.executable, - os.path.join(os.path.dirname(os.path.dirname(__file__)), 'dockerfiles', 'ue4-source', 'linux', 'linker-fixup.py'), - os.path.join(destination, 'Engine/Extras/ThirdPartyNotUE/SDKs/HostLinux/Linux_x64'), - shutil.which('ld') - ]) - - # Report any failures - if exportResult != 0: - print('Error: failed to export Installed Build.', file=sys.stderr) - sys.exit(1) diff --git a/ue4docker/exports/export_packages.py b/ue4docker/exports/export_packages.py deleted file mode 100644 index b9b890cc..00000000 --- a/ue4docker/exports/export_packages.py +++ /dev/null @@ -1,157 +0,0 @@ -from ..infrastructure import DockerUtils, FilesystemUtils, Logger, SubprocessUtils -import docker, os, subprocess, sys, tempfile - -# The name we use for our temporary Conan remote -REMOTE_NAME = '_ue4docker_export_temp' - -# Our conan_server config file data -CONAN_SERVER_CONFIG = ''' -[server] -jwt_secret: jwt_secret -jwt_expire_minutes: 120 -ssl_enabled: False -port: 9300 -public_port: 9300 -host_name: {} -authorize_timeout: 1800 -disk_storage_path: {} -disk_authorize_timeout: 1800 -updown_secret: updown_secret - -[write_permissions] -*/*@*/*: * - -[read_permissions] -*/*@*/*: * - -[users] -user: password -''' - - -def exportPackages(image, destination, extraArgs): - - # Create our logger to generate coloured output on stderr - logger = Logger() - - # Verify that the destination is "cache" - if destination.lower() != 'cache': - logger.error('Error: the only supported package export destination is "cache".') - sys.exit(1) - - # Verify that Conan is installed on the host - try: - SubprocessUtils.run(['conan', '--version']) - except: - logger.error('Error: Conan must be installed on the host system to export packages.') - sys.exit(1) - - # Determine if the container image is a Windows image or a Linux image - imageOS = DockerUtils.listImages(image)[0].attrs['Os'] - - # Use the appropriate commands and paths for the container platform - cmdsAndPaths = { - - 'linux': { - 'rootCommand': ['bash', '-c', 'sleep infinity'], - 'mkdirCommand': ['mkdir'], - 'copyCommand': ['cp', '-f'], - - 'dataDir': '/home/ue4/.conan_server/data', - 'configDir': '/home/ue4/.conan_server/', - 'bindMount': '/hostdir/' - }, - - 'windows': { - 'rootCommand': ['timeout', '/t', '99999', '/nobreak'], - 'mkdirCommand': ['cmd', '/S', '/C', 'mkdir'], - 'copyCommand': ['python', 'C:\\copy.py'], - - 'dataDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\data', - 'configDir': 'C:\\Users\\ContainerAdministrator\\.conan_server\\', - 'bindMount': 'C:\\hostdir\\' - } - - }[imageOS] - - # Create an auto-deleting temporary directory to hold our server config file - with tempfile.TemporaryDirectory() as tempDir: - - # Progress output - print('Starting conan_server in a container...') - - # Start a container from which we will export packages, bind-mounting our temp directory - container = DockerUtils.start( - image, - cmdsAndPaths['rootCommand'], - ports = {'9300/tcp': 9300}, - mounts = [docker.types.Mount(cmdsAndPaths['bindMount'], tempDir, 'bind')], - stdin_open = imageOS == 'windows', - tty = imageOS == 'windows', - remove = True - ) - - # Reload the container attributes from the Docker daemon to ensure the networking fields are populated - container.reload() - - # Under Linux we can simply access the container from the host over the loopback address, but this doesn't work under Windows - # (See ) - externalAddress = '127.0.0.1' if imageOS == 'linux' else container.attrs['NetworkSettings']['Networks']['nat']['IPAddress'] - - # Generate our server config file in the temp directory - FilesystemUtils.writeFile(os.path.join(tempDir, 'server.conf'), CONAN_SERVER_CONFIG.format(externalAddress, cmdsAndPaths['dataDir'])) - - # Keep track of the `conan_server` log output so we can display it in case of an error - serverOutput = None - - try: - - # Copy the server config file to the expected location inside the container - DockerUtils.execMultiple(container, [ - cmdsAndPaths['mkdirCommand'] + [cmdsAndPaths['configDir']], - cmdsAndPaths['copyCommand'] + [cmdsAndPaths['bindMount'] + 'server.conf', cmdsAndPaths['configDir']] - ]) - - # Start `conan_server` - serverOutput = DockerUtils.exec(container, ['conan_server'], stream = True) - - # Progress output - print('Uploading packages to the server...') - - # Upload all of the packages in the container's local cache to the server - DockerUtils.execMultiple(container, [ - ['conan', 'remote', 'add', 'localhost', 'http://127.0.0.1:9300'], - ['conan', 'user', 'user', '-r', 'localhost', '-p', 'password'], - ['conan', 'upload', '*/4.*', '--all', '--confirm', '-r=localhost'] - ]) - - # Configure the server as a temporary remote on the host system - SubprocessUtils.run(['conan', 'remote', 'add', REMOTE_NAME, 'http://{}:9300'.format(externalAddress)]) - SubprocessUtils.run(['conan', 'user', 'user', '-r', REMOTE_NAME, '-p', 'password']) - - # Retrieve the list of packages that were uploaded to the server - packages = SubprocessUtils.extractLines(SubprocessUtils.capture(['conan', 'search', '-r', REMOTE_NAME, '*']).stdout) - packages = [package for package in packages if '/' in package and '@' in package] - - # Download each package in turn - for package in packages: - print('Downloading package {} to host system local cache...'.format(package)) - SubprocessUtils.run(['conan', 'download', '-r', REMOTE_NAME, package]) - - # Once we reach this point, everything has worked and we don't need to output any logs - serverOutput = None - - finally: - - # Stop the container, irrespective of whether or not the export succeeded - print('Stopping conan_server...') - container.stop() - - # If something went wrong then output the logs from `conan_server` to assist in diagnosing the failure - if serverOutput is not None: - print('Log output from conan_server:') - for chunk in serverOutput: - logger.error(chunk.decode('utf-8')) - - # Remove the temporary remote if it was created successfully - SubprocessUtils.run(['conan', 'remote', 'remove', REMOTE_NAME], check = False) diff --git a/ue4docker/info.py b/ue4docker/info.py deleted file mode 100644 index c43f3b63..00000000 --- a/ue4docker/info.py +++ /dev/null @@ -1,65 +0,0 @@ -import humanfriendly, platform, psutil, shutil, sys -from .version import __version__ -from .infrastructure import * - -def _osName(dockerInfo): - if platform.system() == 'Windows': - return WindowsUtils.systemStringLong() - elif platform.system() == 'Darwin': - return DarwinUtils.systemString() - else: - return 'Linux ({}, {})'.format(dockerInfo['OperatingSystem'], dockerInfo['KernelVersion']) - -def _formatSize(size): - return humanfriendly.format_size(size, binary=True) - -def info(): - - # Verify that Docker is installed - if DockerUtils.installed() == False: - print('Error: could not detect Docker version. Please ensure Docker is installed.', file=sys.stderr) - sys.exit(1) - - # Gather our information about the Docker daemon - dockerInfo = DockerUtils.info() - nvidiaDocker = platform.system() == 'Linux' and 'nvidia' in dockerInfo['Runtimes'] - maxSize = DockerUtils.maxsize() - rootDir = dockerInfo['DockerRootDir'] - - # Gather our information about the host system - diskSpace = shutil.disk_usage(rootDir).free - memPhysical = psutil.virtual_memory().total - memVirtual = psutil.swap_memory().total - cpuPhysical = psutil.cpu_count(False) - cpuLogical = psutil.cpu_count() - - # Attempt to query PyPI to determine the latest version of ue4-docker - # (We ignore any errors here to ensure the `ue4-docker info` command still works without network access) - try: - latestVersion = GlobalConfiguration.getLatestVersion() - except: - latestVersion = None - - # Prepare our report items - items = [ - ('ue4-docker version', '{}{}'.format(__version__, '' if latestVersion is None else ' (latest available version is {})'.format(latestVersion))), - ('Operating system', _osName(dockerInfo)), - ('Docker daemon version', dockerInfo['ServerVersion']), - ('NVIDIA Docker supported', 'Yes' if nvidiaDocker == True else 'No'), - ('Maximum image size', '{:.0f}GB'.format(maxSize) if maxSize != -1 else 'No limit detected'), - ('Available disk space', _formatSize(diskSpace)), - ('Total system memory', '{} physical, {} virtual'.format(_formatSize(memPhysical), _formatSize(memVirtual))), - ('Number of processors', '{} physical, {} logical'.format(cpuPhysical, cpuLogical)) - ] - - # Determine the longest item name so we can format our list in nice columns - longestName = max([len(i[0]) for i in items]) - minSpaces = 4 - - # Print our report - for item in items: - print('{}:{}{}'.format( - item[0], - ' ' * ((longestName + minSpaces) - len(item[0])), - item[1] - )) diff --git a/ue4docker/infrastructure/BuildConfiguration.py b/ue4docker/infrastructure/BuildConfiguration.py deleted file mode 100644 index 67aa2975..00000000 --- a/ue4docker/infrastructure/BuildConfiguration.py +++ /dev/null @@ -1,244 +0,0 @@ -from .DockerUtils import DockerUtils -from .PackageUtils import PackageUtils -from .WindowsUtils import WindowsUtils -import humanfriendly, os, platform, random -from pkg_resources import parse_version - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - -# The default Unreal Engine git repository -DEFAULT_GIT_REPO = 'https://github.com/EpicGames/UnrealEngine.git' - -# The base images for Linux containers -LINUX_BASE_IMAGES = { - 'opengl': 'nvidia/opengl:1.0-glvnd-devel-ubuntu18.04', - 'cudagl': { - '9.2': 'nvidia/cudagl:9.2-devel-ubuntu18.04', - '10.0': 'nvidia/cudagl:10.0-devel-ubuntu18.04', - '10.1': 'nvidia/cudagl:10.1-devel-ubuntu18.04' - } -} - -# The default CUDA version to use when `--cuda` is specified without a value -DEFAULT_CUDA_VERSION = '9.2' - -# The default memory limit (in GB) under Windows -DEFAULT_MEMORY_LIMIT = 10.0 - - -class ExcludedComponent(object): - ''' - The different components that we support excluding from the built images - ''' - - # Engine debug symbols - Debug = 'debug' - - # Template projects and samples - Templates = 'templates' - - - @staticmethod - def description(component): - ''' - Returns a human-readable description of the specified component - ''' - return { - - ExcludedComponent.Debug: 'Debug symbols', - ExcludedComponent.Templates: 'Template projects and samples' - - }.get(component, '[Unknown component]') - - -class BuildConfiguration(object): - - @staticmethod - def addArguments(parser): - ''' - Registers our supported command-line arguments with the supplied argument parser - ''' - parser.add_argument('release', help='UE4 release to build, in semver format (e.g. 4.19.0) or "custom" for a custom repo and branch') - parser.add_argument('--linux', action='store_true', help='Build Linux container images under Windows') - parser.add_argument('--rebuild', action='store_true', help='Rebuild images even if they already exist') - parser.add_argument('--dry-run', action='store_true', help='Print `docker build` commands instead of running them') - parser.add_argument('--pull-prerequisites', action='store_true', help='Pull the ue4-build-prerequisites image from Docker Hub instead of building it') - parser.add_argument('--no-engine', action='store_true', help='Don\'t build the ue4-engine image') - parser.add_argument('--no-minimal', action='store_true', help='Don\'t build the ue4-minimal image') - parser.add_argument('--no-full', action='store_true', help='Don\'t build the ue4-full image') - parser.add_argument('--no-cache', action='store_true', help='Disable Docker build cache') - parser.add_argument('--random-memory', action='store_true', help='Use a random memory limit for Windows containers') - parser.add_argument('--exclude', action='append', default=[], choices=[ExcludedComponent.Debug, ExcludedComponent.Templates], help='Exclude the specified component (can be specified multiple times to exclude multiple components)') - parser.add_argument('--cuda', default=None, metavar='VERSION', help='Add CUDA support as well as OpenGL support when building Linux containers') - parser.add_argument('-username', default=None, help='Specify the username to use when cloning the git repository') - parser.add_argument('-password', default=None, help='Specify the password to use when cloning the git repository') - parser.add_argument('-repo', default=None, help='Set the custom git repository to clone when "custom" is specified as the release value') - parser.add_argument('-branch', default=None, help='Set the custom branch/tag to clone when "custom" is specified as the release value') - parser.add_argument('-isolation', default=None, help='Set the isolation mode to use for Windows containers (process or hyperv)') - parser.add_argument('-basetag', default=None, help='Windows Server Core base image tag to use for Windows containers (default is the host OS version)') - parser.add_argument('-dlldir', default=None, help='Set the directory to copy required Windows DLLs from (default is the host System32 directory)') - parser.add_argument('-suffix', default='', help='Add a suffix to the tags of the built images') - parser.add_argument('-m', default=None, help='Override the default memory limit under Windows (also overrides --random-memory)') - - def __init__(self, parser, argv): - ''' - Creates a new build configuration based on the supplied arguments object - ''' - - # If the user has specified `--cuda` without a version value, treat the value as an empty string - argv = [arg + '=' if arg == '--cuda' else arg for arg in argv] - - # Parse the supplied command-line arguments - self.args = parser.parse_args(argv) - - # Determine if we are building a custom version of UE4 rather than an official release - self.args.release = self.args.release.lower() - if self.args.release == 'custom' or self.args.release.startswith('custom:'): - - # Both a custom repository and a custom branch/tag must be specified - if self.args.repo is None or self.args.branch is None: - raise RuntimeError('both a repository and branch/tag must be specified when building a custom version of the Engine') - - # Use the specified repository and branch/tag - customName = self.args.release.split(':', 2)[1].strip() if ':' in self.args.release else '' - self.release = customName if len(customName) > 0 else 'custom' - self.repository = self.args.repo - self.branch = self.args.branch - self.custom = True - - else: - - # Validate the specified version string - try: - ue4Version = semver.parse(self.args.release) - if ue4Version['major'] != 4 or ue4Version['prerelease'] != None: - raise Exception() - self.release = semver.format_version(ue4Version['major'], ue4Version['minor'], ue4Version['patch']) - except: - raise RuntimeError('invalid UE4 release number "{}", full semver format required (e.g. "4.19.0")'.format(self.args.release)) - - # Use the default repository and the release tag for the specified version - self.repository = DEFAULT_GIT_REPO - self.branch = '{}-release'.format(self.release) - self.custom = False - - # Store our common configuration settings - self.containerPlatform = 'windows' if platform.system() == 'Windows' and self.args.linux == False else 'linux' - self.dryRun = self.args.dry_run - self.rebuild = self.args.rebuild - self.pullPrerequisites = self.args.pull_prerequisites - self.noEngine = self.args.no_engine - self.noMinimal = self.args.no_minimal - self.noFull = self.args.no_full - self.suffix = self.args.suffix - self.platformArgs = ['--no-cache'] if self.args.no_cache == True else [] - self.excludedComponents = set(self.args.exclude) - self.baseImage = None - self.prereqsTag = None - - # Generate our flags for keeping or excluding components - self.exclusionFlags = [ - '--build-arg', 'EXCLUDE_DEBUG={}'.format(1 if ExcludedComponent.Debug in self.excludedComponents else 0), - '--build-arg', 'EXCLUDE_TEMPLATES={}'.format(1 if ExcludedComponent.Templates in self.excludedComponents else 0) - ] - - # If we're building Windows containers, generate our Windows-specific configuration settings - if self.containerPlatform == 'windows': - self._generateWindowsConfig() - - # If we're building Linux containers, generate our Linux-specific configuration settings - if self.containerPlatform == 'linux': - self._generateLinuxConfig() - - # If the user-specified suffix passed validation, prefix it with a dash - self.suffix = '-{}'.format(self.suffix) if self.suffix != '' else '' - - def describeExcludedComponents(self): - ''' - Returns a list of strings describing the components that will be excluded (if any.) - ''' - return sorted([ExcludedComponent.description(component) for component in self.excludedComponents]) - - def _generateWindowsConfig(self): - - # Store the path to the directory containing our required Windows DLL files - self.defaultDllDir = os.path.join(os.environ['SystemRoot'], 'System32') - self.dlldir = self.args.dlldir if self.args.dlldir is not None else self.defaultDllDir - - # Determine base tag for the Windows release of the host system - self.hostRelease = WindowsUtils.getWindowsRelease() - self.hostBasetag = WindowsUtils.getReleaseBaseTag(self.hostRelease) - - # Store the tag for the base Windows Server Core image - self.basetag = self.args.basetag if self.args.basetag is not None else self.hostBasetag - self.baseImage = 'mcr.microsoft.com/dotnet/framework/sdk:4.7.2-windowsservercore-' + self.basetag - self.prereqsTag = self.basetag - - # Verify that any user-specified base tag is valid - if WindowsUtils.isValidBaseTag(self.basetag) == False: - raise RuntimeError('unrecognised Windows Server Core base image tag "{}", supported tags are {}'.format(self.basetag, WindowsUtils.getValidBaseTags())) - - # Verify that any user-specified tag suffix does not collide with our base tags - if WindowsUtils.isValidBaseTag(self.suffix) == True: - raise RuntimeError('tag suffix cannot be any of the Windows Server Core base image tags: {}'.format(WindowsUtils.getValidBaseTags())) - - # If the user has explicitly specified an isolation mode then use it, otherwise auto-detect - if self.args.isolation is not None: - self.isolation = self.args.isolation - else: - - # If we are able to use process isolation mode then use it, otherwise fallback to the Docker daemon's default isolation mode - differentKernels = WindowsUtils.isInsiderPreview() or self.basetag != self.hostBasetag - hostSupportsProcess = WindowsUtils.isWindowsServer() or int(self.hostRelease) >= 1809 - dockerSupportsProcess = parse_version(DockerUtils.version()['Version']) >= parse_version('18.09.0') - if not differentKernels and hostSupportsProcess and dockerSupportsProcess: - self.isolation = 'process' - else: - self.isolation = DockerUtils.info()['Isolation'] - - # Set the isolation mode Docker flag - self.platformArgs.append('--isolation=' + self.isolation) - - # If the user has explicitly specified a memory limit then use it, otherwise auto-detect - self.memLimit = None - if self.args.m is not None: - try: - self.memLimit = humanfriendly.parse_size(self.args.m) / (1000*1000*1000) - except: - raise RuntimeError('invalid memory limit "{}"'.format(self.args.m)) - else: - - # Only specify a memory limit when using Hyper-V isolation mode, in order to override the 1GB default limit - # (Process isolation mode does not impose any memory limits by default) - if self.isolation == 'hyperv': - self.memLimit = DEFAULT_MEMORY_LIMIT if self.args.random_memory == False else random.uniform(DEFAULT_MEMORY_LIMIT, DEFAULT_MEMORY_LIMIT + 2.0) - - # Set the memory limit Docker flag - if self.memLimit is not None: - self.platformArgs.extend(['-m', '{:.2f}GB'.format(self.memLimit)]) - - def _generateLinuxConfig(self): - - # Verify that any user-specified tag suffix does not collide with our base tags - if self.suffix.startswith('opengl') or self.suffix.startswith('cudagl'): - raise RuntimeError('tag suffix cannot begin with "opengl" or "cudagl".') - - # Determine if we are building CUDA-enabled container images - self.cuda = None - if self.args.cuda is not None: - - # Verify that the specified CUDA version is valid - self.cuda = self.args.cuda if self.args.cuda != '' else DEFAULT_CUDA_VERSION - if self.cuda not in LINUX_BASE_IMAGES['cudagl']: - raise RuntimeError('unsupported CUDA version "{}", supported versions are: {}'.format( - self.cuda, - ', '.join([v for v in LINUX_BASE_IMAGES['cudagl']]) - )) - - # Use the appropriate base image for the specified CUDA version - self.baseImage = LINUX_BASE_IMAGES['cudagl'][self.cuda] - self.prereqsTag = 'cudagl{}'.format(self.cuda) - else: - self.baseImage = LINUX_BASE_IMAGES['opengl'] - self.prereqsTag = 'opengl' diff --git a/ue4docker/infrastructure/CredentialEndpoint.py b/ue4docker/infrastructure/CredentialEndpoint.py deleted file mode 100644 index bb12e8fd..00000000 --- a/ue4docker/infrastructure/CredentialEndpoint.py +++ /dev/null @@ -1,72 +0,0 @@ -import logging, multiprocessing, os, platform, secrets, time, urllib.parse -from .NetworkUtils import NetworkUtils -from flask import Flask, request - -class CredentialEndpoint(object): - - def __init__(self, username, password): - ''' - Creates an endpoint manager for the supplied credentials - ''' - self.username = username - self.password = password - self.endpoint = None - - # Generate a security token to require when requesting credentials - self.token = secrets.token_hex(16) - - def args(self): - ''' - Returns the Docker build arguments for creating containers that require Git credentials - ''' - - # Resolve the IP address for the host system - hostAddress = NetworkUtils.hostIP() - - # Provide the host address and security token to the container - return [ - '--build-arg', 'HOST_ADDRESS_ARG=' + urllib.parse.quote_plus(hostAddress), - '--build-arg', 'HOST_TOKEN_ARG=' + urllib.parse.quote_plus(self.token) - ] - - def start(self): - ''' - Starts the HTTP endpoint as a child process - ''' - self.endpoint = multiprocessing.Process( - target = CredentialEndpoint._endpoint, - args=(self.username, self.password, self.token) - ) - self.endpoint.start() - time.sleep(2) - - def stop(self): - ''' - Stops the HTTP endpoint child process - ''' - self.endpoint.terminate() - self.endpoint.join() - - @staticmethod - def _endpoint(username, password, token): - ''' - Implements a HTTP endpoint to provide Git credentials to Docker containers - ''' - server = Flask(__name__) - - # Disable the first-run banner message - os.environ['WERKZEUG_RUN_MAIN'] = 'true' - - # Disable Flask log output (from ) - log = logging.getLogger('werkzeug') - log.setLevel(logging.ERROR) - - @server.route('/', methods=['POST']) - def credentials(): - if 'token' in request.args and request.args['token'] == token: - prompt = request.data.decode('utf-8') - return password if "Password for" in prompt else username - else: - return 'Invalid security token' - - server.run(host='0.0.0.0', port=9876) diff --git a/ue4docker/infrastructure/DarwinUtils.py b/ue4docker/infrastructure/DarwinUtils.py deleted file mode 100644 index b61e0e71..00000000 --- a/ue4docker/infrastructure/DarwinUtils.py +++ /dev/null @@ -1,41 +0,0 @@ -from .PackageUtils import PackageUtils -import os, platform - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - -class DarwinUtils(object): - - @staticmethod - def minimumRequiredVersion(): - ''' - Returns the minimum required version of macOS, which is 10.10.3 Yosemite - - (10.10.3 is the minimum required version for Docker for Mac, as per: - ) - ''' - return '10.10.3' - - @staticmethod - def systemString(): - ''' - Generates a human-readable version string for the macOS host system - ''' - return 'macOS {} (Kernel Version {})'.format( - platform.mac_ver()[0], - platform.uname().release - ) - - @staticmethod - def getMacOsVersion(): - ''' - Returns the version number for the macOS host system - ''' - return platform.mac_ver()[0] - - @staticmethod - def isSupportedMacOsVersion(): - ''' - Verifies that the macOS host system meets our minimum version requirements - ''' - return semver.compare(DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion()) >= 0 diff --git a/ue4docker/infrastructure/DockerUtils.py b/ue4docker/infrastructure/DockerUtils.py deleted file mode 100644 index 9f3e61c4..00000000 --- a/ue4docker/infrastructure/DockerUtils.py +++ /dev/null @@ -1,184 +0,0 @@ -import docker, fnmatch, humanfriendly, itertools, json, logging, os, platform, re -from .FilesystemUtils import FilesystemUtils - -class DockerUtils(object): - - @staticmethod - def installed(): - ''' - Determines if Docker is installed - ''' - try: - return DockerUtils.version() is not None - except Exception as e: - logging.debug(str(e)) - return False - - @staticmethod - def version(): - ''' - Retrieves the version information for the Docker daemon - ''' - client = docker.from_env() - return client.version() - - @staticmethod - def info(): - ''' - Retrieves the system information as produced by `docker info` - ''' - client = docker.from_env() - return client.info() - - @staticmethod - def exists(name): - ''' - Determines if the specified image exists - ''' - client = docker.from_env() - try: - image = client.images.get(name) - return True - except: - return False - - @staticmethod - def build(tags, context, args): - ''' - Returns the `docker build` command to build an image - ''' - tagArgs = [['-t', tag] for tag in tags] - return ['docker', 'build'] + list(itertools.chain.from_iterable(tagArgs)) + [context] + args - - @staticmethod - def pull(image): - ''' - Returns the `docker pull` command to pull an image from a remote registry - ''' - return ['docker', 'pull', image] - - @staticmethod - def start(image, command, **kwargs): - ''' - Starts a container in a detached state and returns the container handle - ''' - client = docker.from_env() - return client.containers.run(image, command, detach=True, **kwargs) - - @staticmethod - def configFilePath(): - ''' - Returns the path to the Docker daemon configuration file under Windows - ''' - return '{}\\Docker\\config\\daemon.json'.format(os.environ['ProgramData']) - - @staticmethod - def getConfig(): - ''' - Retrieves and parses the Docker daemon configuration file under Windows - ''' - configPath = DockerUtils.configFilePath() - if os.path.exists(configPath) == True: - with open(configPath) as configFile: - return json.load(configFile) - - return {} - - @staticmethod - def setConfig(config): - ''' - Writes new values to the Docker daemon configuration file under Windows - ''' - configPath = DockerUtils.configFilePath() - with open(configPath, 'w') as configFile: - configFile.write(json.dumps(config)) - - @staticmethod - def maxsize(): - ''' - Determines the configured size limit (in GB) for Windows containers - ''' - if platform.system() != 'Windows': - return -1 - - config = DockerUtils.getConfig() - if 'storage-opts' in config: - sizes = [opt.replace('size=', '') for opt in config['storage-opts'] if 'size=' in opt] - if len(sizes) > 0: - return humanfriendly.parse_size(sizes[0]) / 1000000000 - - # The default limit on image size is 20GB - # (https://docs.microsoft.com/en-us/visualstudio/install/build-tools-container-issues) - return 20.0 - - @staticmethod - def listImages(tagFilter = None, filters = {}): - ''' - Retrieves the details for each image matching the specified filters - ''' - - # Retrieve the list of images matching the specified filters - client = docker.from_env() - images = client.images.list(filters=filters) - - # Apply our tag filter if one was specified - if tagFilter is not None: - images = [i for i in images if len(i.tags) > 0 and len(fnmatch.filter(i.tags, tagFilter)) > 0] - - return images - - @staticmethod - def exec(container, command, **kwargs): - ''' - Executes a command in a container returned by `DockerUtils.start()` and returns the output - ''' - result, output = container.exec_run(command, **kwargs) - if result is not None and result != 0: - container.stop() - raise RuntimeError( - 'Failed to run command {} in container. Process returned exit code {} with output: {}'.format( - command, - result, - output - ) - ) - - return output - - @staticmethod - def execMultiple(container, commands, **kwargs): - ''' - Executes multiple commands in a container returned by `DockerUtils.start()` - ''' - for command in commands: - DockerUtils.exec(container, command, **kwargs) - - @staticmethod - def injectPostRunMessage(dockerfile, platform, messageLines): - ''' - Injects the supplied message at the end of each RUN directive in the specified Dockerfile - ''' - - # Generate the `echo` command for each line of the message - prefix = 'echo.' if platform == 'windows' else "echo '" - suffix = '' if platform == 'windows' else "'" - echoCommands = ''.join([' && {}{}{}'.format(prefix, line, suffix) for line in messageLines]) - - # Read the Dockerfile contents and convert all line endings to \n - contents = FilesystemUtils.readFile(dockerfile) - contents = contents.replace('\r\n', '\n') - - # Determine the escape character for the Dockerfile - escapeMatch = re.search('#[\\s]*escape[\\s]*=[\\s]*([^\n])\n', contents) - escape = escapeMatch[1] if escapeMatch is not None else '\\' - - # Identify each RUN directive in the Dockerfile - runMatches = re.finditer('^RUN(.+?[^{}])\n'.format(re.escape(escape)), contents, re.DOTALL | re.MULTILINE) - if runMatches is not None: - for match in runMatches: - - # Append the `echo` commands to the directive - contents = contents.replace(match[0], 'RUN{}{}\n'.format(match[1], echoCommands)) - - # Write the modified contents back to the Dockerfile - FilesystemUtils.writeFile(dockerfile, contents) diff --git a/ue4docker/infrastructure/FilesystemUtils.py b/ue4docker/infrastructure/FilesystemUtils.py deleted file mode 100644 index 51a59e49..00000000 --- a/ue4docker/infrastructure/FilesystemUtils.py +++ /dev/null @@ -1,17 +0,0 @@ -class FilesystemUtils(object): - - @staticmethod - def readFile(filename): - ''' - Reads data from a file - ''' - with open(filename, 'rb') as f: - return f.read().decode('utf-8') - - @staticmethod - def writeFile(filename, data): - ''' - Writes data to a file - ''' - with open(filename, 'wb') as f: - f.write(data.encode('utf-8')) diff --git a/ue4docker/infrastructure/GlobalConfiguration.py b/ue4docker/infrastructure/GlobalConfiguration.py deleted file mode 100644 index a9b43577..00000000 --- a/ue4docker/infrastructure/GlobalConfiguration.py +++ /dev/null @@ -1,36 +0,0 @@ -from pkg_resources import parse_version -import os, requests - - -# The default namespace for our tagged container images -DEFAULT_TAG_NAMESPACE = 'adamrehn' - - -class GlobalConfiguration(object): - ''' - Manages access to the global configuration settings for ue4-docker itself - ''' - - @staticmethod - def getLatestVersion(): - ''' - Queries PyPI to determine the latest available release of ue4-docker - ''' - releases = [parse_version(release) for release in requests.get('https://pypi.org/pypi/ue4-docker/json').json()['releases']] - return sorted(releases)[-1] - - @staticmethod - def getTagNamespace(): - ''' - Returns the currently-configured namespace for container image tags - ''' - return os.environ.get('UE4DOCKER_TAG_NAMESPACE', DEFAULT_TAG_NAMESPACE) - - @staticmethod - def resolveTag(tag): - ''' - Resolves a Docker image tag with respect to our currently-configured namespace - ''' - - # If the specified tag already includes a namespace, simply return it unmodified - return tag if '/' in tag else '{}/{}'.format(GlobalConfiguration.getTagNamespace(), tag) diff --git a/ue4docker/infrastructure/ImageBuilder.py b/ue4docker/infrastructure/ImageBuilder.py deleted file mode 100644 index 55140662..00000000 --- a/ue4docker/infrastructure/ImageBuilder.py +++ /dev/null @@ -1,107 +0,0 @@ -from .DockerUtils import DockerUtils -from .GlobalConfiguration import GlobalConfiguration -import humanfriendly, os, subprocess, time - -class ImageBuilder(object): - - def __init__(self, root, platform, logger, rebuild=False, dryRun=False): - ''' - Creates an ImageBuilder for the specified build parameters - ''' - self.root = root - self.platform = platform - self.logger = logger - self.rebuild = rebuild - self.dryRun = dryRun - - def build(self, name, tags, args): - ''' - Builds the specified image if it doesn't exist or if we're forcing a rebuild - ''' - - # Inject our filesystem layer commit message after each RUN directive in the Dockerfile - dockerfile = os.path.join(self.context(name), 'Dockerfile') - DockerUtils.injectPostRunMessage(dockerfile, self.platform, [ - '', - 'RUN directive complete. Docker will now commit the filesystem layer to disk.', - 'Note that for large filesystem layers this can take quite some time.', - 'Performing filesystem layer commit...', - '' - ]) - - # Build the image if it doesn't already exist - imageTags = self._formatTags(name, tags) - self._processImage( - imageTags[0], - DockerUtils.build(imageTags, self.context(name), args), - 'build', - 'built' - ) - - def context(self, name): - ''' - Resolve the full path to the build context for the specified image - ''' - return os.path.join(self.root, os.path.basename(name), self.platform) - - def pull(self, image): - ''' - Pulls the specified image if it doesn't exist or if we're forcing a pull of a newer version - ''' - self._processImage( - image, - DockerUtils.pull(image), - 'pull', - 'pulled' - ) - - def willBuild(self, name, tags): - ''' - Determines if we will build the specified image, based on our build settings - ''' - imageTags = self._formatTags(name, tags) - return self._willProcess(imageTags[0]) - - def _formatTags(self, name, tags): - ''' - Generates the list of fully-qualified tags that we will use when building an image - ''' - return ['{}:{}'.format(GlobalConfiguration.resolveTag(name), tag) for tag in tags] - - def _willProcess(self, image): - ''' - Determines if we will build or pull the specified image, based on our build settings - ''' - return self.rebuild == True or DockerUtils.exists(image) == False - - def _processImage(self, image, command, actionPresentTense, actionPastTense): - ''' - Processes the specified image by running the supplied command if it doesn't exist (use rebuild=True to force processing) - ''' - - # Determine if we are processing the image - if self._willProcess(image) == False: - self.logger.info('Image "{}" exists and rebuild not requested, skipping {}.'.format(image, actionPresentTense)) - return - - # Determine if we are running in "dry run" mode - self.logger.action('{}ing image "{}"...'.format(actionPresentTense.capitalize(), image)) - if self.dryRun == True: - print(command) - self.logger.action('Completed dry run for image "{}".'.format(image), newline=False) - return - - # Attempt to process the image using the supplied command - startTime = time.time() - exitCode = subprocess.call(command) - endTime = time.time() - - # Determine if processing succeeded - if exitCode == 0: - self.logger.action('{} image "{}" in {}'.format( - actionPastTense.capitalize(), - image, - humanfriendly.format_timespan(endTime - startTime) - ), newline=False) - else: - raise RuntimeError('failed to {} image "{}".'.format(actionPresentTense, image)) diff --git a/ue4docker/infrastructure/ImageCleaner.py b/ue4docker/infrastructure/ImageCleaner.py deleted file mode 100644 index d876b518..00000000 --- a/ue4docker/infrastructure/ImageCleaner.py +++ /dev/null @@ -1,27 +0,0 @@ -from .DockerUtils import DockerUtils -import humanfriendly, os, subprocess, time - -class ImageCleaner(object): - - def __init__(self, logger): - self.logger = logger - - def clean(self, image, dryRun=False): - ''' - Removes the specified image - ''' - - # Determine if we are running in "dry run" mode - self.logger.action('Removing image "{}"...'.format(image)) - cleanCommand = ['docker', 'rmi', image] - if dryRun == True: - print(cleanCommand) - else: - subprocess.call(cleanCommand) - - def cleanMultiple(self, images, dryRun=False): - ''' - Removes all of the images in the supplied list - ''' - for image in images: - self.clean(image, dryRun) diff --git a/ue4docker/infrastructure/Logger.py b/ue4docker/infrastructure/Logger.py deleted file mode 100644 index fd9db238..00000000 --- a/ue4docker/infrastructure/Logger.py +++ /dev/null @@ -1,33 +0,0 @@ -from termcolor import colored -import colorama, sys - -class Logger(object): - - def __init__(self, prefix=''): - ''' - Creates a logger that will print coloured output to stderr - ''' - colorama.init() - self.prefix = prefix - - def action(self, output, newline=True): - ''' - Prints information about an action that is being performed - ''' - self._print('green', output, newline) - - def error(self, output, newline=False): - ''' - Prints information about an error that has occurred - ''' - self._print('red', output, newline) - - def info(self, output, newline=True): - ''' - Prints information that does not pertain to an action or an error - ''' - self._print('yellow', output, newline) - - def _print(self, colour, output, newline): - whitespace = '\n' if newline == True else '' - print(colored(whitespace + self.prefix + output, color=colour), file=sys.stderr) diff --git a/ue4docker/infrastructure/NetworkUtils.py b/ue4docker/infrastructure/NetworkUtils.py deleted file mode 100644 index acba2116..00000000 --- a/ue4docker/infrastructure/NetworkUtils.py +++ /dev/null @@ -1,19 +0,0 @@ -import socket - -class NetworkUtils(object): - - @staticmethod - def hostIP(): - ''' - Determines the IP address of the host - ''' - # Code from - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - try: - s.connect(('10.255.255.255', 1)) - IP = s.getsockname()[0] - except: - IP = '127.0.0.1' - finally: - s.close() - return IP diff --git a/ue4docker/infrastructure/PackageUtils.py b/ue4docker/infrastructure/PackageUtils.py deleted file mode 100644 index 0948e3ad..00000000 --- a/ue4docker/infrastructure/PackageUtils.py +++ /dev/null @@ -1,20 +0,0 @@ -import importlib.util, pkg_resources - -class PackageUtils(object): - - @staticmethod - def getPackageLocation(package): - ''' - Attempts to retrieve the filesystem location for the specified Python package - ''' - return pkg_resources.get_distribution(package).location - - @staticmethod - def importFile(moduleName, filePath): - ''' - Directly imports a Python module from a source file - ''' - spec = importlib.util.spec_from_file_location(moduleName, filePath) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module diff --git a/ue4docker/infrastructure/PrettyPrinting.py b/ue4docker/infrastructure/PrettyPrinting.py deleted file mode 100644 index 03a5a54d..00000000 --- a/ue4docker/infrastructure/PrettyPrinting.py +++ /dev/null @@ -1,15 +0,0 @@ -class PrettyPrinting(object): - - @staticmethod - def printColumns(pairs, indent = 2, minSpaces = 6): - ''' - Prints a list of paired values in two nicely aligned columns - ''' - - # Determine the length of the longest item in the left-hand column - longestName = max([len(pair[0]) for pair in pairs]) - - # Print the two columns - for pair in pairs: - whitespace = ' ' * ((longestName + minSpaces) - len(pair[0])) - print('{}{}{}{}'.format(' ' * indent, pair[0], whitespace, pair[1])) diff --git a/ue4docker/infrastructure/SubprocessUtils.py b/ue4docker/infrastructure/SubprocessUtils.py deleted file mode 100644 index 00805628..00000000 --- a/ue4docker/infrastructure/SubprocessUtils.py +++ /dev/null @@ -1,54 +0,0 @@ -import subprocess - - -class VerboseCalledProcessError(RuntimeError): - '''' - A verbose wrapper for `subprocess.CalledProcessError` that prints stdout and stderr - ''' - - def __init__(self, wrapped): - self.wrapped = wrapped - - def __str__(self): - return '{}\nstdout: {}\nstderr: {}'.format( - self.wrapped, - self.wrapped.output, - self.wrapped.stderr - ) - - -class SubprocessUtils(object): - - @staticmethod - def extractLines(output): - ''' - Extracts the individual lines from the output of a child process - ''' - return output.decode('utf-8').replace('\r\n', '\n').strip().split('\n') - - @staticmethod - def capture(command, check = True, **kwargs): - ''' - Executes a child process and captures its output. - - If the child process fails and `check` is True then a verbose exception will be raised. - ''' - try: - return subprocess.run( - command, - stdout = subprocess.PIPE, - stderr = subprocess.PIPE, - check = check, - **kwargs - ) - except subprocess.CalledProcessError as e: - raise VerboseCalledProcessError(e) from None - - @staticmethod - def run(command, check = True, **kwargs): - ''' - Executes a child process. - - If the child process fails and `check` is True then a verbose exception will be raised. - ''' - return SubprocessUtils.capture(command, check, **kwargs) diff --git a/ue4docker/infrastructure/WindowsUtils.py b/ue4docker/infrastructure/WindowsUtils.py deleted file mode 100644 index 1790275f..00000000 --- a/ue4docker/infrastructure/WindowsUtils.py +++ /dev/null @@ -1,158 +0,0 @@ -from .PackageUtils import PackageUtils -import os, platform - -if platform.system() == 'Windows': - import winreg - -# Import the `semver` package even when the conflicting `node-semver` package is present -semver = PackageUtils.importFile('semver', os.path.join(PackageUtils.getPackageLocation('semver'), 'semver.py')) - -class WindowsUtils(object): - - # The latest Windows build version we recognise as a non-Insider build - _latestReleaseBuild = 17763 - - # The list of Windows Server Core base image tags that we support, in ascending version number order - _validTags = ['ltsc2016', '1709', '1803', 'ltsc2019'] - - @staticmethod - def _getVersionRegKey(subkey): - ''' - Retrieves the specified Windows version key from the registry - ''' - key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, 'SOFTWARE\\Microsoft\\Windows NT\\CurrentVersion') - value = winreg.QueryValueEx(key, subkey) - winreg.CloseKey(key) - return value[0] - - @staticmethod - def requiredHostDlls(basetag): - ''' - Returns the list of required host DLL files for the specified container image base tag - ''' - - # `ddraw.dll` is only required under Windows Server 2016 version 1607 - common = ['dsound.dll', 'opengl32.dll', 'glu32.dll'] - return ['ddraw.dll'] + common if basetag == 'ltsc2016' else common - - @staticmethod - def requiredSizeLimit(): - ''' - Returns the minimum required image size limit (in GB) for Windows containers - ''' - return 200.0 - - @staticmethod - def minimumRequiredVersion(): - ''' - Returns the minimum required version of Windows 10 / Windows Server, which is release 1607 - - (1607 is the first build to support Windows containers, as per: - ) - ''' - return '10.0.14393' - - @staticmethod - def systemStringShort(): - ''' - Generates a concise human-readable version string for the Windows host system - ''' - return 'Windows {} version {}'.format( - 'Server' if WindowsUtils.isWindowsServer() else '10', - WindowsUtils.getWindowsRelease() - ) - - @staticmethod - def systemStringLong(): - ''' - Generates a verbose human-readable version string for the Windows host system - ''' - return '{} Version {} (OS Build {}.{})'.format( - WindowsUtils._getVersionRegKey('ProductName'), - WindowsUtils.getWindowsRelease(), - WindowsUtils.getWindowsVersion()['patch'], - WindowsUtils._getVersionRegKey('UBR') - ) - - @staticmethod - def getWindowsVersion(): - ''' - Returns the version information for the Windows host system as a semver instance - ''' - return semver.parse(platform.win32_ver()[1]) - - @staticmethod - def getWindowsRelease(): - ''' - Determines the Windows 10 / Windows Server release (1607, 1709, 1803, etc.) of the Windows host system - ''' - return WindowsUtils._getVersionRegKey('ReleaseId') - - @staticmethod - def getWindowsBuild(): - ''' - Returns the full Windows version number as a string, including the build number - ''' - version = platform.win32_ver()[1] - build = WindowsUtils._getVersionRegKey('BuildLabEx').split('.')[1] - return '{}.{}'.format(version, build) - - @staticmethod - def isSupportedWindowsVersion(): - ''' - Verifies that the Windows host system meets our minimum Windows version requirements - ''' - return semver.compare(platform.win32_ver()[1], WindowsUtils.minimumRequiredVersion()) >= 0 - - @staticmethod - def isWindowsServer(): - ''' - Determines if the Windows host system is Windows Server - ''' - return 'Windows Server' in WindowsUtils._getVersionRegKey('ProductName') - - @staticmethod - def isInsiderPreview(): - ''' - Determines if the Windows host system is a Windows Insider preview build - ''' - version = WindowsUtils.getWindowsVersion() - return version['patch'] > WindowsUtils._latestReleaseBuild - - @staticmethod - def getReleaseBaseTag(release): - ''' - Retrieves the tag for the Windows Server Core base image matching the specified Windows 10 / Windows Server release - ''' - - # For Windows Insider preview builds, build the latest release tag - if WindowsUtils.isInsiderPreview(): - return WindowsUtils._validTags[-1] - - # This lookup table is based on the list of valid tags from - return { - '1709': '1709', - '1803': '1803', - '1809': 'ltsc2019' - }.get(release, 'ltsc2016') - - @staticmethod - def getValidBaseTags(): - ''' - Returns the list of valid tags for the Windows Server Core base image, in ascending chronological release order - ''' - return WindowsUtils._validTags - - @staticmethod - def isValidBaseTag(tag): - ''' - Determines if the specified tag is a valid Windows Server Core base image tag - ''' - return tag in WindowsUtils._validTags - - @staticmethod - def isNewerBaseTag(older, newer): - ''' - Determines if the base tag `newer` is chronologically newer than the base tag `older` - ''' - return WindowsUtils._validTags.index(newer) > WindowsUtils._validTags.index(older) diff --git a/ue4docker/main.py b/ue4docker/main.py deleted file mode 100644 index 3b7b5ef8..00000000 --- a/ue4docker/main.py +++ /dev/null @@ -1,88 +0,0 @@ -from .infrastructure import DarwinUtils, DockerUtils, Logger, PrettyPrinting, WindowsUtils -from .build import build -from .clean import clean -from .export import export -from .info import info -from .setup_cmd import setup -from .version_cmd import version -import logging, os, platform, sys - -def _exitWithError(err): - Logger().error(err) - sys.exit(1) - -def main(): - - # Configure verbose logging if the user requested it - # (NOTE: in a future version of ue4-docker the `Logger` class will be properly integrated with standard logging) - if '-v' in sys.argv or '--verbose' in sys.argv: - sys.argv = list([arg for arg in sys.argv if arg not in ['-v', '--verbose']]) - logging.getLogger().setLevel(logging.DEBUG) - - # Verify that Docker is installed - if DockerUtils.installed() == False: - _exitWithError('Error: could not detect Docker daemon version. Please ensure Docker is installed.') - - # Under Windows, verify that the host is a supported version - if platform.system() == 'Windows' and WindowsUtils.isSupportedWindowsVersion() == False: - _exitWithError('Error: the detected version of Windows ({}) is not supported. Windows 10 / Windows Server version 1607 or newer is required.'.format(platform.win32_ver()[1])) - - # Under macOS, verify that the host is a supported version - if platform.system() == 'Darwin' and DarwinUtils.isSupportedMacOsVersion() == False: - _exitWithError('Error: the detected version of macOS ({}) is not supported. macOS {} or newer is required.'.format(DarwinUtils.getMacOsVersion(), DarwinUtils.minimumRequiredVersion())) - - # Our supported commands - COMMANDS = { - 'build': { - 'function': build, - 'description': 'Builds container images for a specific version of UE4' - }, - 'clean': { - 'function': clean, - 'description': 'Cleans built container images' - }, - 'export': { - 'function': export, - 'description': 'Exports components from built container images to the host system' - }, - 'info': { - 'function': info, - 'description': 'Displays information about the host system and Docker daemon' - }, - 'setup': { - 'function': setup, - 'description': 'Automatically configures the host system where possible' - }, - 'version': { - 'function': version, - 'description': 'Prints the ue4-docker version number' - } - } - - # Truncate argv[0] to just the command name without the full path - sys.argv[0] = os.path.basename(sys.argv[0]) - - # Determine if a command has been specified - if len(sys.argv) > 1: - - # Verify that the specified command is valid - command = sys.argv[1] - if command not in COMMANDS: - print('Error: unrecognised command "{}".'.format(command), file=sys.stderr) - sys.exit(1) - - # Invoke the command - sys.argv = [sys.argv[0]] + sys.argv[2:] - COMMANDS[command]['function']() - - else: - - # Print usage syntax - print('Usage: {} COMMAND [OPTIONS]\n'.format(sys.argv[0])) - print('Windows and Linux containers for Unreal Engine 4\n') - print('Commands:') - PrettyPrinting.printColumns([ - (command, COMMANDS[command]['description']) - for command in COMMANDS - ]) - print('\nRun `{} COMMAND --help` for more information on a command.'.format(sys.argv[0])) diff --git a/ue4docker/setup_cmd.py b/ue4docker/setup_cmd.py deleted file mode 100644 index 95958f8f..00000000 --- a/ue4docker/setup_cmd.py +++ /dev/null @@ -1,152 +0,0 @@ -import docker, os, platform, requests, shutil, subprocess, sys -from .infrastructure import * - -# Runs a command without displaying its output and returns the exit code -def _runSilent(command): - result = SubprocessUtils.capture(command, check=False) - return result.returncode - -# Performs setup for Linux hosts -def _setupLinux(): - - # Pull the latest version of the Alpine container image - alpineImage = 'alpine:latest' - SubprocessUtils.capture(['docker', 'pull', alpineImage]) - - # Start the credential endpoint with blank credentials - endpoint = CredentialEndpoint('', '') - endpoint.start() - - try: - - # Run an Alpine container to see if we can access the host port for the credential endpoint - SubprocessUtils.capture([ - 'docker', 'run', '--rm', alpineImage, - 'wget', '--timeout=1', '--post-data=dummy', 'http://{}:9876'.format(NetworkUtils.hostIP()) - ], check=True) - - # If we reach this point then the host port is accessible - print('No firewall configuration required.') - - except: - - # The host port is blocked, so we need to perform firewall configuration - print('Creating firewall rule for credential endpoint...') - - # Create the firewall rule - subprocess.run(['iptables', '-A', 'INPUT', '-p', 'tcp', '--dport', '9876', '-j', 'ACCEPT'], check=True) - - # Ensure the firewall rule persists after reboot - # (Requires the `iptables-persistent` service to be installed and running) - os.makedirs('/etc/iptables', exist_ok=True) - subprocess.run('iptables-save > /etc/iptables/rules.v4', shell=True, check=True) - - # Inform users of the `iptables-persistent` requirement - print('Firewall rule created. Note that the `iptables-persistent` service will need to') - print('be installed for the rule to persist after the host system reboots.') - - finally: - - # Stop the credential endpoint - endpoint.stop() - -# Performs setup for Windows Server hosts -def _setupWindowsServer(): - - # Check if we need to configure the maximum image size - requiredLimit = WindowsUtils.requiredSizeLimit() - if DockerUtils.maxsize() < requiredLimit: - - # Attempt to stop the Docker daemon - print('Stopping the Docker daemon...') - subprocess.run(['sc.exe', 'stop', 'docker'], check=True) - - # Attempt to set the maximum image size - print('Setting maximum image size to {}GB...'.format(requiredLimit)) - config = DockerUtils.getConfig() - sizeOpt = 'size={}GB'.format(requiredLimit) - if 'storage-opts' in config: - config['storage-opts'] = list([o for o in config['storage-opts'] if o.lower().startswith('size=') == False]) - config['storage-opts'].append(sizeOpt) - else: - config['storage-opts'] = [sizeOpt] - DockerUtils.setConfig(config) - - # Attempt to start the Docker daemon - print('Starting the Docker daemon...') - subprocess.run(['sc.exe', 'start', 'docker'], check=True) - - else: - print('Maximum image size is already correctly configured.') - - # Determine if we need to configure Windows firewall - ruleName = 'Open TCP port 9876 for ue4-docker credential endpoint' - ruleExists = _runSilent(['netsh', 'advfirewall', 'firewall', 'show', 'rule', 'name={}'.format(ruleName)]) == 0 - if ruleExists == False: - - # Add a rule to ensure Windows firewall allows access to the credential helper from our containers - print('Creating firewall rule for credential endpoint...') - subprocess.run([ - 'netsh', 'advfirewall', - 'firewall', 'add', 'rule', - 'name={}'.format(ruleName), 'dir=in', 'action=allow', 'protocol=TCP', 'localport=9876' - ], check=True) - - else: - print('Firewall rule for credential endpoint is already configured.') - - # Determine if the host system is Windows Server Core and lacks the required DLL files for building our containers - hostRelease = WindowsUtils.getWindowsRelease() - requiredDLLs = WindowsUtils.requiredHostDlls(hostRelease) - dllDir = os.path.join(os.environ['SystemRoot'], 'System32') - existing = [dll for dll in requiredDLLs if os.path.exists(os.path.join(dllDir, dll))] - if len(existing) != len(requiredDLLs): - - # Determine if we can extract DLL files from the full Windows base image (version 1809 and newer only) - tags = requests.get('https://mcr.microsoft.com/v2/windows/tags/list').json()['tags'] - if hostRelease in tags: - - # Pull the full Windows base image with the appropriate tag if it does not already exist - image = 'mcr.microsoft.com/windows:{}'.format(hostRelease) - print('Pulling full Windows base image "{}"...'.format(image)) - subprocess.run(['docker', 'pull', image], check=True) - - # Start a container from which we will copy the DLL files, bind-mounting our DLL destination directory - print('Starting a container to copy DLL files from...') - mountPath = 'C:\\dlldir' - container = DockerUtils.start( - image, - ['timeout', '/t', '99999', '/nobreak'], - mounts = [docker.types.Mount(mountPath, dllDir, 'bind')], - stdin_open = True, - tty = True, - remove = True - ) - - # Copy the DLL files to the host - print('Copying DLL files to the host system...') - DockerUtils.execMultiple(container, [['xcopy', '/y', os.path.join(dllDir, dll), mountPath + '\\'] for dll in requiredDLLs]) - - # Stop the container - print('Stopping the container...') - container.stop() - - else: - print('The following DLL files will need to be manually copied into {}:'.format(dllDir)) - print('\n'.join(['- {}'.format(dll) for dll in requiredDLLs if dll not in existing])) - - else: - print('All required DLL files are already present on the host system.') - -def setup(): - - # We don't currently support auto-config for VM-based containers - if platform.system() == 'Darwin' or (platform.system() == 'Windows' and WindowsUtils.isWindowsServer() == False): - print('Manual configuration is required under Windows 10 and macOS. Automatic configuration is not available.') - return - - # Perform setup based on the host system type - if platform.system() == 'Linux': - _setupLinux() - else: - _setupWindowsServer() diff --git a/ue4docker/version.py b/ue4docker/version.py deleted file mode 100644 index df5521bd..00000000 --- a/ue4docker/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '0.0.36'