From 8480508bc4d93b4340bb30701c629090595ded70 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Wed, 29 Jan 2025 10:27:13 +0100 Subject: [PATCH] fix(flake.nix): limit the amount of maximum layers to 32 on dogfood nix image Change-Id: I7d34f5d50509bfa1b5e8f6ed9e150462151fbbe0 Signed-off-by: Thomas Kosiewski --- flake.nix | 15 ++- nix/docker.nix | 246 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 nix/docker.nix diff --git a/flake.nix b/flake.nix index f4654ccb378b1..8503c55cb17d7 100644 --- a/flake.nix +++ b/flake.nix @@ -136,6 +136,8 @@ zstd ]; + docker = pkgs.callPackage ./nix/docker.nix { }; + # buildSite packages the site directory. buildSite = pnpm2nix.packages.${system}.mkPnpmPackage { inherit nodejs pnpm; @@ -237,12 +239,21 @@ aarch64-windows = buildFat "windows_arm64.exe"; } // (pkgs.lib.optionalAttrs pkgs.stdenv.isLinux { - dev_image = pkgs.dockerTools.buildNixShellImage { + dev_image = docker.buildNixShellImage { name = "codercom/oss-dogfood-nix"; tag = "latest-${system}"; + maxLayers = 32; + drv = devShells.default.overrideAttrs (oldAttrs: { - buildInputs = oldAttrs.buildInputs ++ [ pkgs.nix ]; + # (ThomasK33): Workaround for images with too many layers (>64 layers) causing sysbox + # to have issues on dogfood envs. + buildInputs = + oldAttrs.buildInputs + ++ (with pkgs; [ + nix + coreutils + ]); }); }; }); diff --git a/nix/docker.nix b/nix/docker.nix new file mode 100644 index 0000000000000..226813e761f5f --- /dev/null +++ b/nix/docker.nix @@ -0,0 +1,246 @@ +# (ThomasK33): Inlined the relevant dockerTools functions, so that we can +# set the maxLayers attribute on the attribute set passed +# to the buildNixShellImage function. +# +# I'll create an upstream PR to nixpkgs with those changes, making this +# eventually unnecessary and ripe for removal. +{ + lib, + dockerTools, + devShellTools, + bashInteractive, + fakeNss, + runCommand, + writeShellScriptBin, + writeText, + cacert, + storeDir ? builtins.storeDir, + pigz, + zstd, +}: +let + inherit (lib) + optionalString + ; + + inherit (devShellTools) + valueToString + ; + + inherit (dockerTools) + streamLayeredImage + binSh + usrBinEnv + ; + + compressors = { + none = { + ext = ""; + nativeInputs = [ ]; + compress = "cat"; + decompress = "cat"; + }; + gz = { + ext = ".gz"; + nativeInputs = [ pigz ]; + compress = "pigz -p$NIX_BUILD_CORES -nTR"; + decompress = "pigz -d -p$NIX_BUILD_CORES"; + }; + zstd = { + ext = ".zst"; + nativeInputs = [ zstd ]; + compress = "zstd -T$NIX_BUILD_CORES"; + decompress = "zstd -d -T$NIX_BUILD_CORES"; + }; + }; + compressorForImage = + compressor: imageName: + compressors.${compressor} + or (throw "in docker image ${imageName}: compressor must be one of: [${toString builtins.attrNames compressors}]"); + + streamNixShellImage = + { + drv, + name ? drv.name + "-env", + tag ? null, + uid ? 1000, + gid ? 1000, + homeDirectory ? "/build", + shell ? bashInteractive + "/bin/bash", + command ? null, + run ? null, + maxLayers ? 100, + }: + assert lib.assertMsg (!(drv.drvAttrs.__structuredAttrs or false)) + "streamNixShellImage: Does not work with the derivation ${drv.name} because it uses __structuredAttrs"; + assert lib.assertMsg ( + command == null || run == null + ) "streamNixShellImage: Can't specify both command and run"; + let + + # A binary that calls the command to build the derivation + builder = writeShellScriptBin "buildDerivation" '' + exec ${lib.escapeShellArg (valueToString drv.drvAttrs.builder)} ${lib.escapeShellArgs (map valueToString drv.drvAttrs.args)} + ''; + + staticPath = "${dirOf shell}:${lib.makeBinPath [ builder ]}"; + + # https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L493-L526 + rcfile = writeText "nix-shell-rc" '' + unset PATH + dontAddDisableDepTrack=1 + # TODO: https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L506 + [ -e $stdenv/setup ] && source $stdenv/setup + PATH=${staticPath}:"$PATH" + SHELL=${lib.escapeShellArg shell} + BASH=${lib.escapeShellArg shell} + set +e + [ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] ' + if [ "$(type -t runHook)" = function ]; then + runHook shellHook + fi + unset NIX_ENFORCE_PURITY + shopt -u nullglob + shopt -s execfail + ${optionalString (command != null || run != null) '' + ${optionalString (command != null) command} + ${optionalString (run != null) run} + exit + ''} + ''; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/globals.hh#L464-L465 + sandboxBuildDir = "/build"; + + drvEnv = + devShellTools.unstructuredDerivationInputEnv { inherit (drv) drvAttrs; } + // devShellTools.derivationOutputEnv { + outputList = drv.outputs; + outputMap = drv; + }; + + # Environment variables set in the image + envVars = + { + + # Root certificates for internet access + SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + NIX_SSL_CERT_FILE = "${cacert}/etc/ssl/certs/ca-bundle.crt"; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1027-L1030 + # PATH = "/path-not-set"; + # Allows calling bash and `buildDerivation` as the Cmd + PATH = staticPath; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1032-L1038 + HOME = homeDirectory; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1040-L1044 + NIX_STORE = storeDir; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1046-L1047 + # TODO: Make configurable? + NIX_BUILD_CORES = "1"; + + } + // drvEnv + // { + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1008-L1010 + NIX_BUILD_TOP = sandboxBuildDir; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1012-L1013 + TMPDIR = sandboxBuildDir; + TEMPDIR = sandboxBuildDir; + TMP = sandboxBuildDir; + TEMP = sandboxBuildDir; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1015-L1019 + PWD = sandboxBuildDir; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1071-L1074 + # We don't set it here because the output here isn't handled in any special way + # NIX_LOG_FD = "2"; + + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L1076-L1077 + TERM = "xterm-256color"; + }; + + in + streamLayeredImage { + inherit name tag maxLayers; + contents = [ + binSh + usrBinEnv + (fakeNss.override { + # Allows programs to look up the build user's home directory + # https://github.com/NixOS/nix/blob/ffe155abd36366a870482625543f9bf924a58281/src/libstore/build/local-derivation-goal.cc#L906-L910 + # Slightly differs however: We use the passed-in homeDirectory instead of sandboxBuildDir. + # We're doing this because it's arguably a bug in Nix that sandboxBuildDir is used here: https://github.com/NixOS/nix/issues/6379 + extraPasswdLines = [ + "nixbld:x:${toString uid}:${toString gid}:Build user:${homeDirectory}:/noshell" + ]; + extraGroupLines = [ + "nixbld:!:${toString gid}:" + ]; + }) + ]; + + fakeRootCommands = '' + # Effectively a single-user installation of Nix, giving the user full + # control over the Nix store. Needed for building the derivation this + # shell is for, but also in case one wants to use Nix inside the + # image + mkdir -p ./nix/{store,var/nix} ./etc/nix + chown -R ${toString uid}:${toString gid} ./nix ./etc/nix + + # Gives the user control over the build directory + mkdir -p .${sandboxBuildDir} + chown -R ${toString uid}:${toString gid} .${sandboxBuildDir} + ''; + + # Run this image as the given uid/gid + config.User = "${toString uid}:${toString gid}"; + config.Cmd = + # https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L185-L186 + # https://github.com/NixOS/nix/blob/2.8.0/src/nix-build/nix-build.cc#L534-L536 + if run == null then + [ + shell + "--rcfile" + rcfile + ] + else + [ + shell + rcfile + ]; + config.WorkingDir = sandboxBuildDir; + config.Env = lib.mapAttrsToList (name: value: "${name}=${value}") envVars; + }; +in +{ + + # This function streams a docker image that behaves like a nix-shell for a derivation + # Docs: doc/build-helpers/images/dockertools.section.md + # Tests: nixos/tests/docker-tools-nix-shell.nix + + # Wrapper around streamNixShellImage to build an image from the result + # Docs: doc/build-helpers/images/dockertools.section.md + # Tests: nixos/tests/docker-tools-nix-shell.nix + buildNixShellImage = + { + drv, + compressor ? "gz", + ... + }@args: + let + stream = streamNixShellImage (builtins.removeAttrs args [ "compressor" ]); + compress = compressorForImage compressor drv.name; + in + runCommand "${drv.name}-env.tar${compress.ext}" { + inherit (stream) imageName; + passthru = { inherit (stream) imageTag; }; + nativeBuildInputs = compress.nativeInputs; + } "${stream} | ${compress.compress} > $out"; +}