celonis · OliLay · Jul 20, 2022 · Jul 12, 2022 · Jul 12, 2022 · Jul 18, 2022
diff --git a/.github/workflows/build-main.yml b/.github/workflows/build-main.yml
@@ -70,8 +70,13 @@ jobs:
           sudo debootstrap focal /var/chroot/focal http://archive.ubuntu.com/ubuntu
           sudo ./.github/setup_chroot_env.sh focal $(id -nG | sed 's/ /,/g')
           sudo schroot -c focal -- apt-get -y install g++
+      - name: Install docker
+        uses: docker-practice/actions-setup-docker@master
+      - name: Setup docker container
+        run: |
+          docker run -dit --name gcc -v /tmp:/tmp gcc:bullseye
       - name: Run tests
-        run: pytest -v -rfE --cov=homcc --capture=tee-sys --runschroot=focal
+        run: pytest -v -rfE --cov=homcc --capture=tee-sys --runschroot=focal --rundocker=gcc
   build:
     runs-on: ubuntu-22.04
     steps:

diff --git a/.pylintrc b/.pylintrc
@@ -76,7 +76,7 @@ disable=abstract-method,
         global-statement,
         hex-method,
         idiv-method,
-        implicit-str-concat-in-sequence,
+        implicit-str-concat,
         import-error,
         import-self,
         import-star-module-level,
@@ -155,12 +155,6 @@ disable=abstract-method,
 # mypackage.mymodule.MyReporterClass.
 output-format=text
 
-# Put messages in a separate file for each module / package specified on the
-# command line instead of printing them on stdout. Reports (if any) will be
-# written in a file name "pylint_global.[txt|html]". This option is deprecated
-# and it will be removed in Pylint 2.0.
-files-output=no
-
 # Tells whether to display a full report or only the messages
 reports=no
 
@@ -279,12 +273,6 @@ ignore-long-lines=(?x)(
 # else.
 single-line-if-stmt=yes
 
-# List of optional constructs for which whitespace checking is disabled. `dict-
-# separator` is used to allow tabulation in dicts, etc.: {1  : 1,\n222: 2}.
-# `trailing-comma` allows a space between comma and closing bracket: (a, ).
-# `empty-line` allows space-only lines.
-no-space-check=
-
 # Maximum number of lines in a module
 max-module-lines=99999
 

diff --git a/README.md b/README.md
@@ -36,6 +36,7 @@ Additionally, `HOMCC` provides sandboxed compiler execution for remote compilati
    4. [Formatting](#formatting)
    5. [Build Debian packages](#build-debian-packages)
    6. [`schroot` testing setup for Debian systems](#schroot-testing-setup-for-debian-systems)
+   7. [`docker` testing setup](#docker-testing-setup)
 
 ---
 
@@ -115,12 +116,19 @@ Additionally, `HOMCC` provides sandboxed compiler execution for remote compilati
   ```sh
   $ homccd --help
   ```
-- \[Optional]:
-  Set up your `schroot` environments at `/etc/schroot/schroot.conf` or in the `/etc/schroot/chroot.d/` directory and mount the `/tmp/` directory to enable sandboxed compiler execution.
+- \[Optional] Sandboxed execution:
+  - `schroot`: Set up your `schroot` environments at `/etc/schroot/schroot.conf` or in the `/etc/schroot/chroot.d/` directory and mount the `/tmp/` directory to enable sandboxed compiler execution.
   Currently, in order for these changes to apply, you have to restart `homccd`:
-  ```sh
-  $ sudo systemctl restart homccd.service
-  ```
+    ```sh
+    $ sudo systemctl restart homccd.service
+    ```
+  - `docker`:
+    - Make sure that the docker containers that you want to compile in have mounted the host's `/tmp` directory to `/tmp` (this is necessary to access cached dependencies):
+      ```sh
+      $ sudo docker run --name example_container -v /tmp:/tmp -dit ubuntu:22.04
+      ```
+    - Make sure the docker containers you want to compile in are running and have the appropriate compilers installed
+
 
 
 ## Configuration
@@ -143,7 +151,8 @@ Additionally, `HOMCC` provides sandboxed compiler execution for remote compilati
     compiler=g++
     timeout=60
     compression=lzo
-    profile=jammy
+    schroot_profile=jammy
+    docker_container=example_container
     log_level=DEBUG
     verbose=True
     [homccd]
@@ -159,6 +168,7 @@ Additionally, `HOMCC` provides sandboxed compiler execution for remote compilati
     Default timeout value in seconds
     Default compression algorithm: {lzo, lzma}
     Profile to specify the schroot environment for remote compilations
+    Docker container that should be used on the server for remote compilations
     Detail level for log messages: {DEBUG, INFO, WARNING, ERROR, CRITICAL}
     Enable verbosity mode which implies detailed and colored logging
     # Server configuration
@@ -264,8 +274,18 @@ Additionally, `HOMCC` provides sandboxed compiler execution for remote compilati
   ```sh
   $ sudo schroot -c jammy -- apt -y install build-essential
   ```
-- Execute *schrooted* compilations by specifying `--profile=jammy` via the CLI or in the `client.conf` file
+- Execute *schrooted* compilations by specifying `--schroot-profile=jammy` via the CLI or in the `homcc.conf` file
 - Execute all tests in `./tests/` and perform test coverage:
   ```sh
   $ pytest -v -rfEs --cov=homcc --runschroot=jammy
   ```
+
+### `docker` testing setup
+- Create a docker container with a working `gcc` compiler, the easiest image to get is probably the official `gcc` docker image:
+  ```sh
+  docker run -dit --name gcc -v /tmp:/tmp gcc:bullseye
+  ```
+- Execute all tests (including the docker tests by specifying `--rundocker=gcc`) and perform test coverage:
+  ```sh
+  $ pytest -v -rfEs --cov=homcc --rundocker=gcc
+  ```
diff --git a/homcc/client/client.py b/homcc/client/client.py
@@ -411,10 +411,24 @@ async def _send(self, message: Message):
         await self._writer.drain()  # type: ignore[union-attr]
 
     async def send_argument_message(
-        self, arguments: Arguments, cwd: str, dependency_dict: Dict[str, str], profile: Optional[str]
+        self,
+        arguments: Arguments,
+        cwd: str,
+        dependency_dict: Dict[str, str],
+        schroot_profile: Optional[str],
+        docker_container: Optional[str],
     ):
         """send an argument message to homcc server"""
-        await self._send(ArgumentMessage(list(arguments), cwd, dependency_dict, profile, self.compression))
+        await self._send(
+            ArgumentMessage(
+                args=list(arguments),
+                cwd=cwd,
+                dependencies=dependency_dict,
+                schroot_profile=schroot_profile,
+                docker_container=docker_container,
+                compression=self.compression,
+            )
+        )
 
     async def send_dependency_reply_message(self, dependency: str):
         """send dependency reply message to homcc server"""

diff --git a/homcc/client/compilation.py b/homcc/client/compilation.py
@@ -54,14 +54,17 @@ async def compile_remotely(arguments: Arguments, hosts: List[Host], config: Clie
     # try to connect to 3 different remote hosts before falling back to local compilation
     for host in HostSelector(hosts, 3):
         timeout: float = config.timeout or DEFAULT_COMPILATION_REQUEST_TIMEOUT
-        profile: Optional[str] = config.profile
+        schroot_profile: Optional[str] = config.schroot_profile
+        docker_container: Optional[str] = config.docker_container
 
         # overwrite host compression if none was explicitly specified but provided via config
         host.compression = host.compression or config.compression
 
         try:
             with RemoteHostSemaphore(host):
-                return await asyncio.wait_for(compile_remotely_at(arguments, host, profile), timeout=timeout)
+                return await asyncio.wait_for(
+                    compile_remotely_at(arguments, host, schroot_profile, docker_container), timeout=timeout
+                )
 
         # remote semaphore could not be acquired
         except SlotsExhaustedError as error:
@@ -80,14 +83,22 @@ async def compile_remotely(arguments: Arguments, hosts: List[Host], config: Clie
     raise HostsExhaustedError(f"All hosts '{', '.join(str(host) for host in hosts)}' are exhausted.")
 
 
-async def compile_remotely_at(arguments: Arguments, host: Host, profile: Optional[str]) -> int:
+async def compile_remotely_at(
+    arguments: Arguments, host: Host, schroot_profile: Optional[str], docker_container: Optional[str]
+) -> int:
     """main function for the communication between client and a remote compilation host"""
 
     async with TCPClient(host) as client:
         dependency_dict: Dict[str, str] = calculate_dependency_dict(find_dependencies(arguments))
         remote_arguments: Arguments = arguments.copy().remove_local_args()
 
-        await client.send_argument_message(remote_arguments, os.getcwd(), dependency_dict, profile)
+        await client.send_argument_message(
+            arguments=remote_arguments,
+            cwd=os.getcwd(),
+            dependency_dict=dependency_dict,
+            schroot_profile=schroot_profile,
+            docker_container=docker_container,
+        )
 
         # invert dependency dictionary to access dependencies via hash
         dependency_dict = {file_hash: dependency for dependency, file_hash in dependency_dict.items()}

diff --git a/homcc/client/main.py b/homcc/client/main.py
@@ -113,11 +113,26 @@ def main():
         if not has_local:
             hosts.append(localhost)
 
-    # PROFILE; if --no-profile is specified do not use any specified profiles from cli or config file
-    if homcc_args_dict["no_profile"]:
-        homcc_config.profile = None
-    elif (profile := homcc_args_dict["profile"]) is not None:
-        homcc_config.profile = profile
+    # SCHROOT_PROFILE; if --no-schroot-profile is specified do not use
+    # any specified schroot profiles from cli or config file
+    if homcc_args_dict["no_schroot_profile"]:
+        homcc_config.schroot_profile = None
+    elif (profile := homcc_args_dict["schroot_profile"]) is not None:
+        homcc_config.schroot_profile = profile
+
+    # DOCKER_CONTAINER; if --no-docker-container is specified do not use
+    # any specified docker containers from cli or config file
+    if homcc_args_dict["no_docker_container"]:
+        homcc_config.docker_container = None
+    elif (docker_container := homcc_args_dict["docker_container"]) is not None:
+        homcc_config.docker_container = docker_container
+
+    if homcc_config.schroot_profile is not None and homcc_config.docker_container is not None:
+        logger.error(
+            "Can not specify a schroot profile and a docker container to use simultaneously."
+            "Please remove one of the config options."
+        )
+        sys.exit(os.EX_USAGE)
 
     # TIMEOUT
     if (timeout := homcc_args_dict["timeout"]) is not None:

diff --git a/homcc/client/parsing.py b/homcc/client/parsing.py
@@ -160,7 +160,8 @@ class ClientConfig:
 
     compiler: str
     compression: Compression
-    profile: Optional[str]
+    schroot_profile: Optional[str]
+    docker_container: Optional[str]
     timeout: Optional[float]
     log_level: Optional[LogLevel]
     verbose: bool
@@ -170,14 +171,16 @@ def __init__(
         *,
         compiler: Optional[str] = None,
         compression: Optional[str] = None,
-        profile: Optional[str] = None,
+        schroot_profile: Optional[str] = None,
+        docker_container: Optional[str] = None,
         timeout: Optional[float] = None,
         log_level: Optional[str] = None,
         verbose: Optional[bool] = None,
     ):
         self.compiler = compiler or Arguments.DEFAULT_COMPILER
         self.compression = Compression.from_name(compression)
-        self.profile = profile
+        self.schroot_profile = schroot_profile
+        self.docker_container = docker_container
         self.timeout = timeout
         self.log_level = LogLevel[log_level] if log_level else None
         self.verbose = verbose is not None and verbose
@@ -186,15 +189,17 @@ def __init__(
     def from_config_section(cls, homcc_config: SectionProxy) -> ClientConfig:
         compiler: Optional[str] = homcc_config.get("compiler")
         compression: Optional[str] = homcc_config.get("compression")
-        profile: Optional[str] = homcc_config.get("profile")
+        schroot_profile: Optional[str] = homcc_config.get("schroot_profile")
+        docker_container: Optional[str] = homcc_config.get("docker_container")
         timeout: Optional[float] = homcc_config.getfloat("timeout")
         log_level: Optional[str] = homcc_config.get("log_level")
         verbose: Optional[bool] = homcc_config.getboolean("verbose")
 
         return ClientConfig(
             compiler=compiler,
             compression=compression,
-            profile=profile,
+            schroot_profile=schroot_profile,
+            docker_container=docker_container,
             timeout=timeout,
             log_level=log_level,
             verbose=verbose,
@@ -252,17 +257,30 @@ def parse_cli_args(args: List[str]) -> Tuple[Dict[str, Any], Arguments]:
         f"{indented_newline.join(Compression.descriptions())}",
     )
 
-    profile = parser.add_mutually_exclusive_group()
-    profile.add_argument(
-        "--profile",
+    schroot_profile = parser.add_mutually_exclusive_group()
+    schroot_profile.add_argument(
+        "--schroot-profile",
         type=str,
-        help="PROFILE which will be mapped to predefined chroot environments on the selected remote compilation server,"
-        " no profile is being used on default",
+        help="SCHROOT_PROFILE which will be mapped to predefined chroot environments on "
+        "the selected remote compilation server, no schroot profile is being used on default",
     )
-    profile.add_argument(
-        "--no-profile",
+    schroot_profile.add_argument(
+        "--no-schroot-profile",
         action="store_true",
-        help="enforce that no PROFILE is used even if one is specified in the configuration file",
+        help="enforce that no SCHROOT_PROFILE is used even if one is specified in the configuration file",
+    )
+
+    docker_container = parser.add_mutually_exclusive_group()
+    docker_container.add_argument(
+        "--docker-container",
+        type=str,
+        help="DOCKER_CONTAINER name which will be used to compile in on the selected remote compilation server,"
+        " no docker container is being used on default",
+    )
+    docker_container.add_argument(
+        "--no-docker-container",
+        action="store_true",
+        help="enforce that no DOCKER_CONTAINER is used even if one is specified in the configuration file",
     )
 
     parser.add_argument(

diff --git a/homcc/common/arguments.py b/homcc/common/arguments.py
@@ -576,3 +576,11 @@ def schroot_execute(self, profile: str, **kwargs) -> ArgumentsExecutionResult:
         """
         schroot_args: List[str] = ["schroot", "-c", profile, "--"]
         return self._execute_args(schroot_args + list(self), **kwargs)
+
+    def docker_execute(self, container: str, cwd: str, **kwargs) -> ArgumentsExecutionResult:
+        """
+        Execute arguments in a docker container.
+        If possible, all parameters to this method will also be forwarded directly to the subprocess function call.
+        """
+        docker_args: List[str] = ["docker", "exec", "--workdir", cwd, container]
+        return self._execute_args(docker_args + list(self), **kwargs)
diff --git a/homcc/common/logging.py b/homcc/common/logging.py
@@ -124,19 +124,20 @@ class _ServerFormatter(_Formatter):
     def _level_format(self, level_format: str) -> str:
         # colored logging messages with debug information
         if self._config.is_colored() and self._config.is_detailed():
-            return f"[{level_format}%(levelname)s{self.RESET}] %(asctime)s - %(pathname)s:%(lineno)d:\n%(message)s"
+            return f"""[{level_format}%(levelname)s{self.RESET}] %(asctime)s - %(threadName)s
+             - %(pathname)s:%(lineno)d:\n%(message)s"""
 
         # uncolored logging messages with debug information
         if self._config.is_detailed():
-            return "[%(levelname)s] %(asctime)s - %(pathname)s:%(lineno)d:\n%(message)s"
+            return "[%(levelname)s] %(asctime)s - %(threadName)s - %(pathname)s:%(lineno)d:\n%(message)s"
 
         # user-friendly, colored logging messages
         if self._config.is_colored():
-            return f"[{level_format}%(levelname)s{self.RESET}] %(asctime)s - %(message)s"
+            return f"[{level_format}%(levelname)s{self.RESET}] %(asctime)s - %(threadName)s - %(message)s"
 
         # user-friendly, uncolored logging messages
         if self._config.is_none():
-            return "[%(levelname)s] %(asctime)s - %(message)s"
+            return "[%(levelname)s] %(asctime)s - %(threadName)s - %(message)s"
 
         raise ValueError(f"Unrecognized formatter configuration {self._config}")