celonis · spirsch · Jul 12, 2022 · May 24, 2022 · May 24, 2022 · May 25, 2022
diff --git a/homcc/client/client.py b/homcc/client/client.py
@@ -114,9 +114,12 @@ def _handle_termination(self, _, frame):
     def __enter__(self):
         pass
 
-    def __exit__(self, *_):
-        self._semaphore.release()
-        self._semaphore.close()
+    def __exit__(self, *exc):
+        logger.debug("Exiting semaphore '%s' with value '%i'", self._semaphore.name, self._semaphore.value)
+
+        if self._semaphore is not None:
+            self._semaphore.__exit__(*exc)  # releases the semaphore
+            self._semaphore = self._semaphore.close()  # closes and sets the semaphore to None
 
 
 class RemoteHostSemaphore(HostSemaphore):
@@ -140,6 +143,8 @@ def __init__(self, host: Host):
         super().__init__(host)
 
     def __enter__(self) -> RemoteHostSemaphore:
+        logger.debug("Entering semaphore '%s' with value '%i'", self._semaphore.name, self._semaphore.value)
+
         try:
             self._semaphore.acquire(0)  # non-blocking acquisition
         except posix_ipc.BusyError as error:
@@ -182,6 +187,8 @@ def __init__(self, host: Host, compilation_time: float = DEFAULT_COMPILATION_TIM
         super().__init__(host)
 
     def __enter__(self) -> LocalHostSemaphore:
+        logger.debug("Entering semaphore '%s' with value '%i'", self._semaphore.name, self._semaphore.value)
+
         while True:
             try:
                 self._semaphore.acquire(self._compilation_time - self._timeout)  # blocking acquisition

diff --git a/homcc/client/compilation.py b/homcc/client/compilation.py
@@ -21,6 +21,7 @@
     FailedHostNameResolutionError,
     HostsExhaustedError,
     RemoteCompilationError,
+    PreprocessorError,
     UnexpectedMessageTypeError,
     SlotsExhaustedError,
 )
@@ -41,7 +42,7 @@
 DEFAULT_LOCALHOST_LIMIT: int = (
     len(os.sched_getaffinity(0))  # number of available CPUs for this process
     or os.cpu_count()  # total number of physical CPUs on the machine
-    or 2  # fallback value to enable minor level of concurrency
+    or 4  # fallback value to enable minor level of concurrency
 )
 DEFAULT_LOCALHOST: Host = Host.localhost_with_limit(DEFAULT_LOCALHOST_LIMIT)
 EXCLUDED_DEPENDENCY_PREFIXES: Tuple = ("/usr/include", "/usr/lib")
@@ -162,37 +163,45 @@ def compile_locally(arguments: Arguments, localhost: Host) -> int:
 def scan_includes(arguments: Arguments) -> List[str]:
     """find all included dependencies"""
     dependencies: Set[str] = find_dependencies(arguments)
-    return [dependency for dependency in dependencies if dependency not in arguments.source_files]
-
-
-def is_sendable_dependency(dependency: str) -> bool:
-    # filter preprocessor output target and line breaks
-    if dependency in [f"{Arguments.PREPROCESSOR_TARGET}:", "\\"]:
-        return False
-
-    # normalize paths, e.g. convert /usr/bin/../lib/ to /usr/lib/
-    dependency_path: Path = Path(dependency).resolve()
-
-    if str(dependency_path).startswith(EXCLUDED_DEPENDENCY_PREFIXES):
-        return False
-
-    return True
+    return [dependency for dependency in dependencies if not Arguments.is_source_file_arg(dependency)]
 
 
 def find_dependencies(arguments: Arguments) -> Set[str]:
     """get unique set of dependencies by calling the preprocessor and filtering the result"""
+
+    arguments, filename = arguments.dependency_finding()
     try:
         # execute preprocessor command, e.g.: "g++ foo.cpp -M -MT $(homcc)"
-        result: ArgumentsExecutionResult = arguments.dependency_finding().execute(check=True)
+        result: ArgumentsExecutionResult = arguments.execute(check=True)
     except subprocess.CalledProcessError as error:
         logger.error("Preprocessor error:\n%s", error.stderr)
         sys.exit(error.returncode)
 
-    if result.stdout:
-        logger.debug("Preprocessor result:\n%s", result.stdout)
-
-    # create unique set of dependencies by filtering the preprocessor result for meaningfully sendable dependencies
-    return set(filter(is_sendable_dependency, result.stdout.split()))
+    # read from the dependency file if it was created as a side effect
+    dependency_result: str = (
+        Path(filename).read_text(encoding="utf-8") if filename is not None and filename != "-" else result.stdout
+    )
+
+    if not dependency_result:
+        raise PreprocessorError("Empty preprocessor result.")
+
+    logger.debug("Preprocessor result:\n%s", dependency_result)
+
+    def extract_dependencies(line: str) -> List[str]:
+        split: List[str] = line.split(":")  # remove preprocessor output targets specified via -MT
+        dependency_line: str = split[1] if len(split) == 2 else split[0]  # e.g. ignore "foo.o bar.o:"
+        return [
+            str(Path(dependency).resolve())  # normalize paths, e.g. convert /usr/bin/../lib/ to /usr/lib/
+            for dependency in dependency_line.rstrip("\\").split()  # remove line break char "\"
+        ]
+
+    # extract dependencies from the preprocessor result and filter for sendability
+    return {
+        dependency
+        for line in dependency_result.splitlines()
+        for dependency in extract_dependencies(line)
+        if not dependency.startswith(EXCLUDED_DEPENDENCY_PREFIXES)  # check sendability
+    }
 
 
 def calculate_dependency_dict(dependencies: Set[str]) -> Dict[str, str]:

diff --git a/homcc/client/errors.py b/homcc/client/errors.py
@@ -7,6 +7,10 @@ class RecoverableClientError(Exception):
     """Base class for TCPClient exceptions to indicate recoverability for the client main function"""
 
 
+class PreprocessorError(RecoverableClientError):
+    """Exception for errors during the preprocessor stage"""
+
+
 class CompilationTimeoutError(RecoverableClientError):
     """Exception for a timed out compilation request"""
 

diff --git a/homcc/client/parsing.py b/homcc/client/parsing.py
@@ -15,7 +15,7 @@
 
 from homcc.common.arguments import Arguments
 from homcc.common.compression import Compression
-from configparser import ConfigParser, SectionProxy
+from configparser import ConfigParser, Error, SectionProxy
 from homcc.common.logging import LogLevel
 from homcc.common.parsing import HOMCC_CONFIG_FILENAME, default_locations, parse_configs
 from homcc.client.errors import HostParsingError, NoHostsFoundError
@@ -399,7 +399,11 @@ def filtered_lines(text: str) -> List[str]:
 
 
 def parse_config(filenames: List[Path] = None) -> ClientConfig:
-    cfg: ConfigParser = parse_configs(filenames or default_locations(HOMCC_CONFIG_FILENAME))
+    try:
+        cfg: ConfigParser = parse_configs(filenames or default_locations(HOMCC_CONFIG_FILENAME))
+    except Error as err:
+        print(f"{err}; using default configuration instead")
+        return ClientConfig()
 
     if HOMCC_CLIENT_CONFIG_SECTION not in cfg.sections():
         return ClientConfig()

diff --git a/homcc/common/arguments.py b/homcc/common/arguments.py
@@ -9,7 +9,8 @@
 
 from dataclasses import dataclass
 from functools import cached_property
-from typing import Any, Iterator, List, Optional
+from pathlib import Path
+from typing import Any, Iterator, List, Optional, Tuple
 
 logger = logging.getLogger(__name__)
 
@@ -46,6 +47,8 @@ class Arguments:
     OUTPUT_ARG: str = "-o"
     SPECIFY_LANGUAGE_ARG: str = "-x"
 
+    DEPENDENCY_SIDE_EFFECT_ARG: str = "-MD"
+
     INCLUDE_ARGS: List[str] = ["-I", "-isysroot", "-isystem"]
 
     # languages
@@ -58,7 +61,7 @@ class Local:
         """
 
         # preprocessor args
-        PREPROCESSOR_ARGS: List[str] = ["-MD", "-MMD", "-MG", "-MP"]
+        PREPROCESSOR_ARGS: List[str] = ["-MG", "-MP"]
         PREPROCESSOR_OPTION_PREFIX_ARGS: List[str] = ["-MF", "-MT", "-MQ"]
 
         # linking args
@@ -74,6 +77,7 @@ class Unsendable:
         # preprocessing args
         PREPROCESSOR_ONLY_ARG: str = "-E"
         PREPROCESSOR_DEPENDENCY_ARG: str = "-M"
+        PREPROCESSOR_USER_HEADER_ONLY_DEPENDENCY_ARG: str = "-MM"
 
         # args that rely on native machine
         NATIVE_ARGS: List[str] = ["-march=native", "-mtune=native"]
@@ -147,6 +151,11 @@ def from_args(cls, args: List[str]) -> Arguments:
         # compiler with args, e.g. ["g++", "foo.cpp", "-c"]
         return cls(args[0], args[1:])
 
+    @classmethod
+    def from_str(cls, args_str: str) -> Arguments:
+        """construct arguments from an args string"""
+        return Arguments.from_args(args_str.split())
+
     @classmethod
     def from_cli(cls, compiler_or_argument: str, args: List[str]) -> Arguments:
         """construct Arguments from args given via the CLI"""
@@ -197,12 +206,16 @@ def is_sendable_arg(arg: str) -> bool:
             logger.debug("[%s] implies a preprocessor only call", arg)
             return False
 
-        if arg in Arguments.Local.PREPROCESSOR_ARGS:
+        if arg in Arguments.Local.PREPROCESSOR_ARGS + [Arguments.DEPENDENCY_SIDE_EFFECT_ARG]:
             return True
 
         if arg.startswith(tuple(Arguments.Local.PREPROCESSOR_OPTION_PREFIX_ARGS)):
             return True
 
+        if arg.startswith(Arguments.Unsendable.PREPROCESSOR_USER_HEADER_ONLY_DEPENDENCY_ARG):  # -MM prefix
+            logger.debug("[%s] implies two different preprocessor calls", arg)
+            return False
+
         # all remaining preprocessing arg types with prefix "-M" imply Unsendability
         if arg.startswith(Arguments.Unsendable.PREPROCESSOR_DEPENDENCY_ARG):
             logger.debug(
@@ -273,14 +286,6 @@ def add_arg(self, arg: str) -> Arguments:
         self._args.append(arg)
         return self
 
-    def remove_arg(self, arg: str) -> Arguments:
-        """
-        if present, remove the specified arg, this may remove multiple occurrences of this arg and break cached
-        properties when used inconsiderately
-        """
-        self._args = list(filter(lambda _arg: _arg != arg, self.args))
-        return self
-
     @property
     def compiler(self) -> Optional[str]:
         """if present, return the specified compiler"""
@@ -304,6 +309,20 @@ def output(self) -> Optional[str]:
                     output = arg[2:]
         return output
 
+    @cached_property
+    def dependency_output(self) -> Optional[str]:
+        """if present, return the last explicitly specified dependency output target"""
+        dependency_output: Optional[str] = None
+
+        it: Iterator[str] = iter(self.args)
+        for arg in it:
+            if arg.startswith("-MF"):
+                if arg == "-MF":  # dependency output argument with output target following: e.g.: -MF out
+                    dependency_output = next(it)  # skip dependency output file target
+                else:  # compact dependency output argument: e.g.: -MFout
+                    dependency_output = arg[3:]  # skip "-MF" prefix
+        return dependency_output
+
     @cached_property
     def source_files(self) -> List[str]:
         """extract and return all source files that will be compiled"""
@@ -347,7 +366,7 @@ def specified_language(self) -> Optional[str]:
 
     def is_sendable(self) -> bool:
         """check whether the remote execution of arguments would be successful"""
-        # "-o -" might be treated as "write result to stdout" by some compilers
+        # "-o -" might either be treated as "write result to stdout" or "write result to file named '-'"
         if self.output == "-":
             logger.info('Cannot compile %s remotely because output "%s" is ambiguous', self, self.output)
             return False
@@ -384,16 +403,30 @@ def is_linking_only(self) -> bool:
         """check whether the execution of arguments leads to calling only the linker"""
         return not self.source_files and self.is_linking()
 
-    def dependency_finding(self) -> Arguments:
-        """return a copy of arguments with which to find dependencies via the preprocessor"""
-        return (
-            self.copy()
-            .remove_arg(self.NO_LINKING_ARG)
-            .remove_output_args()
-            .add_arg("-M")  # output dependencies
-            .add_arg("-MT")  # change target of the dependency generation
-            .add_arg(self.PREPROCESSOR_TARGET)
-        )
+    def dependency_finding(self) -> Tuple[Arguments, Optional[str]]:
+        """return a dependency finding arguments with which to find dependencies via the preprocessor"""
+
+        # gcc and clang handle the combination of -MD -M differently, this function provides a uniform approach for
+        # both compilers that also preserves side effects like the creation of dependency files
+
+        if self.DEPENDENCY_SIDE_EFFECT_ARG not in self.args:
+            # TODO(s.pirsch): benchmark -M -MF- and writing stdout to specified file afterwards
+            return self.copy().remove_output_args().add_arg(self.Unsendable.PREPROCESSOR_DEPENDENCY_ARG), None
+
+        dependency_output_file: str
+
+        if self.dependency_output is not None:  # e.g. "-MF foo.d"
+            dependency_output_file = self.dependency_output
+        elif self.output is not None:  # e.g. "-o foo.o" -> "foo.d"
+            dependency_output_file = f"{Path(self.output).stem}.d"
+        else:  # e.g. "foo.cpp" -> "foo.d"
+            dependency_output_file = f"{Path(self.source_files[0]).stem}.d"
+
+        # TODO(s.pirsch): disallow multiple source files in the future when linker issue was investigated
+        if len(self.source_files) > 1:
+            logger.warning("Executing [%s] might not create the intended dependency files.", self)
+
+        return self.copy(), dependency_output_file
 
     def no_linking(self) -> Arguments:
         """return a copy of arguments where all output args are removed and the no linking arg is added"""