From d1234840973764fa337fdade76d5642102cbc9a8 Mon Sep 17 00:00:00 2001 From: Victor Zhestkov Date: Mon, 10 Mar 2025 10:13:39 +0100 Subject: [PATCH 1/3] Add DEB822 apt source format support Co-authored-by: Marek Czernek --- salt/modules/aptpkg.py | 135 +++++- salt/utils/pkg/deb.py | 979 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 1018 insertions(+), 96 deletions(-) diff --git a/salt/modules/aptpkg.py b/salt/modules/aptpkg.py index ea7fc9b99f76..b406dd8b49ec 100644 --- a/salt/modules/aptpkg.py +++ b/salt/modules/aptpkg.py @@ -44,7 +44,13 @@ SaltInvocationError, ) from salt.modules.cmdmod import _parse_env -from salt.utils.pkg.deb import SourceEntry, SourcesList +from salt.utils.pkg.deb import ( + Deb822SourceEntry, + Section, + SourceEntry, + SourcesList, + _invalid, +) log = logging.getLogger(__name__) @@ -1639,21 +1645,41 @@ def list_repos(**kwargs): """ repos = {} sources = SourcesList() - for source in sources.list: + for source in sources: if _skip_source(source): continue signedby = source.signedby repo = {} repo["file"] = source.file - repo["comps"] = getattr(source, "comps", []) + repo_comps = getattr(source, "comps", []) + repo_dists = source.dist.split(" ") + repo["comps"] = repo_comps repo["disabled"] = source.disabled - repo["dist"] = source.dist + repo["enabled"] = not repo[ + "disabled" + ] # This is for compatibility with the other modules + repo["dist"] = repo_dists.pop(0) + repo["suites"] = list(source.suites) repo["type"] = source.type repo["uri"] = source.uri - repo["line"] = source.line.strip() + if "Types: " in source.line and "\n" in source.line: + repo["line"] = ( + f"{source.type} {source.uri} {repo['dist']} {' '.join(repo_comps)}" + ) + else: + repo["line"] = source.line.strip() repo["architectures"] = getattr(source, "architectures", []) repo["signedby"] = signedby repos.setdefault(source.uri, []).append(repo) + if len(repo_dists): + for dist in repo_dists: + repo_copy = repo.copy() + repo_copy["dist"] = dist + if "Types: " in source.line and "\n" in source.line: + repo_copy["line"] = ( + f"{source.type} {source.uri} {repo_copy['dist']} {' '.join(repo_comps)}" + ) + repos[source.uri].append(repo_copy) return repos @@ -1662,12 +1688,17 @@ def get_repo(repo, **kwargs): Display a repo from the sources.list / sources.list.d The repo passed in needs to be a complete repo entry. + When system uses repository in the deb822 format, + get_repo uses a partial match of distributions. + + In that case, include any distribution of the deb822 + repository in the repo name to match that repo. CLI Examples: .. code-block:: bash - salt '*' pkg.get_repo "myrepo definition" + salt '*' pkg.get_repo "deb URL noble main" """ ppa_auth = kwargs.get("ppa_auth", None) # we have to be clever about this since the repo definition formats @@ -1726,11 +1757,17 @@ def del_repo(repo, **kwargs): The repo passed in must be a fully formed repository definition string. + When system uses repository in the deb822 format, + del_repo uses a partial match of distributions. + + In that case, include any distribution of the deb822 + repository in the repo name to match that repo. + CLI Examples: .. code-block:: bash - salt '*' pkg.del_repo "myrepo definition" + salt '*' pkg.del_repo "deb URL noble main" """ is_ppa = False if repo.startswith("ppa:") and __grains__["os"] in ("Ubuntu", "Mint", "neon"): @@ -1761,11 +1798,22 @@ def del_repo(repo, **kwargs): source.type == repo_entry["type"] and source.architectures == repo_entry["architectures"] and source.uri.rstrip("/") == repo_entry["uri"].rstrip("/") - and source.dist == repo_entry["dist"] + and repo_entry["dist"] in source.suites ): - s_comps = set(source.comps) r_comps = set(repo_entry["comps"]) + if s_comps == r_comps: + r_suites = list(source.suites) + r_suites.remove(repo_entry["dist"]) + source.suites = r_suites + deleted_from[source.file] = 0 + if not source.suites: + try: + sources.remove(source) + except ValueError: + pass + sources.save() + continue if s_comps.intersection(r_comps) or (not s_comps and not r_comps): deleted_from[source.file] = 0 source.comps = list(s_comps.difference(r_comps)) @@ -1782,11 +1830,23 @@ def del_repo(repo, **kwargs): and repo_entry["type"] == "deb" and source.type == "deb-src" and source.uri == repo_entry["uri"] - and source.dist == repo_entry["dist"] + and repo_entry["dist"] in source.suites ): s_comps = set(source.comps) r_comps = set(repo_entry["comps"]) + if s_comps == r_comps: + r_suites = list(source.suites) + r_suites.remove(repo_entry["dist"]) + source.suites = r_suites + deleted_from[source.file] = 0 + if not source.suites: + try: + sources.remove(source) + except ValueError: + pass + sources.save() + continue if s_comps.intersection(r_comps) or (not s_comps and not r_comps): deleted_from[source.file] = 0 source.comps = list(s_comps.difference(r_comps)) @@ -1799,6 +1859,8 @@ def del_repo(repo, **kwargs): if deleted_from: ret = "" for source in sources: + if source.invalid: + continue if source.file in deleted_from: deleted_from[source.file] += 1 for repo_file, count in deleted_from.items(): @@ -2238,6 +2300,12 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): ``ppa:/repo`` format is acceptable. ``ppa:`` format can only be used to create a new repository. + When system uses repository in the deb822 format, mod_repo uses a partial + match of distributions. + + In that case, include any distribution of the deb822 repository in the + repo definition to match that repo. + The following options are available to modify a repo definition: architectures @@ -2292,8 +2360,8 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): .. code-block:: bash - salt '*' pkg.mod_repo 'myrepo definition' uri=http://new/uri - salt '*' pkg.mod_repo 'myrepo definition' comps=main,universe + salt '*' pkg.mod_repo 'deb URL noble main' uri=http://new/uri + salt '*' pkg.mod_repo 'deb URL noble main' comps=main,universe """ if "refresh_db" in kwargs: refresh = kwargs["refresh_db"] @@ -2413,6 +2481,13 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): repos = [] for source in sources: + if isinstance(source, Deb822SourceEntry): + if source.types == [""] or not bool(source.types) or not source.type: + continue + else: + _, invalid, _, _ = _invalid(source.line) + if invalid: + continue repos.append(source) mod_source = None @@ -2569,9 +2644,9 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): repo_matches = ( apt_source.type == repo_entry["type"] and apt_source.uri.rstrip("/") == repo_entry["uri"].rstrip("/") - and apt_source.dist == repo_entry["dist"] + and repo_entry["dist"] in apt_source.suites ) - kw_matches = apt_source.dist == kw_dist and apt_source.type == kw_type + kw_matches = kw_dist in apt_source.suites and apt_source.type == kw_type if repo_matches or kw_matches: for comp in full_comp_list: @@ -2589,17 +2664,35 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): repo_source_entry = SourceEntry(repo) if not mod_source: - mod_source = SourceEntry(repo) + apt_source_file = kwargs.get("file") + if not apt_source_file: + raise SaltInvocationError( + "missing 'file' argument when defining a new repository" + ) + + if not apt_source_file.endswith(".list"): + section = Section("") + section["Types"] = repo_entry["type"] + section["URIs"] = repo_entry["uri"] + section["Suites"] = repo_entry["dist"] + section["Components"] = " ".join(repo_entry["comps"]) + if kwargs.get("trusted") is True or kwargs.get("Trusted") is True: + section["Trusted"] = "yes" + mod_source = Deb822SourceEntry(section, apt_source_file) + else: + mod_source = SourceEntry(repo) if "comments" in kwargs: mod_source.comment = kwargs["comments"] sources.list.append(mod_source) elif "comments" in kwargs: mod_source.comment = kwargs["comments"] - mod_source.line = repo_source_entry.line if not mod_source.line.endswith("\n"): mod_source.line = mod_source.line + "\n" + if not kwargs["architectures"] and not mod_source.architectures: + kwargs.pop("architectures") + for key in kwargs: if key in _MODIFY_OK and hasattr(mod_source, key): setattr(mod_source, key, kwargs[key]) @@ -2615,15 +2708,21 @@ def mod_repo(repo, saltenv="base", aptkey=True, **kwargs): signedby = mod_source.signedby + repo_source_line = mod_source.line + if "Types: " in repo_source_line and "\n" in repo_source_line: + repo_source_line = f"{mod_source.type} {mod_source.uri} {repo_entry['dist']} {' '.join(mod_source.comps)}" + return { repo: { "architectures": getattr(mod_source, "architectures", []), + "dist": mod_source.dist, + "suites": mod_source.suites, "comps": mod_source.comps, "disabled": mod_source.disabled, "file": mod_source.file, "type": mod_source.type, "uri": mod_source.uri, - "line": mod_source.line, + "line": repo_source_line, "signedby": signedby, } } @@ -2726,7 +2825,7 @@ def _expand_repo_def(os_name, os_codename=None, **kwargs): sanitized["dist"] = _source_entry.dist sanitized["type"] = _source_entry.type sanitized["uri"] = _source_entry.uri - sanitized["line"] = _source_entry.line.strip() + sanitized["line"] = getattr(_source_entry, "line", "").strip() sanitized["architectures"] = getattr(_source_entry, "architectures", []) sanitized["signedby"] = signedby diff --git a/salt/utils/pkg/deb.py b/salt/utils/pkg/deb.py index 6a3b3a833b21..2ef1c20668f2 100644 --- a/salt/utils/pkg/deb.py +++ b/salt/utils/pkg/deb.py @@ -4,33 +4,643 @@ import logging import os -import pathlib import re -import shutil -import tempfile +import weakref from collections import OrderedDict +from typing import Generic, TypeVar, Union import salt.utils.files log = logging.getLogger(__name__) -class SourceEntry: # pylint: disable=function-redefined - def __init__(self, line, file=None): - self.invalid = False - self.comps = [] - self.disabled = False - self.comment = "" - self.dist = "" - self.type = "" - self.uri = "" - self.line = line - self.architectures = [] - self.signedby = "" +_APT_SOURCES_LIST = "/etc/apt/sources.list" +_APT_SOURCES_PARTSDIR = "/etc/apt/sources.list.d/" + + +def string_to_bool(s): + """ + Convert string representation of bool values to integer + """ + s = s.lower() + if s in ("no", "false", "without", "off", "disable"): + return 0 + elif s in ("yes", "true", "with", "on", "enable"): + return 1 + return -1 + + +class TagSection: + + def __init__(self, section): + self._data = section + self._re = re.compile(r"\A(\S+): (.*)") + + def __iter__(self): + lines = self._data.split("\n") + tag = None + value = None + while lines: + line = lines.pop(0) + match = self._re.match(line) + if match: + if tag is not None: + yield tag, value.strip() + tag = match.group(1) + value = match.group(2) + elif line == "" and tag is not None: + yield tag, value.strip() + else: + value = f"{value}\n{line}" + if tag is not None: + yield tag, value.strip() + + +class Section: + """A single deb822 section, possibly with comments. + + This represents a single deb822 section. + """ + + tags: OrderedDict + _case_mapping: dict + header: str + footer: str + + def __init__(self, section): + if isinstance(section, Section): + self.tags = OrderedDict(section.tags) + self._case_mapping = {k.casefold(): k for k in self.tags} + self.header = section.header + self.footer = section.footer + return + + comments = ["", ""] + in_section = False + trimmed_section = "" + + for line in section.split("\n"): + if line.startswith("#"): + # remove the leading # + line = line[1:] + comments[in_section] += line + "\n" + continue + + in_section = True + trimmed_section += line + "\n" + + self.tags = OrderedDict(TagSection(trimmed_section)) + self._case_mapping = {k.casefold(): k for k in self.tags} + self.header, self.footer = comments + + def __getitem__(self, key): + """Get the value of a field.""" + return self.tags[self._case_mapping.get(key.casefold(), key)] + + def __delitem__(self, key): + """Delete a field""" + del self.tags[self._case_mapping.get(key.casefold(), key)] + + def __setitem__(self, key, val): + """Set the value of a field.""" + if key.casefold() not in self._case_mapping: + self._case_mapping[key.casefold()] = key + self.tags[self._case_mapping[key.casefold()]] = val + + def __bool__(self): + return bool(self.tags) + + def get(self, key, default=None): + try: + return self[key] + except KeyError: + return default + + @staticmethod + def __comment_lines(content): + return ( + "\n".join("#" + line for line in content.splitlines()) + "\n" + if content + else "" + ) + + def __str__(self): + """Canonical string rendering of this section.""" + return ( + self.__comment_lines(self.header) + + "".join(f"{k}: {v}\n" for k, v in self.tags.items()) + + self.__comment_lines(self.footer) + ) + + +class File: + """ + Parse a given file object into a list of Section objects. + """ + + def __init__(self, fobj): + self.sections = [] + section = "" + for line in fobj: + if not line.isspace(): + # A line is part of the section if it has non-whitespace characters + section += line + elif section: + # Our line is just whitespace and we have gathered section content, so let's write out the section + self.sections.append(Section(section)) + section = "" + + # The final section may not be terminated by an empty line + if section: + self.sections.append(Section(section)) + + def __iter__(self): + return iter(self.sections) + + def __str__(self): + return "\n".join(str(s) for s in self.sections) + + +class SingleValueProperty(property): + def __init__(self, key, doc): + self.key = key + self.__doc__ = doc + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return obj.section.get(self.key, None) + + def __set__(self, obj, value): + if value is None: + del obj.section[self.key] + else: + obj.section[self.key] = value + + +class MultiValueProperty(property): + def __init__(self, key, doc): + self.key = key + self.__doc__ = doc + + def __get__(self, obj, objtype=None): + if obj is None: + return self + return SourceEntry.mysplit(obj.section.get(self.key, "")) + + def __set__(self, obj, values): + obj.section[self.key] = " ".join(values) + + +class ExplodedEntryProperty(property, Generic[TypeVar("T")]): + def __init__(self, parent): + self.parent = parent + + def __get__( + self, + obj, + objtype=None, + ): + if obj is None: + return self + return self.parent.__get__(obj.parent) + + def __set__(self, obj, value): + obj.split_out() + self.parent.__set__(obj.parent, value) + + +def DeprecatedProperty(prop): + """Wrapper to mark deprecated properties""" + return prop + + +def _null_weakref(): + """Behaves like an expired weakref.ref, returning None""" + return None + + +class Deb822SourceEntry: + def __init__( + self, + section, + file, + list=None, + ): + if section is None: + self.section = Section("") + elif isinstance(section, str): + self.section = Section(section) + else: + self.section = section + + self._line = str(self.section) self.file = file - if not self.file: - self.file = str(pathlib.Path(os.sep, "etc", "apt", "sources.list")) + self.template = None # type DistInfo.Suite + self.may_merge = False + self._children = weakref.WeakSet() + + if list: + self._list = weakref.ref(list) + else: + self._list = _null_weakref + + self.signedby = self.section.tags.get("Signed-By", "") + + def __eq__(self, other): + # FIXME: Implement plurals more correctly + """equal operator for two sources.list entries""" + return ( + self.disabled == other.disabled + and self.type == other.type + and self.uri + and self.uri.rstrip("/") == other.uri.rstrip("/") + and self.dist == other.dist + and self.comps == other.comps + ) + + architectures = MultiValueProperty("Architectures", "The list of architectures") + types = MultiValueProperty("Types", "The list of types") + type = DeprecatedProperty(SingleValueProperty("Types", "The list of types")) + uris = MultiValueProperty("URIs", "URIs in the source") + uri = DeprecatedProperty(SingleValueProperty("URIs", "URIs in the source")) + suites = MultiValueProperty("Suites", "Suites in the source") + dist = DeprecatedProperty(SingleValueProperty("Suites", "Suites in the source")) + comps = MultiValueProperty("Components", "Components in the source") + + @property + def comment(self): + """Legacy attribute describing the paragraph header.""" + return self.section.header + + @comment.setter + def comment(self, comment): + """Legacy attribute describing the paragraph header.""" + self.section.header = comment + + @property + def trusted(self): + """Return the value of the Trusted field""" + try: + return string_to_bool(self.section["Trusted"]) + except KeyError: + return None + + @trusted.setter + def trusted(self, value): + if value is None: + try: + del self.section["Trusted"] + except KeyError: + pass + else: + self.section["Trusted"] = "yes" if value else "no" + + @property + def disabled(self): + """Check if Enabled: no is set.""" + return not string_to_bool(self.section.get("Enabled", "yes")) + + @disabled.setter + def disabled(self, value): + if value: + self.section["Enabled"] = "no" + else: + try: + del self.section["Enabled"] + except KeyError: + pass + + @property + def invalid(self): + """A section is invalid if it doesn't have proper entries.""" + return not self.section + + @property + def line(self): + """The entire (original) paragraph.""" + return self._line + + def __str__(self): + return self.str().strip() + + def str(self): + """Section as a string, newline terminated.""" + return str(self.section) + + def set_enabled(self, enabled): + """Deprecated (for deb822) accessor for .disabled""" + self.disabled = not enabled + + def merge(self, other): + """Merge the two entries if they are compatible.""" + if ( + not self.may_merge + and self.template is None + and not all(child.template for child in self._children) + ): + return False + if self.file != other.file: + return False + if not isinstance(other, Deb822SourceEntry): + return False + if self.comment != other.comment and not any( + "Added by software-properties" in c for c in (self.comment, other.comment) + ): + return False + + for tag in set(list(self.section.tags) + list(other.section.tags)): + if tag.lower() in ( + "types", + "uris", + "suites", + "components", + "architectures", + "signed-by", + ): + continue + in_self = self.section.get(tag, None) + in_other = other.section.get(tag, None) + if in_self != in_other: + return False + + if ( + sum( + [ + set(self.types) != set(other.types), + set(self.uris) != set(other.uris), + set(self.suites) != set(other.suites), + set(self.comps) != set(other.comps), + set(self.architectures) != set(other.architectures), + ] + ) + > 1 + ): + return False + + for typ in other.types: + if typ not in self.types: + self.types += [typ] + + for uri in other.uris: + if uri not in self.uris: + self.uris += [uri] + + for suite in other.suites: + if suite not in self.suites: + self.suites += [suite] + + for component in other.comps: + if component not in self.comps: + self.comps += [component] + + for arch in other.architectures: + if arch not in self.architectures: + self.architectures += [arch] + + return True + + def _reparent_children(self, to): + """If we end up being split, check if any of our children need to be reparented to the new parent.""" + for child in self._children: + for typ in to.types: + for uri in to.uris: + for suite in to.suites: + if (child._type, child._uri, child._suite) == ( + typ, + uri, + suite, + ): + assert child.parent == self + child._parent = weakref.ref(to) + + +class ExplodedDeb822SourceEntry: + """This represents a bit of a deb822 paragraph corresponding to a legacy sources.list entry""" + + # Mostly we use slots to prevent accidentally assigning unproxied attributes + __slots__ = ["_parent", "_type", "_uri", "_suite", "template", "__weakref__"] + + def __init__(self, parent, typ, uri, suite): + self._parent = weakref.ref(parent) + self._type = typ + self._uri = uri + self._suite = suite + self.template = parent.template + parent._children.add(self) + + @property + def parent(self): + if self._parent is not None: + parent = self._parent() + if parent is not None: + return parent + raise ValueError("The parent entry is no longer valid") + + @property + def uri(self): + self.__check_valid() + return self._uri + + @uri.setter + def uri(self, uri): + self.split_out() + self.parent.uris = [u if u != self._uri else uri for u in self.parent.uris] + self._uri = uri + + @property + def types(self): + return [self.type] + + @property + def suites(self): + return [self.dist] + + @property + def uris(self): + return [self.uri] + + @property + def type(self): + self.__check_valid() + return self._type + + @type.setter + def type(self, typ): + self.split_out() + self.parent.types = [typ] + self._type = typ + self.__check_valid() + assert self._type == typ + assert self.parent.types == [self._type] + + @property + def dist(self): + self.__check_valid() + return self._suite + + @dist.setter + def dist(self, suite): + self.split_out() + self.parent.suites = [suite] + self._suite = suite + self.__check_valid() + assert self._suite == suite + assert self.parent.suites == [self._suite] + + def __check_valid(self): + if self.parent._list() is None: + raise ValueError("The parent entry is dead") + for type in self.parent.types: + for uri in self.parent.uris: + for suite in self.parent.suites: + if (type, uri, suite) == (self._type, self._uri, self._suite): + return + raise ValueError(f"Could not find parent of {self}") + + def split_out(self): + parent = self.parent + if (parent.types, parent.uris, parent.suites) == ( + [self._type], + [self._uri], + [self._suite], + ): + return + sources_list = parent._list() + if sources_list is None: + raise ValueError("The parent entry is dead") + + try: + index = sources_list.list.index(parent) + except ValueError as e: + raise ValueError( + f"Parent entry for partial deb822 {self} no longer valid" + ) from e + + sources_list.remove(parent) + + reparented = False + for type in reversed(parent.types): + for uri in reversed(parent.uris): + for suite in reversed(parent.suites): + new = Deb822SourceEntry( + section=Section(parent.section), + file=parent.file, + list=sources_list, + ) + new.types = [type] + new.uris = [uri] + new.suites = [suite] + new.may_merge = True + + parent._reparent_children(new) + sources_list.list.insert(index, new) + if (type, uri, suite) == (self._type, self._uri, self._suite): + self._parent = weakref.ref(new) + reparented = True + if not reparented: + raise ValueError(f"Could not find parent of {self}") + + def __repr__(self): + return f" 0: + pieces.append(tmp) + return pieces + + def parse(self, line): + """parse a given sources.list (textual) line and break it up + into the field we have""" + return self._parse_sources(line) + + def __str__(self): + """debug helper""" + return self.str().strip() def str(self): return self.repo_line() @@ -98,79 +708,292 @@ def _parse_sources(self, line): self.comps = repo_line[3:] return True + @property + def types(self): + """deb822 compatible accessor for the type""" + return [self.type] + + @property + def uris(self): + """deb822 compatible accessor for the uri""" + return [self.uri] + + @property + def suites(self): + """deb822 compatible accessor for the suite""" + if self.dist: + return [self.dist] + return [] + + @suites.setter + def suites(self, suites): + """deb822 compatible setter for the suite""" + if len(suites) > 1: + raise ValueError("Only one suite is possible for non deb822 source entry") + if suites: + self.dist = str(suites[0]) + assert self.dist == suites[0] + else: + self.dist = "" + assert self.dist == "" -class SourcesList: # pylint: disable=function-redefined - def __init__(self): + +AnySourceEntry = Union[SourceEntry, Deb822SourceEntry] +AnyExplodedSourceEntry = Union[ + SourceEntry, Deb822SourceEntry, ExplodedDeb822SourceEntry +] + + +class SourcesList: + """represents the full sources.list + sources.list.d file""" + + def __init__( + self, + deb822=True, + ): + self.list = [] # the actual SourceEntries Type + self.deb822 = deb822 + self.refresh() + + def refresh(self): + """update the list of known entries""" self.list = [] - self.files = [ - pathlib.Path(os.sep, "etc", "apt", "sources.list"), - pathlib.Path(os.sep, "etc", "apt", "sources.list.d"), - ] - for file in self.files: - if file.is_dir(): - for fp in file.glob("**/*.list"): - self.add_file(file=fp) - else: - self.add_file(file) + # read sources.list + file = _APT_SOURCES_LIST + if os.path.isfile(file): + self.load(file) + # read sources.list.d + partsdir = _APT_SOURCES_PARTSDIR + if os.path.isdir(partsdir): + for file in os.listdir(partsdir): + if (self.deb822 and file.endswith(".sources")) or file.endswith( + ".list" + ): + self.load(os.path.join(partsdir, file)) def __iter__(self): + """simple iterator to go over self.list, returns SourceEntry + types""" yield from self.list - def add_file(self, file): + def __find(self, *predicates, **attrs): + uri = attrs.pop("uri", None) + for source in self.exploded_list(): + if uri and source.uri and uri.rstrip("/") != source.uri.rstrip("/"): + continue + if all(getattr(source, key) == attrs[key] for key in attrs) and all( + predicate(source) for predicate in predicates + ): + yield source + + def add( + self, + type, + uri, + dist, + orig_comps, + comment="", + pos=-1, + file=None, + architectures=None, + signedby="", + parent=None, + ): """ - Add the lines of a file to self.list + Add a new source to the sources.list. + The method will search for existing matching repos and will try to + reuse them as far as possible """ - if file.is_file(): - with salt.utils.files.fopen(str(file)) as source: - for line in source: - self.list.append(SourceEntry(line, file=str(file))) + + type = type.strip() + disabled = type.startswith("#") + if disabled: + type = type[1:].lstrip() + if architectures is None: + architectures = [] + architectures = set(architectures) + # create a working copy of the component list so that + # we can modify it later + comps = orig_comps[:] + sources = self.__find( + lambda s: set(s.architectures) == architectures, + disabled=disabled, + invalid=False, + type=type, + uri=uri, + dist=dist, + ) + # check if we have this source already in the sources.list + for source in sources: + for new_comp in comps: + if new_comp in source.comps: + # we have this component already, delete it + # from the new_comps list + del comps[comps.index(new_comp)] + if len(comps) == 0: + return source + + sources = self.__find( + lambda s: set(s.architectures) == architectures, + invalid=False, + type=type, + uri=uri, + dist=dist, + ) + for source in sources: + if source.disabled == disabled: + # if there is a repo with the same (disabled, type, uri, dist) + # just add the components + if set(source.comps) != set(comps): + source.comps = list(set(source.comps + comps)) + return source + elif source.disabled and not disabled: + # enable any matching (type, uri, dist), but disabled repo + if set(source.comps) == set(comps): + source.disabled = False + return source + + new_entry: AnySourceEntry + if file is None: + file = _APT_SOURCES_LIST + if file.endswith(".sources"): + new_entry = Deb822SourceEntry(None, file=file, list=self) + if parent: + parent = getattr(parent, "parent", parent) + assert isinstance(parent, Deb822SourceEntry) + for k in parent.section.tags: + new_entry.section[k] = parent.section[k] + new_entry.types = [type] + new_entry.uris = [uri] + new_entry.suites = [dist] + new_entry.comps = comps + if architectures: + new_entry.architectures = list(architectures) + new_entry.section.header = comment + new_entry.disabled = disabled else: - log.debug("The apt sources file %s does not exist", file) - - def add(self, type, uri, dist, orig_comps, architectures, signedby): - opts_count = [] - opts_line = "" - if architectures: - architectures = f"arch={','.join(architectures)}" - opts_count.append(architectures) - if signedby: - signedby = f"signed-by={signedby}" - opts_count.append(signedby) - if len(opts_count) > 1: - opts_line = f"[{' '.join(opts_count)}]" - elif len(opts_count) == 1: - opts_line = f"[{''.join(opts_count)}]" - repo_line = [ - type, - opts_line, - uri, - dist, - " ".join(orig_comps), - ] - return SourceEntry(" ".join([line for line in repo_line if line.strip()])) - - def remove(self, source): - """ - remove a source from the list of sources - """ - self.list.remove(source) + # there isn't any matching source, so create a new line and parse it + parts = [ + "#" if disabled else "", + type, + ("[arch=%s]" % ",".join(architectures)) if architectures else "", + uri, + dist, + ] + parts.extend(comps) + if comment: + parts.append("#" + comment) + line = " ".join(part for part in parts if part) + "\n" + + new_entry = SourceEntry(line) + if file is not None: + new_entry.file = file + + if pos < 0: + self.list.append(new_entry) + else: + self.list.insert(pos, new_entry) + return new_entry + + def remove(self, source_entry): + """remove the specified entry from the sources.list""" + if isinstance(source_entry, ExplodedDeb822SourceEntry): + source_entry.split_out() + source_entry = source_entry.parent + self.list.remove(source_entry) + + def load(self, file): + """(re)load the current sources""" + try: + with salt.utils.files.fopen(file) as f: + if file.endswith(".sources"): + for section in File(f): + self.list.append(Deb822SourceEntry(section, file, list=self)) + else: + for line in f: + source = SourceEntry(line, file) + self.list.append(source) + except Exception as exc: # pylint: disable=broad-except + logging.warning(f"could not parse source file '{file}': {exc}\n") + + def index(self, entry): + if isinstance(entry, ExplodedDeb822SourceEntry): + return self.list.index(entry.parent) + return self.list.index(entry) + + def merge(self): + """Merge consecutive entries that have been split back together.""" + merged = True + while merged: + i = 0 + merged = False + while i + 1 < len(self.list): + entry = self.list[i] + if isinstance(entry, Deb822SourceEntry): + j = i + 1 + while j < len(self.list): + if entry.merge(self.list[j]): + del self.list[j] + merged = True + else: + j += 1 + i += 1 def save(self): + """save the current sources""" + # write an empty default config file if there aren't any sources + if len(self.list) == 0: + path = _APT_SOURCES_LIST + header = ( + "## See sources.list(5) for more information, especialy\n" + "# Remember that you can only use http, ftp or file URIs\n" + "# CDROMs are managed through the apt-cdrom tool.\n" + ) + + try: + with salt.utils.files.fopen(path, "w") as f: + f.write(header) + except FileNotFoundError: + # No need to create file if there is no apt directory + pass + return + + self.merge() + files = {} + for source in self.list: + if source.file not in files: + files[source.file] = [] + elif isinstance(source, Deb822SourceEntry): + files[source.file].append("\n") + files[source.file].append(source.str()) + for file in files: + with salt.utils.files.fopen(file, "w") as f: + f.write("".join(files[file])) + + def exploded_list(self): + """Present an exploded view of the list where each entry corresponds exactly to a Release file. + + A release file is uniquely identified by the triplet (type, uri, suite). Old style entries + always referred to a single release file, but deb822 entries allow multiple values for each + of those fields. """ - write all of the sources from the list of sources - to the file. - """ - filemap = {} - with tempfile.TemporaryDirectory() as tmpdir: - for source in self.list: - fname = pathlib.Path(tmpdir, pathlib.Path(source.file).name) - with salt.utils.files.fopen(str(fname), "a") as fp: - fp.write(source.repo_line()) - if source.file not in filemap: - filemap[source.file] = {"tmp": fname} - - for fp in filemap: - shutil.move(str(filemap[fp]["tmp"]), fp) + res: list[AnyExplodedSourceEntry] = [] + for entry in self.list: + if isinstance(entry, SourceEntry): + res.append(entry) + elif ( + len(entry.types) == 1 + and len(entry.uris) == 1 + and len(entry.suites) == 1 + ): + res.append(entry) + else: + for typ in entry.types: + for uri in entry.uris: + for sui in entry.suites: + res.append(ExplodedDeb822SourceEntry(entry, typ, uri, sui)) + + return res def _invalid(line): From 92333ba79697fc0f960e705a6e8d49259bdce665 Mon Sep 17 00:00:00 2001 From: vzhestkov Date: Wed, 2 Apr 2025 14:37:50 +0200 Subject: [PATCH 2/3] Fixes for test_aptpkg Co-authored-by: Marek Czernek --- tests/pytests/unit/modules/test_aptpkg.py | 103 ++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/tests/pytests/unit/modules/test_aptpkg.py b/tests/pytests/unit/modules/test_aptpkg.py index e0a07d8bb266..91787303a945 100644 --- a/tests/pytests/unit/modules/test_aptpkg.py +++ b/tests/pytests/unit/modules/test_aptpkg.py @@ -189,9 +189,11 @@ def __init__(self, uri, source_type, line, invalid, dist="", file=None): self.file = file self.disabled = False self.dist = dist + self.suites = [dist] self.comps = [] self.architectures = [] self.signedby = "" + self.types = [] def mysplit(self, line): return line.split() @@ -213,6 +215,107 @@ def configure_loader_modules(): return {aptpkg: {"__grains__": {}}} +@pytest.fixture +def deb822_repo_content(): + return """ +Types: deb +URIs: http://cz.archive.ubuntu.com/ubuntu/ +Suites: noble noble-updates noble-backports +Components: main +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +""" + + +@pytest.fixture +def deb822_repo_file(tmp_path: pathlib.Path, deb822_repo_content: str): + """ + Create a Debian-style repository in the deb822 format and return + the path of the repository file. + """ + repo = tmp_path / "sources.list.d" / "test.sources" + repo.parent.mkdir(parents=True, exist_ok=True) + repo.write_text(deb822_repo_content.strip(), encoding="UTF-8") + return repo + + +@pytest.fixture +def mock_apt_config(deb822_repo_file: pathlib.Path): + """ + Mocking common to deb822 testing so that apt_pkg uses the + tmp_path/sources.list.d as the sourceparts location + """ + with patch.dict( + aptpkg.__salt__, + {"config.option": MagicMock()}, + ) as mock_config, patch( + "salt.utils.pkg.deb._APT_SOURCES_PARTSDIR", + os.path.dirname(str(deb822_repo_file)), + ): + yield mock_config + + +def test_mod_repo_deb822_modify(deb822_repo_file: pathlib.Path, mock_apt_config): + """ + Test that aptpkg can modify an existing repository in the deb822 format. + In this test, we match the repository by name and disable it. + """ + uri = "http://cz.archive.ubuntu.com/ubuntu/" + repo = f"deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] {uri} noble main" + + aptpkg.mod_repo(repo, enabled=False, file=str(deb822_repo_file), refresh_db=False) + + repo_file = deb822_repo_file.read_text(encoding="UTF-8") + assert "Enabled: no" in repo_file + assert f"URIs: {uri}" in repo_file + + +def test_mod_repo_deb822_add(deb822_repo_file: pathlib.Path, mock_apt_config): + """ + Test that aptpkg can add a repository in the deb822 format. + """ + uri = "http://security.ubuntu.com/ubuntu/" + repo = f"deb [signed-by=/usr/share/keyrings/ubuntu-archive-keyring.gpg] {uri} noble-security main" + + aptpkg.mod_repo(repo, file=str(deb822_repo_file), refresh_db=False) + + repo_file = deb822_repo_file.read_text(encoding="UTF-8") + assert f"URIs: {uri}" in repo_file + assert "URIs: http://cz.archive.ubuntu.com/ubuntu/" in repo_file + + +def test_del_repo_deb822(deb822_repo_file: pathlib.Path, mock_apt_config): + """ + Test that aptpkg can delete a repository in the deb822 format. + """ + uri = "http://cz.archive.ubuntu.com/ubuntu/" + + with patch.object(aptpkg, "refresh_db"): + repo = f"deb {uri} noble main" + aptpkg.del_repo(repo, file=str(deb822_repo_file)) + assert os.path.isfile(str(deb822_repo_file)) + + repo = f"deb {uri} noble-updates main" + aptpkg.del_repo(repo, file=str(deb822_repo_file)) + assert os.path.isfile(str(deb822_repo_file)) + + repo = f"deb {uri} noble-backports main" + aptpkg.del_repo(repo, file=str(deb822_repo_file)) + assert not os.path.isfile(str(deb822_repo_file)) + + +def test_get_repo_deb822(deb822_repo_file: pathlib.Path, mock_apt_config): + """ + Test that aptpkg can match a repository in the deb822 format. + """ + uri = "http://cz.archive.ubuntu.com/ubuntu/" + repo = f"deb {uri} noble main" + + result = aptpkg.get_repo(repo) + + assert bool(result) + assert result["uri"] == uri + + def test_version(lowpkg_info_var): """ Test - Returns a string representing the package version or an empty string if From 00c5697d977420c35c0ac184f9a3803c3a77b794 Mon Sep 17 00:00:00 2001 From: vzhestkov Date: Thu, 24 Apr 2025 12:28:37 +0200 Subject: [PATCH 3/3] Changelog entry --- changelog/67956.added.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/67956.added.md diff --git a/changelog/67956.added.md b/changelog/67956.added.md new file mode 100644 index 000000000000..6e5d2b509d8d --- /dev/null +++ b/changelog/67956.added.md @@ -0,0 +1 @@ +Add deb822 fromat support to aptpkg module