[gentoo-commits] proj/portage:master commit in: lib/portage/

["Sam James" <sam@gentoo.org>]

commit:     9e24d0143450628f334cdb62e579efafd1bfd2ba
Author:     Kenneth Raplee <kenrap <AT> kennethraplee <DOT> com>
AuthorDate: Sat Apr  2 01:13:57 2022 +0000
Commit:     Sam James <sam <AT> gentoo <DOT> org>
CommitDate: Mon Apr  4 19:04:34 2022 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=9e24d014

Simplify with declarative programming

Signed-off-by: Kenneth Raplee <kenrap <AT> kennethraplee.com>
Signed-off-by: Sam James <sam <AT> gentoo.org>

 lib/portage/manifest.py | 144 ++++++++++++++++++++++++------------------------
 lib/portage/metadata.py |  15 ++---
 lib/portage/module.py   |  23 +++++---
 lib/portage/news.py     |  14 ++---
 lib/portage/output.py   |  13 ++---
 5 files changed, 105 insertions(+), 104 deletions(-)

diff --git a/lib/portage/manifest.py b/lib/portage/manifest.py
index 5472e8fb1..ff166faa8 100644
--- a/lib/portage/manifest.py
+++ b/lib/portage/manifest.py
@@ -3,6 +3,7 @@
 
 import errno
 import io
+import itertools
 import logging
 import re
 import stat
@@ -107,9 +108,8 @@ class Manifest2Entry(ManifestEntry):
         myhashkeys = list(self.hashes)
         myhashkeys.remove("size")
         myhashkeys.sort()
-        for h in myhashkeys:
-            myline += " " + h + " " + str(self.hashes[h])
-        return myline
+        with_hashes = " ".join(f"{h} {self.hashes[h]}" for h in myhashkeys)
+        return f"{myline} {with_hashes}"
 
     def __eq__(self, other):
         if (
@@ -162,7 +162,6 @@ class Manifest:
             find_invalid_path_char = _find_invalid_path_char
         self._find_invalid_path_char = find_invalid_path_char
         self.pkgdir = _unicode_decode(pkgdir).rstrip(os.sep) + os.sep
-        self.fhashdict = {}
         self.hashes = set()
         self.required_hashes = set()
 
@@ -182,8 +181,8 @@ class Manifest:
         self.required_hashes.update(required_hashes)
         self.required_hashes.intersection_update(self.hashes)
 
-        for t in MANIFEST2_IDENTIFIERS:
-            self.fhashdict[t] = {}
+        self.fhashdict = {t: {} for t in MANIFEST2_IDENTIFIERS}
+
         if not from_scratch:
             self._read()
         if fetchlist_dict != None:
@@ -206,9 +205,9 @@ class Manifest:
 
     def getDigests(self):
         """Compability function for old digest/manifest code, returns dict of filename:{hashfunction:hashvalue}"""
-        rval = {}
-        for t in MANIFEST2_IDENTIFIERS:
-            rval.update(self.fhashdict[t])
+        rval = {
+            k: v for t in MANIFEST2_IDENTIFIERS for k, v in self.fhashdict[t].items()
+        }
         return rval
 
     def getTypeDigests(self, ftype):
@@ -269,18 +268,16 @@ class Manifest:
 
     def _getDigestData(self, distlist):
         """create a hash dict for a specific list of files"""
-        myhashdict = {}
-        for myname in distlist:
-            for mytype in self.fhashdict:
-                if myname in self.fhashdict[mytype]:
-                    myhashdict.setdefault(mytype, {})
-                    myhashdict[mytype].setdefault(myname, {})
-                    myhashdict[mytype][myname].update(self.fhashdict[mytype][myname])
+        myhashdict = {
+            mytype: {myname: self.fhashdict[mytype][myname]}
+            for myname in distlist
+            for mytype in self.fhashdict
+            if myname in self.fhashdict[mytype]
+        }
         return myhashdict
 
     def _createManifestEntries(self):
-        valid_hashes = set(get_valid_checksum_keys())
-        valid_hashes.add("size")
+        valid_hashes = set(itertools.chain(get_valid_checksum_keys(), ("size")))
         mytypes = list(self.fhashdict)
         mytypes.sort()
         for t in mytypes:
@@ -296,13 +293,22 @@ class Manifest:
                 yield myentry
 
     def checkIntegrity(self):
-        for t in self.fhashdict:
-            for f in self.fhashdict[t]:
-                diff = self.required_hashes.difference(set(self.fhashdict[t][f]))
-                if diff:
-                    raise MissingParameter(
-                        _("Missing %s checksum(s): %s %s") % (" ".join(diff), t, f)
+        manifest_data = (
+            (
+                self.required_hashes.difference(set(self.fhashdict[mytype][myfile])),
+                mytype,
+                myfile,
+            )
+            for mytype in self.fhashdict
+            for myfile in self.fhashdict[mytype]
+        )
+        for needed_hashes, its_type, its_file in manifest_data:
+            if needed_hashes:
+                raise MissingParameter(
+                    _(
+                        f"Missing {' '.join(needed_hashes)} checksum(s): {its_type} {its_file}"
                     )
+                )
 
     def write(self, sign=False, force=False):
         """Write Manifest instance to disk, optionally signing it. Returns
@@ -488,10 +494,8 @@ class Manifest:
 
     def findFile(self, fname):
         """Return entrytype of the given file if present in Manifest or None if not present"""
-        for t in MANIFEST2_IDENTIFIERS:
-            if fname in self.fhashdict[t]:
-                return t
-        return None
+        found_entries = (t for t in MANIFEST2_IDENTIFIERS if fname in self.fhashdict[t])
+        return next(found_entries, None)
 
     def create(
         self,
@@ -529,18 +533,19 @@ class Manifest:
             find_invalid_path_char=self._find_invalid_path_char,
             strict_misc_digests=self.strict_misc_digests,
         )
-        pn = os.path.basename(self.pkgdir.rstrip(os.path.sep))
-        cat = self._pkgdir_category()
 
-        pkgdir = self.pkgdir
+        update_pkgdir = self._update_thick_pkgdir
         if self.thin:
-            cpvlist = self._update_thin_pkgdir(cat, pn, pkgdir)
-        else:
-            cpvlist = self._update_thick_pkgdir(cat, pn, pkgdir)
+            update_pkgdir = self._update_thin_pkgdir
 
-        distlist = set()
-        for cpv in cpvlist:
-            distlist.update(self._getCpvDistfiles(cpv))
+        cpvlist = update_pkgdir(
+            self._pkgdir_category(),
+            os.path.basename(self.pkgdir.rstrip(os.path.sep)),
+            self.pkgdir,
+        )
+        distlist = set(
+            distfile for cpv in cpvlist for distfile in self._getCpvDistfiles(cpv)
+        )
 
         if requiredDistfiles is None:
             # This allows us to force removal of stale digests for the
@@ -550,9 +555,7 @@ class Manifest:
             # repoman passes in an empty list, which implies that all distfiles
             # are required.
             requiredDistfiles = distlist.copy()
-        required_hash_types = set()
-        required_hash_types.add("size")
-        required_hash_types.update(self.required_hashes)
+        required_hash_types = set(itertools.chain(self.required_hashes, ("size")))
         for f in distlist:
             fname = os.path.join(self.distdir, f)
             mystat = None
@@ -599,25 +602,28 @@ class Manifest:
         return cpv
 
     def _update_thin_pkgdir(self, cat, pn, pkgdir):
-        for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
-            break
-        cpvlist = []
-        for f in pkgdir_files:
+        _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None))
+
+        def _process_for_cpv(filename):
             try:
-                f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
+                filename = _unicode_decode(
+                    filename, encoding=_encodings["fs"], errors="strict"
+                )
             except UnicodeDecodeError:
-                continue
-            if f[:1] == ".":
-                continue
-            pf = self._is_cpv(cat, pn, f)
+                return None
+            if filename.startswith("."):
+                return None
+            pf = self._is_cpv(cat, pn, filename)
             if pf is not None:
-                cpvlist.append(pf)
+                return pf
+
+        processed = (_process_for_cpv(filename) for filename in pkgdir_files)
+        cpvlist = [pf for pf in processed if pf]
         return cpvlist
 
     def _update_thick_pkgdir(self, cat, pn, pkgdir):
+        _, _, pkgdir_files = next(os.walk(pkgdir), (None, None, None))
         cpvlist = []
-        for pkgdir, pkgdir_dirs, pkgdir_files in os.walk(pkgdir):
-            break
         for f in pkgdir_files:
             try:
                 f = _unicode_decode(f, encoding=_encodings["fs"], errors="strict")
@@ -714,9 +720,7 @@ class Manifest:
         return self.fetchlist_dict[cpv]
 
     def getDistfilesSize(self, fetchlist):
-        total_bytes = 0
-        for f in fetchlist:
-            total_bytes += int(self.fhashdict["DIST"][f]["size"])
+        total_bytes = sum(int(self.fhashdict["DIST"][f]["size"]) for f in fetchlist)
         return total_bytes
 
     def updateFileHashes(
@@ -784,28 +788,26 @@ class Manifest:
 
     def getVersions(self):
         """Returns a list of manifest versions present in the manifest file."""
-        rVal = []
         mfname = self.getFullname()
         if not os.path.exists(mfname):
-            return rVal
-        myfile = io.open(
+            return []
+        with io.open(
             _unicode_encode(mfname, encoding=_encodings["fs"], errors="strict"),
             mode="r",
             encoding=_encodings["repo.content"],
             errors="replace",
-        )
-        lines = myfile.readlines()
-        myfile.close()
-        for l in lines:
-            mysplit = l.split()
-            if (
-                len(mysplit) > 4
-                and mysplit[0] in MANIFEST2_IDENTIFIERS
-                and ((len(mysplit) - 3) % 2) == 0
-                and not 2 in rVal
-            ):
-                rVal.append(2)
-        return rVal
+        ) as myfile:
+            line_splits = (line.split() for line in myfile.readlines())
+            validation = (
+                True
+                for line_split in line_splits
+                if len(line_split) > 4
+                and line_split[0] in MANIFEST2_IDENTIFIERS
+                and (len(line_split) - 3) % 2 == 0
+            )
+            if any(validation):
+                return [2]
+        return []
 
     def _catsplit(self, pkg_key):
         """Split a category and package, returning a list of [cat, pkg].

diff --git a/lib/portage/metadata.py b/lib/portage/metadata.py
index 357917051..869c10bb3 100644
--- a/lib/portage/metadata.py
+++ b/lib/portage/metadata.py
@@ -75,8 +75,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
             eclass_db.update_eclasses()
             porttrees_data.append(TreeData(portdb.auxdb[path], eclass_db, path, src_db))
 
-    porttrees = [tree_data.path for tree_data in porttrees_data]
-
     quiet = (
         settings.get("TERM") == "dumb" or "--quiet" in myopts or not sys.stdout.isatty()
     )
@@ -100,7 +98,7 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
     # Temporarily override portdb.porttrees so portdb.cp_all()
     # will only return the relevant subset.
     portdb_porttrees = portdb.porttrees
-    portdb.porttrees = porttrees
+    portdb.porttrees = (tree_data.path for tree_data in porttrees_data)
     try:
         cp_all = portdb.cp_all()
     finally:
@@ -119,7 +117,6 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
 
     for cp in cp_all:
         for tree_data in porttrees_data:
-
             src_chf = tree_data.src_db.validation_chf
             dest_chf = tree_data.dest_db.validation_chf
             dest_chf_key = f"_{dest_chf}_"
@@ -190,11 +187,11 @@ def action_metadata(settings, portdb, myopts, porttrees=None):
                         # We don't want to skip the write unless we're really
                         # sure that the existing cache is identical, so don't
                         # trust _mtime_ and _eclasses_ alone.
-                        for k in auxdbkeys:
-                            if dest.get(k, "") != src.get(k, ""):
-                                dest = None
-                                break
-
+                        cache_is_identical = (
+                            True for k in auxdbkeys if dest.get(k, "") != src.get(k, "")
+                        )
+                        if any(cache_is_identical):
+                            dest = None
                 if dest is not None:
                     # The existing data is valid and identical,
                     # so there's no need to overwrite it.

diff --git a/lib/portage/module.py b/lib/portage/module.py
index 61c85aa47..8e63cd545 100644
--- a/lib/portage/module.py
+++ b/lib/portage/module.py
@@ -110,19 +110,27 @@ class Modules:
         @rtype: dictionary of module_plugins
         """
         module_dir = self._module_path
-        importables = []
         names = os.listdir(module_dir)
-        for entry in names:
-            # skip any __init__ or __pycache__ files or directories
-            if entry.startswith("__"):
-                continue
+
+        def _a_real_module(entry):
             try:
                 # test for statinfo to ensure it should a real module
                 # it will bail if it errors
                 os.lstat(os.path.join(module_dir, entry, "__init__.py"))
-                importables.append(entry)
             except EnvironmentError:
-                pass
+                return False
+            return True
+
+        # The importables list cannot be a generator.
+        # If it was a generator, it would be consumed by self.parents.extend()
+        # and the following for loop wouldn't have anything to iterate with.
+        importables = [
+            entry
+            for entry in names
+            if not entry.startswith("__") and _a_real_module(entry)
+        ]
+        self.parents.extend(importables)
+
         kids = {}
         for entry in importables:
             new_module = Module(entry, self._namepath)
@@ -131,7 +139,6 @@ class Modules:
                 kid = new_module.kids[module_name]
                 kid["parent"] = new_module
                 kids[kid["name"]] = kid
-            self.parents.append(entry)
         return kids
 
     def get_module_names(self):

diff --git a/lib/portage/news.py b/lib/portage/news.py
index 9ef6efde0..9f373d3d7 100644
--- a/lib/portage/news.py
+++ b/lib/portage/news.py
@@ -280,14 +280,12 @@ class NewsItem:
 
         kwargs = {"vardb": vardb, "config": config, "profile": profile}
 
-        all_match = True
-        for values in self.restrictions.values():
-            any_match = False
-            for restriction in values:
-                if restriction.checkRestriction(**kwargs):
-                    any_match = True
-            if not any_match:
-                all_match = False
+        all_match = all(
+            True
+            for values in self.restrictions.values()
+            for restriction in values
+            if restriction.checkRestriction(**kwargs)
+        )
 
         return all_match
 

diff --git a/lib/portage/output.py b/lib/portage/output.py
index 33c477012..e20046fc5 100644
--- a/lib/portage/output.py
+++ b/lib/portage/output.py
@@ -5,6 +5,7 @@ __docformat__ = "epytext"
 
 import errno
 import io
+import itertools
 import re
 import subprocess
 import sys
@@ -74,16 +75,12 @@ codes["bg_darkyellow"] = codes["bg_brown"]
 
 
 def color(fg, bg="default", attr=["normal"]):
-    mystr = codes[fg]
-    for x in [bg] + attr:
-        mystr += codes[x]
-    return mystr
+    myansicodechain = itertools.chain((codes[fg]), (codes[x] for x in [bg, *attr]))
+    return "".join(myansicodechain)
 
 
-ansi_codes = []
-for x in range(30, 38):
-    ansi_codes.append("%im" % x)
-    ansi_codes.append("%i;01m" % x)
+ansi_codes = [y for x in range(30, 38) for y in (f"{x}m", f"{x};01m")]
+
 
 rgb_ansi_colors = [
     "0x000000",