[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...

[gentoo-commits] proj/portage:master commit in: lib/portage/util/futures/_asyncio/, lib/portage/package/ebuild/, ...

[Zac Medico]

commit:     18cdb6331a66c1cc92f296b1aaf0538f63586875
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Feb 13 08:44:50 2024 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Feb 21 15:27:31 2024 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=18cdb633

EbuildPhase: async_check_locale

Change config.environ() check_locale calls to async_check_locale
calls in the EbuildPhase _async_start method in order to eliminate
synchronous waiting for child processes in the main event loop
thread.

Bug: https://bugs.gentoo.org/923841
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/_emerge/EbuildMetadataPhase.py            |  4 ++++
 lib/_emerge/EbuildPhase.py                    | 28 ++++++++++++++++++++++++++-
 lib/portage/package/ebuild/config.py          | 26 +++++++++++--------------
 lib/portage/util/futures/_asyncio/__init__.py |  9 +++++++++
 lib/portage/util/locale.py                    | 28 ++++++++++++++++++---------
 5 files changed, 70 insertions(+), 25 deletions(-)

diff --git a/lib/_emerge/EbuildMetadataPhase.py b/lib/_emerge/EbuildMetadataPhase.py
index fc64d84c94..54177840c7 100644
--- a/lib/_emerge/EbuildMetadataPhase.py
+++ b/lib/_emerge/EbuildMetadataPhase.py
@@ -8,6 +8,7 @@ import portage
 
 portage.proxy.lazyimport.lazyimport(
     globals(),
+    "_emerge.EbuildPhase:_setup_locale",
     "portage.package.ebuild._metadata_invalid:eapi_invalid",
 )
 from portage import os
@@ -83,6 +84,9 @@ class EbuildMetadataPhase(SubProcess):
         settings.setcpv(self.cpv)
         settings.configdict["pkg"]["EAPI"] = parsed_eapi
 
+        # This requires above setcpv and EAPI setup.
+        await _setup_locale(self.settings)
+
         debug = settings.get("PORTAGE_DEBUG") == "1"
         master_fd = None
         slave_fd = None

diff --git a/lib/_emerge/EbuildPhase.py b/lib/_emerge/EbuildPhase.py
index 3b366f39c7..b472803438 100644
--- a/lib/_emerge/EbuildPhase.py
+++ b/lib/_emerge/EbuildPhase.py
@@ -1,4 +1,4 @@
-# Copyright 1999-2021 Gentoo Authors
+# Copyright 1999-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 import functools
@@ -24,6 +24,7 @@ from portage.package.ebuild.prepare_build_dirs import (
     _prepare_fake_distdir,
     _prepare_fake_filesdir,
 )
+from portage.eapi import _get_eapi_attrs
 from portage.util import writemsg, ensure_dirs
 from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
 from portage.util._async.BuildLogger import BuildLogger
@@ -54,12 +55,34 @@ portage.proxy.lazyimport.lazyimport(
     + "_post_src_install_write_metadata,"
     + "_preinst_bsdflags",
     "portage.util.futures.unix_events:_set_nonblocking",
+    "portage.util.locale:async_check_locale,split_LC_ALL",
 )
 from portage import os
 from portage import _encodings
 from portage import _unicode_encode
 
 
+async def _setup_locale(settings):
+    eapi_attrs = _get_eapi_attrs(settings["EAPI"])
+    if eapi_attrs.posixish_locale:
+        split_LC_ALL(settings)
+        settings["LC_COLLATE"] = "C"
+        # check_locale() returns None when check can not be executed.
+        if await async_check_locale(silent=True, env=settings.environ()) is False:
+            # try another locale
+            for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
+                settings["LC_CTYPE"] = l
+                if await async_check_locale(silent=True, env=settings.environ()):
+                    # TODO: output the following only once
+                    # writemsg(
+                    #     _("!!! LC_CTYPE unsupported, using %s instead\n")
+                    #     % self.settings["LC_CTYPE"]
+                    # )
+                    break
+            else:
+                raise AssertionError("C locale did not pass the test!")
+
+
 class EbuildPhase(CompositeTask):
     __slots__ = ("actionmap", "fd_pipes", "phase", "settings") + ("_ebuild_lock",)
 
@@ -95,6 +118,9 @@ class EbuildPhase(CompositeTask):
         self._start_task(AsyncTaskFuture(future=future), self._async_start_exit)
 
     async def _async_start(self):
+
+        await _setup_locale(self.settings)
+
         need_builddir = self.phase not in EbuildProcess._phases_without_builddir
 
         if need_builddir:

diff --git a/lib/portage/package/ebuild/config.py b/lib/portage/package/ebuild/config.py
index c89354cbf7..bafdc55a08 100644
--- a/lib/portage/package/ebuild/config.py
+++ b/lib/portage/package/ebuild/config.py
@@ -29,7 +29,6 @@ portage.proxy.lazyimport.lazyimport(
     "portage.dbapi.vartree:vartree",
     "portage.package.ebuild.doebuild:_phase_func_map",
     "portage.util.compression_probe:_compressors",
-    "portage.util.locale:check_locale,split_LC_ALL",
 )
 from portage import bsd_chflags, load_mod, os, selinux, _unicode_decode
 from portage.const import (
@@ -3371,20 +3370,17 @@ class config:
                 mydict["EBUILD_PHASE_FUNC"] = phase_func
 
         if eapi_attrs.posixish_locale:
-            split_LC_ALL(mydict)
-            mydict["LC_COLLATE"] = "C"
-            # check_locale() returns None when check can not be executed.
-            if check_locale(silent=True, env=mydict) is False:
-                # try another locale
-                for l in ("C.UTF-8", "en_US.UTF-8", "en_GB.UTF-8", "C"):
-                    mydict["LC_CTYPE"] = l
-                    if check_locale(silent=True, env=mydict):
-                        # TODO: output the following only once
-                        # 						writemsg(_("!!! LC_CTYPE unsupported, using %s instead\n")
-                        # 								% mydict["LC_CTYPE"])
-                        break
-                else:
-                    raise AssertionError("C locale did not pass the test!")
+            if mydict.get("LC_ALL"):
+                # Sometimes this method is called for processes
+                # that are not ebuild phases, so only raise
+                # AssertionError for actual ebuild phases.
+                if phase and phase not in ("clean", "cleanrm", "fetch"):
+                    raise AssertionError(
+                        f"LC_ALL={mydict['LC_ALL']} for posixish locale. It seems that split_LC_ALL was not called for phase {phase}?"
+                    )
+            elif "LC_ALL" in mydict:
+                # Delete placeholder from split_LC_ALL.
+                del mydict["LC_ALL"]
 
         if not eapi_attrs.exports_PORTDIR:
             mydict.pop("PORTDIR", None)

diff --git a/lib/portage/util/futures/_asyncio/__init__.py b/lib/portage/util/futures/_asyncio/__init__.py
index b6481c281e..22241f335d 100644
--- a/lib/portage/util/futures/_asyncio/__init__.py
+++ b/lib/portage/util/futures/_asyncio/__init__.py
@@ -16,6 +16,7 @@ __all__ = (
     "set_child_watcher",
     "get_event_loop_policy",
     "set_event_loop_policy",
+    "run",
     "shield",
     "sleep",
     "Task",
@@ -109,6 +110,14 @@ def set_child_watcher(watcher):
     return get_event_loop_policy().set_child_watcher(watcher)
 
 
+# Emulate run since it's the preferred python API.
+def run(coro):
+    return _safe_loop().run_until_complete(coro)
+
+
+run.__doc__ = _real_asyncio.run.__doc__
+
+
 def create_subprocess_exec(*args, **kwargs):
     """
     Create a subprocess.

diff --git a/lib/portage/util/locale.py b/lib/portage/util/locale.py
index b5da8d949b..b6a41e7655 100644
--- a/lib/portage/util/locale.py
+++ b/lib/portage/util/locale.py
@@ -17,6 +17,7 @@ import traceback
 import portage
 from portage.util import _unicode_decode, writemsg_level
 from portage.util._ctypes import find_library, LoadLibrary
+from portage.util.futures import asyncio
 
 
 locale_categories = (
@@ -121,7 +122,10 @@ def check_locale(silent=False, env=None):
     warning and returns False if it is not. Returns None if the check
     can not be executed due to platform limitations.
     """
+    return asyncio.run(async_check_locale(silent=silent, env=env))
 
+
+async def async_check_locale(silent=False, env=None):
     if env is not None:
         for v in ("LC_ALL", "LC_CTYPE", "LANG"):
             if v in env:
@@ -135,20 +139,17 @@ def check_locale(silent=False, env=None):
         except KeyError:
             pass
 
-    # TODO: Make async version of check_locale and call it from
-    # EbuildPhase instead of config.environ(), since it's bad to
-    # synchronously wait for the process in the main event loop
-    # thread where config.environ() tends to be called.
     proc = multiprocessing.Process(
         target=_set_and_check_locale,
         args=(silent, env, None if env is None else portage._native_string(mylocale)),
     )
     proc.start()
-    proc.join()
+    proc = portage.process.MultiprocessingProcess(proc)
+    await proc.wait()
 
     pyret = None
-    if proc.exitcode >= 0:
-        ret = proc.exitcode
+    if proc.returncode >= 0:
+        ret = proc.returncode
         if ret != 2:
             pyret = ret == 0
 
@@ -157,13 +158,22 @@ def check_locale(silent=False, env=None):
     return pyret
 
 
+async_check_locale.__doc__ = check_locale.__doc__
+async_check_locale.__doc__ += """
+    This function is a coroutine.
+"""
+
+
 def split_LC_ALL(env):
     """
     Replace LC_ALL with split-up LC_* variables if it is defined.
     Works on the passed environment (or settings instance).
     """
     lc_all = env.get("LC_ALL")
-    if lc_all is not None:
+    if lc_all:
         for c in locale_categories:
             env[c] = lc_all
-        del env["LC_ALL"]
+        # Set empty so that config.reset() can restore LC_ALL state,
+        # since del can permanently delete variables which are not
+        # stored in the config's backupenv.
+        env["LC_ALL"] = ""