* [gentoo-commits] proj/portage:master commit in: bin/, pym/portage/tests/, pym/portage/util/_eventloop/, ...
@ 2018-05-06 22:54 Zac Medico
0 siblings, 0 replies; only message in thread
From: Zac Medico @ 2018-05-06 22:54 UTC (permalink / raw
To: gentoo-commits
commit: 2d500ce2bc96995752dfc2fb475a7abe907e38b6
Author: Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sun May 6 21:28:25 2018 +0000
Commit: Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun May 6 22:51:23 2018 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d500ce2
asyncio: explicitly close event loops (bug 654390)
The default asyncio event loop triggers a resource warning if it
is not explicitly closed, therefore close it when appropriate.
Bug: https://bugs.gentoo.org/654390
bin/ebuild | 4 ++++
bin/ebuild-ipc.py | 5 ++++-
bin/egencache | 5 ++++-
bin/emaint | 3 +++
bin/emerge | 5 +++++
bin/emirrordist | 6 +++++-
bin/portageq | 6 +++++-
bin/quickpkg | 3 +++
pym/portage/tests/runTests.py | 6 +++++-
pym/portage/tests/util/futures/asyncio/test_child_watcher.py | 5 +++++
pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py | 6 ++++++
pym/portage/tests/util/futures/asyncio/test_pipe_closed.py | 7 +++++++
pym/portage/tests/util/futures/asyncio/test_run_until_complete.py | 5 +++++
pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py | 7 ++++++-
pym/portage/tests/util/futures/test_retry.py | 5 ++++-
pym/portage/util/_eventloop/global_event_loop.py | 5 ++++-
16 files changed, 75 insertions(+), 8 deletions(-)
diff --git a/bin/ebuild b/bin/ebuild
index 5221b21a8..710257549 100755
--- a/bin/ebuild
+++ b/bin/ebuild
@@ -55,6 +55,7 @@ from portage.exception import PermissionDenied, PortageKeyError, \
PortagePackageException, UnsupportedAPIException
from portage.localization import _
import portage.util
+from portage.util._eventloop.global_event_loop import global_event_loop
from _emerge.Package import Package
from _emerge.RootConfig import RootConfig
@@ -371,4 +372,7 @@ for arg in pargs:
print("Could not run the required binary?")
a = 127
if a:
+ global_event_loop().close()
sys.exit(a)
+
+global_event_loop().close()
diff --git a/bin/ebuild-ipc.py b/bin/ebuild-ipc.py
index 6d0cdbef9..1f323bdc5 100755
--- a/bin/ebuild-ipc.py
+++ b/bin/ebuild-ipc.py
@@ -273,4 +273,7 @@ def ebuild_ipc_main(args):
return ebuild_ipc.communicate(args)
if __name__ == '__main__':
- sys.exit(ebuild_ipc_main(sys.argv[1:]))
+ try:
+ sys.exit(ebuild_ipc_main(sys.argv[1:]))
+ finally:
+ global_event_loop().close()
diff --git a/bin/egencache b/bin/egencache
index e994b4ab1..70fb5aa00 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -1116,4 +1116,7 @@ def egencache_main(args):
if __name__ == "__main__":
portage._disable_legacy_globals()
portage.util.noiselimit = -1
- sys.exit(egencache_main(sys.argv[1:]))
+ try:
+ sys.exit(egencache_main(sys.argv[1:]))
+ finally:
+ global_event_loop().close()
diff --git a/bin/emaint b/bin/emaint
index 08e75851a..a26dae1e7 100755
--- a/bin/emaint
+++ b/bin/emaint
@@ -31,6 +31,7 @@ if osp.isfile(osp.join(osp.dirname(osp.dirname(osp.realpath(__file__))), ".porta
import portage
portage._internal_caller = True
from portage.emaint.main import emaint_main
+from portage.util._eventloop.global_event_loop import global_event_loop
try:
emaint_main(sys.argv[1:])
@@ -40,3 +41,5 @@ except IOError as e:
sys.exit(1)
else:
raise
+finally:
+ global_event_loop().close()
diff --git a/bin/emerge b/bin/emerge
index 5f08861e5..e1262d544 100755
--- a/bin/emerge
+++ b/bin/emerge
@@ -13,6 +13,7 @@ import sys
# exiting from signal handlers intermittently causes python to ignore
# the SystemExit exception with a message like this:
# Exception SystemExit: 130 in <function remove at 0x7fd2146c1320> ignored
+global_event_loop = None
try:
def exithandler(signum, _frame):
@@ -41,6 +42,7 @@ try:
import portage
portage._internal_caller = True
portage._disable_legacy_globals()
+ from portage.util._eventloop.global_event_loop import global_event_loop
from _emerge.main import emerge_main
if __name__ == "__main__":
@@ -84,3 +86,6 @@ except KeyboardInterrupt:
{"signal": signal.SIGINT})
sys.stderr.flush()
sys.exit(128 + signal.SIGINT)
+finally:
+ if global_event_loop is not None:
+ global_event_loop().close()
diff --git a/bin/emirrordist b/bin/emirrordist
index 17f99f590..3ea08379e 100755
--- a/bin/emirrordist
+++ b/bin/emirrordist
@@ -9,6 +9,7 @@ import portage
portage._internal_caller = True
portage._disable_legacy_globals()
from portage._emirrordist.main import emirrordist_main
+from portage.util._eventloop.global_event_loop import global_event_loop
if __name__ == "__main__":
@@ -18,4 +19,7 @@ if __name__ == "__main__":
signal.signal(signal.SIGUSR1, debug_signal)
- sys.exit(emirrordist_main(sys.argv[1:]))
+ try:
+ sys.exit(emirrordist_main(sys.argv[1:]))
+ finally:
+ global_event_loop().close()
diff --git a/bin/portageq b/bin/portageq
index e9b8b20e0..35499afd2 100755
--- a/bin/portageq
+++ b/bin/portageq
@@ -53,6 +53,7 @@ portage.proxy.lazyimport.lazyimport(globals(),
'_emerge.is_valid_package_atom:insert_category_into_atom',
'portage.dbapi._expand_new_virt:expand_new_virt',
'portage._sets.base:InternalPackageSet',
+ 'portage.util._eventloop.global_event_loop:global_event_loop',
'portage.xml.metadata:MetaDataXML'
)
@@ -1466,6 +1467,9 @@ def main(argv):
sys.exit(1)
if __name__ == '__main__':
- sys.exit(main(sys.argv))
+ try:
+ sys.exit(main(sys.argv))
+ finally:
+ global_event_loop().close()
#-----------------------------------------------------------------------------
diff --git a/bin/quickpkg b/bin/quickpkg
index 9765ec717..f071dd904 100755
--- a/bin/quickpkg
+++ b/bin/quickpkg
@@ -30,6 +30,8 @@ from portage.checksum import perform_md5
from portage._sets import load_default_config, SETPREFIX
from portage.process import find_binary
from portage.util.compression_probe import _compressors
+from portage.util._eventloop.global_event_loop import global_event_loop
+
def quickpkg_atom(options, infos, arg, eout):
settings = portage.settings
@@ -390,4 +392,5 @@ if __name__ == "__main__":
finally:
os.umask(old_umask)
signal.signal(signal.SIGWINCH, signal.SIG_DFL)
+ global_event_loop().close()
sys.exit(retval)
diff --git a/pym/portage/tests/runTests.py b/pym/portage/tests/runTests.py
index 9c452764f..d4d1f7c76 100755
--- a/pym/portage/tests/runTests.py
+++ b/pym/portage/tests/runTests.py
@@ -42,6 +42,7 @@ if os.environ.get('NOCOLOR') in ('yes', 'true'):
portage.output.nocolor()
import portage.tests as tests
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.const import PORTAGE_BIN_PATH
path = os.environ.get("PATH", "").split(":")
path = [x for x in path if x]
@@ -58,4 +59,7 @@ if insert_bin_path:
os.environ["PATH"] = ":".join(path)
if __name__ == "__main__":
- sys.exit(tests.main())
+ try:
+ sys.exit(tests.main())
+ finally:
+ global_event_loop().close()
diff --git a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
index 8ef497544..0fc73ab49 100644
--- a/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
+++ b/pym/portage/tests/util/futures/asyncio/test_child_watcher.py
@@ -5,6 +5,7 @@ import os
from portage.process import find_binary, spawn
from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -18,6 +19,7 @@ class ChildWatcherTestCase(TestCase):
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+ loop = None
try:
try:
asyncio.set_child_watcher(None)
@@ -43,3 +45,6 @@ class ChildWatcherTestCase(TestCase):
(pids[0], os.EX_OK, args_tuple))
finally:
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
diff --git a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
index 19588bf3a..177953437 100644
--- a/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
+++ b/pym/portage/tests/util/futures/asyncio/test_event_loop_in_fork.py
@@ -5,6 +5,7 @@ import multiprocessing
import os
from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -15,6 +16,7 @@ def fork_main(parent_conn, child_conn):
# This fails with python's default event loop policy,
# see https://bugs.python.org/issue22087.
loop.run_until_complete(asyncio.sleep(0.1, loop=loop))
+ loop.close()
def async_main(fork_exitcode, loop=None):
@@ -47,6 +49,7 @@ class EventLoopInForkTestCase(TestCase):
initial_policy = asyncio.get_event_loop_policy()
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+ loop = None
try:
loop = asyncio._wrap_loop()
fork_exitcode = loop.create_future()
@@ -57,3 +60,6 @@ class EventLoopInForkTestCase(TestCase):
assert loop.run_until_complete(fork_exitcode) == os.EX_OK
finally:
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
diff --git a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
index c2b468064..507385c04 100644
--- a/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
+++ b/pym/portage/tests/util/futures/asyncio/test_pipe_closed.py
@@ -10,6 +10,7 @@ import sys
import tempfile
from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures.unix_events import (
DefaultEventLoopPolicy,
@@ -83,6 +84,9 @@ class ReaderPipeClosedTestCase(_PipeClosedTestCase, TestCase):
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase):
@@ -142,3 +146,6 @@ class WriterPipeClosedTestCase(_PipeClosedTestCase, TestCase):
write_end.close()
read_end.close()
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
diff --git a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py
index 1a37e4922..c0e86ae5e 100644
--- a/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py
+++ b/pym/portage/tests/util/futures/asyncio/test_run_until_complete.py
@@ -2,6 +2,7 @@
# Distributed under the terms of the GNU General Public License v2
from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -12,6 +13,7 @@ class RunUntilCompleteTestCase(TestCase):
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+ loop = None
try:
loop = asyncio._wrap_loop()
f1 = loop.create_future()
@@ -27,3 +29,6 @@ class RunUntilCompleteTestCase(TestCase):
self.assertEqual(f2.done(), True)
finally:
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
diff --git a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
index 8dc5fa7b9..534d79c53 100644
--- a/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
+++ b/pym/portage/tests/util/futures/asyncio/test_subprocess_exec.py
@@ -6,6 +6,7 @@ import subprocess
from portage.process import find_binary
from portage.tests import TestCase
+from portage.util._eventloop.global_event_loop import global_event_loop
from portage.util.futures import asyncio
from portage.util.futures.executor.fork import ForkExecutor
from portage.util.futures.unix_events import DefaultEventLoopPolicy
@@ -60,10 +61,14 @@ class SubprocessExecTestCase(TestCase):
if not isinstance(initial_policy, DefaultEventLoopPolicy):
asyncio.set_event_loop_policy(DefaultEventLoopPolicy())
+ loop = asyncio._wrap_loop()
try:
- test(asyncio._wrap_loop())
+ test(loop)
finally:
asyncio.set_event_loop_policy(initial_policy)
+ if loop not in (None, global_event_loop()):
+ loop.close()
+ self.assertFalse(global_event_loop().is_closed())
def testEcho(self):
if not hasattr(asyncio, 'create_subprocess_exec'):
diff --git a/pym/portage/tests/util/futures/test_retry.py b/pym/portage/tests/util/futures/test_retry.py
index 781eac9a1..baf293d56 100644
--- a/pym/portage/tests/util/futures/test_retry.py
+++ b/pym/portage/tests/util/futures/test_retry.py
@@ -183,7 +183,10 @@ class RetryExecutorTestCase(RetryTestCase):
return result.result()
else:
# child process
- return loop.run_until_complete(coroutine_func())
+ try:
+ return loop.run_until_complete(coroutine_func())
+ finally:
+ loop.close()
def execute_wrapper():
kill_switch = parent_loop.create_future()
diff --git a/pym/portage/util/_eventloop/global_event_loop.py b/pym/portage/util/_eventloop/global_event_loop.py
index e2c7d71ea..a3ee9248d 100644
--- a/pym/portage/util/_eventloop/global_event_loop.py
+++ b/pym/portage/util/_eventloop/global_event_loop.py
@@ -29,6 +29,9 @@ def global_event_loop():
if not constructor.supports_multiprocessing and pid != _MAIN_PID:
constructor = _multiprocessing_constructor
- instance = constructor()
+ # Use the _asyncio_wrapper attribute, so that unit tests can compare
+ # the reference to one retured from _wrap_loop(), since they should
+ # not close the loop if it refers to a global event loop.
+ instance = constructor()._asyncio_wrapper
_instances[pid] = instance
return instance
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2018-05-06 22:54 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2018-05-06 22:54 [gentoo-commits] proj/portage:master commit in: bin/, pym/portage/tests/, pym/portage/util/_eventloop/, Zac Medico
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox