From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 7F898138350 for ; Thu, 5 Mar 2020 08:26:11 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id B0AC6E094E; Thu, 5 Mar 2020 08:26:10 +0000 (UTC) Received: from smtp.gentoo.org (dev.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 84A82E094E for ; Thu, 5 Mar 2020 08:26:10 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 50A8034EADB for ; Thu, 5 Mar 2020 08:26:09 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id E9CDC103 for ; Thu, 5 Mar 2020 08:26:07 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1583395579.d389b3b378d88b8c41dfaba2a90bc9643a9ba99e.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/_emerge/ X-VCS-Repository: proj/portage X-VCS-Files: lib/_emerge/Scheduler.py X-VCS-Directories: lib/_emerge/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: d389b3b378d88b8c41dfaba2a90bc9643a9ba99e X-VCS-Branch: master Date: Thu, 5 Mar 2020 08:26:07 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: 09e155aa-5e72-452b-8e78-72e6c3b0578d X-Archives-Hash: 71ba12a867b8590ba35fe1f72e8c5811 commit: d389b3b378d88b8c41dfaba2a90bc9643a9ba99e Author: Zac Medico gentoo org> AuthorDate: Thu Mar 5 06:46:26 2020 +0000 Commit: Zac Medico gentoo org> CommitDate: Thu Mar 5 08:06:19 2020 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=d389b3b3 Scheduler: use add_done_callback (bug 711322) Use add_done_callback instead of addExistListener, in order to avoid callback races like the SequentialTaskQueue exit listener race that triggered bug 711322. The addExistListener method is prone to races because its listeners are executed in quick succession. In contrast, callbacks scheduled via add_done_callback are placed in a fifo queue, ensuring that they execute in an order that is unsurprising relative to other callbacks. Bug: https://bugs.gentoo.org/711322 Signed-off-by: Zac Medico gentoo.org> lib/_emerge/Scheduler.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py index 98eaf3bcc..98bc789ff 100644 --- a/lib/_emerge/Scheduler.py +++ b/lib/_emerge/Scheduler.py @@ -1,9 +1,10 @@ -# Copyright 1999-2019 Gentoo Authors +# Copyright 1999-2020 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division, print_function, unicode_literals from collections import deque +import functools import gc import gzip import logging @@ -1259,11 +1260,13 @@ class Scheduler(PollScheduler): child not in completed_tasks: unsatisfied.add(child) - def _merge_wait_exit_handler(self, task): + def _merge_wait_exit_handler(self, task, future): + future.cancelled() or future.result() self._merge_wait_scheduled.remove(task) - self._merge_exit(task) + self._merge_exit(task, future) - def _merge_exit(self, merge): + def _merge_exit(self, merge, future): + future.cancelled() or future.result() self._running_tasks.pop(id(merge), None) self._do_merge_exit(merge) self._deallocate_config(merge.merge.settings) @@ -1327,7 +1330,8 @@ class Scheduler(PollScheduler): del mtimedb["resume"] mtimedb.commit() - def _build_exit(self, build): + def _build_exit(self, build, future): + future.cancelled() or future.result() self._running_tasks.pop(id(build), None) if build.returncode == os.EX_OK and self._terminated_tasks: # We've been interrupted, so we won't @@ -1336,7 +1340,7 @@ class Scheduler(PollScheduler): self._deallocate_config(build.settings) elif build.returncode == os.EX_OK: self.curval += 1 - merge = PackageMerge(merge=build) + merge = PackageMerge(merge=build, scheduler=self._sched_iface) self._running_tasks[id(merge)] = merge if not build.build_opts.buildpkgonly and \ build.pkg in self._deep_system_deps: @@ -1345,8 +1349,8 @@ class Scheduler(PollScheduler): self._merge_wait_queue.append(merge) merge.addStartListener(self._system_merge_started) else: - merge.addExitListener(self._merge_exit) self._task_queues.merge.add(merge) + merge.async_wait().add_done_callback(functools.partial(self._merge_exit, merge)) self._status_display.merges = len(self._task_queues.merge) else: settings = build.settings @@ -1365,8 +1369,9 @@ class Scheduler(PollScheduler): self._status_display.running = self._jobs self._schedule() - def _extract_exit(self, build): - self._build_exit(build) + def _extract_exit(self, build, future): + future.cancelled() or future.result() + self._build_exit(build, future) def _task_complete(self, pkg): self._completed_tasks.add(pkg) @@ -1580,9 +1585,10 @@ class Scheduler(PollScheduler): if (self._merge_wait_queue and not self._jobs and not self._task_queues.merge): task = self._merge_wait_queue.popleft() - task.addExitListener(self._merge_wait_exit_handler) + task.scheduler = self._sched_iface self._merge_wait_scheduled.append(task) self._task_queues.merge.add(task) + task.async_wait().add_done_callback(functools.partial(self._merge_wait_exit_handler, task)) self._status_display.merges = len(self._task_queues.merge) state_change += 1 @@ -1699,26 +1705,28 @@ class Scheduler(PollScheduler): task = self._task(pkg) if pkg.installed: - merge = PackageMerge(merge=task) + merge = PackageMerge(merge=task, scheduler=self._sched_iface) self._running_tasks[id(merge)] = merge - merge.addExitListener(self._merge_exit) self._task_queues.merge.addFront(merge) + merge.async_wait().add_done_callback(functools.partial(self._merge_exit, merge)) elif pkg.built: self._jobs += 1 self._previous_job_start_time = time.time() self._status_display.running = self._jobs self._running_tasks[id(task)] = task - task.addExitListener(self._extract_exit) + task.scheduler = self._sched_iface self._task_queues.jobs.add(task) + task.async_wait().add_done_callback(functools.partial(self._extract_exit, task)) else: self._jobs += 1 self._previous_job_start_time = time.time() self._status_display.running = self._jobs self._running_tasks[id(task)] = task - task.addExitListener(self._build_exit) + task.scheduler = self._sched_iface self._task_queues.jobs.add(task) + task.async_wait().add_done_callback(functools.partial(self._build_exit, task)) return bool(state_change)