From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Q3ZDP-0007j0-73 for garchives@archives.gentoo.org; Sat, 26 Mar 2011 19:32:03 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 344A31C06F; Sat, 26 Mar 2011 19:31:05 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id E959A1C06F for ; Sat, 26 Mar 2011 19:31:04 +0000 (UTC) Received: from pelican.gentoo.org (unknown [66.219.59.40]) (using TLSv1 with cipher ADH-CAMELLIA256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 92BC71B4115 for ; Sat, 26 Mar 2011 19:31:04 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by pelican.gentoo.org (Postfix) with ESMTP id 079F380072 for ; Sat, 26 Mar 2011 19:31:04 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <23b334d6553fb637ebe8e38f78183e0955180f0f.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:2.1.9 commit in: pym/_emerge/ X-VCS-Repository: proj/portage X-VCS-Files: pym/_emerge/BlockerDB.py pym/_emerge/FakeVartree.py pym/_emerge/Scheduler.py X-VCS-Directories: pym/_emerge/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 23b334d6553fb637ebe8e38f78183e0955180f0f Date: Sat, 26 Mar 2011 19:31:04 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: X-Archives-Hash: cbd5b56eb6dea0af58a1508b82badb6b commit: 23b334d6553fb637ebe8e38f78183e0955180f0f Author: David James google com> AuthorDate: Fri Mar 25 01:14:45 2011 +0000 Commit: Zac Medico gentoo org> CommitDate: Sat Mar 26 19:21:27 2011 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/portage.git;a= =3Dcommit;h=3D23b334d6 Update Portage to sync BlockerDB at init. Currently, Portage syncs the BlockerDB before every merge. This slows down merges considerably because it requires reading the entire vardb from disk. Since Portage doesn't merge conflicting packages anyway, we can optimize this by just reading the vardb at initialization and caching that. Change-Id: I6701926f022ef3aa2da10482fc8b09573ae24610 Review URL: http://codereview.chromium.org/6688037 --- pym/_emerge/BlockerDB.py | 13 ++++++------- pym/_emerge/FakeVartree.py | 18 ++++++++++++------ pym/_emerge/Scheduler.py | 24 +++++++----------------- 3 files changed, 25 insertions(+), 30 deletions(-) diff --git a/pym/_emerge/BlockerDB.py b/pym/_emerge/BlockerDB.py index fd4bf5e..f5adc4d 100644 --- a/pym/_emerge/BlockerDB.py +++ b/pym/_emerge/BlockerDB.py @@ -30,12 +30,7 @@ class BlockerDB(object): "vartree" : fake_vartree, }} =20 - def _get_fake_vartree(self, acquire_lock=3D0): - fake_vartree =3D self._fake_vartree - fake_vartree.sync(acquire_lock=3Dacquire_lock) - return fake_vartree - - def findInstalledBlockers(self, new_pkg, acquire_lock=3D0): + def findInstalledBlockers(self, new_pkg): """ Search for installed run-time blockers in the root where new_pkg is planned to be installed. This ignores build-time @@ -45,7 +40,7 @@ class BlockerDB(object): dep_keys =3D ["RDEPEND", "PDEPEND"] settings =3D self._vartree.settings stale_cache =3D set(blocker_cache) - fake_vartree =3D self._get_fake_vartree(acquire_lock=3Dacquire_lock) + fake_vartree =3D self._fake_vartree dep_check_trees =3D self._dep_check_trees vardb =3D fake_vartree.dbapi installed_pkgs =3D list(vardb) @@ -118,3 +113,7 @@ class BlockerDB(object): =20 return blocking_pkgs =20 + def discardBlocker(self, pkg): + """Discard a package from the list of potential blockers.""" + self._fake_vartree.cpv_discard(pkg) + diff --git a/pym/_emerge/FakeVartree.py b/pym/_emerge/FakeVartree.py index 977824c..21bd355 100644 --- a/pym/_emerge/FakeVartree.py +++ b/pym/_emerge/FakeVartree.py @@ -100,6 +100,16 @@ class FakeVartree(vartree): pkg, self.dbapi, self._global_updates) return self._aux_get(pkg, wants) =20 + def cpv_discard(self, pkg): + """ + Discard a package from the fake vardb if it exists. + """ + old_pkg =3D self.dbapi.get(pkg) + if old_pkg is not None: + self.dbapi.cpv_remove(old_pkg) + self._pkg_cache.pop(old_pkg, None) + self._aux_get_history.discard(old_pkg.cpv) + def sync(self, acquire_lock=3D1): """ Call this method to synchronize state with the real vardb @@ -141,9 +151,7 @@ class FakeVartree(vartree): # Remove any packages that have been uninstalled. for pkg in list(pkg_vardb): if pkg.cpv not in current_cpv_set: - pkg_vardb.cpv_remove(pkg) - pkg_cache.pop(pkg, None) - aux_get_history.discard(pkg.cpv) + self.cpv_discard(pkg) =20 # Validate counters and timestamps. slot_counters =3D {} @@ -162,9 +170,7 @@ class FakeVartree(vartree): =20 if counter !=3D pkg.counter or \ mtime !=3D pkg.mtime: - pkg_vardb.cpv_remove(pkg) - pkg_cache.pop(pkg, None) - aux_get_history.discard(pkg.cpv) + self.cpv_discard(pkg) pkg =3D None =20 if pkg is None: diff --git a/pym/_emerge/Scheduler.py b/pym/_emerge/Scheduler.py index fa4825c..e4b6926 100644 --- a/pym/_emerge/Scheduler.py +++ b/pym/_emerge/Scheduler.py @@ -317,10 +317,6 @@ class Scheduler(PollScheduler): Initialization structures used for dependency calculations involving currently installed packages. """ - # TODO: Replace the BlockerDB with a depgraph of installed packages - # that's updated incrementally with each upgrade/uninstall operation - # This will be useful for making quick and safe decisions with respect - # to aggressive parallelization discussed in bug #279623. self._set_graph_config(graph_config) self._blocker_db =3D {} for root in self.trees: @@ -329,6 +325,7 @@ class Scheduler(PollScheduler): pkg_cache=3Dself._pkg_cache) else: fake_vartree =3D graph_config.trees[root]['vartree'] + fake_vartree.sync() self._blocker_db[root] =3D BlockerDB(fake_vartree) =20 def _destroy_graph(self): @@ -643,27 +640,20 @@ class Scheduler(PollScheduler): =20 def _find_blockers(self, new_pkg): """ - Returns a callable which should be called only when - the vdb lock has been acquired. + Returns a callable. """ def get_blockers(): - return self._find_blockers_with_lock(new_pkg, acquire_lock=3D0) + return self._find_blockers_impl(new_pkg) return get_blockers =20 - def _find_blockers_with_lock(self, new_pkg, acquire_lock=3D0): + def _find_blockers_impl(self, new_pkg): if self._opts_ignore_blockers.intersection(self.myopts): return None =20 - # Call gc.collect() here to avoid heap overflow that - # triggers 'Cannot allocate memory' errors (reported - # with python-2.5). - gc.collect() - blocker_db =3D self._blocker_db[new_pkg.root] =20 blocker_dblinks =3D [] - for blocking_pkg in blocker_db.findInstalledBlockers( - new_pkg, acquire_lock=3Dacquire_lock): + for blocking_pkg in blocker_db.findInstalledBlockers(new_pkg): if new_pkg.slot_atom =3D=3D blocking_pkg.slot_atom: continue if new_pkg.cpv =3D=3D blocking_pkg.cpv: @@ -673,8 +663,6 @@ class Scheduler(PollScheduler): self.pkgsettings[blocking_pkg.root], treetype=3D"vartree", vartree=3Dself.trees[blocking_pkg.root]["vartree"])) =20 - gc.collect() - return blocker_dblinks =20 def _dblink_pkg(self, pkg_dblink): @@ -1527,6 +1515,8 @@ class Scheduler(PollScheduler): self._completed_tasks.add(pkg) self._unsatisfied_system_deps.discard(pkg) self._choose_pkg_return_early =3D False + blocker_db =3D self._blocker_db[pkg.root] + blocker_db.discardBlocker(pkg) =20 def _merge(self): =20