[gentoo-commits] proj/portage:master commit in: man/, pym/_emerge/

["Zac Medico" <zmedico@gentoo.org>]

commit:     55c8c8bc7a781e3f71ce92922eea64ad4cafce3c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Nov  5 05:07:26 2014 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sun Dec  7 23:10:48 2014 +0000
URL:        http://sources.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=55c8c8bc

Add emerge --search-index option.

The new emerge --search-index option, which is enabled by default,
causes pkg_desc_index to be used for search optimization. The search
index needs to be regenerated by egencache after changes are made to
a repository (see the --update-pkg-desc-index action).

For users that would like to modify ebuilds in a repository without
running egencache afterwards, emerge --search-index=n can be used to
get non-indexed search. Alternatively, the user could simply remove
the stale index file, in order to disable the search index for a
particular repository.

In order to conserve memory, indices are read as streams, and
MultiIterGroupBy is used to group results from IndexedPortdb and
IndexedVardb. Stream-oriented search also makes it possible to
display search results incrementally (for bug #412471).

X-Gentoo-Bug: 525718
X-Gentoo-Bug-URL: https://bugs.gentoo.org/show_bug.cgi?id=525718

---
 man/emerge.1            |  8 ++++++++
 pym/_emerge/actions.py  |  3 ++-
 pym/_emerge/depgraph.py |  2 +-
 pym/_emerge/main.py     |  5 +++++
 pym/_emerge/search.py   | 31 ++++++++++++++++++++++++++-----
 5 files changed, 42 insertions(+), 7 deletions(-)

diff --git a/man/emerge.1 b/man/emerge.1
index be46a7d..b9ce79c 100644
--- a/man/emerge.1
+++ b/man/emerge.1
@@ -804,6 +804,14 @@ If ebuilds using EAPIs which \fIdo not\fR support \fBHDEPEND\fR are built in
 the same \fBemerge\fR run as those using EAPIs which \fIdo\fR support
 \fBHDEPEND\fR, this option affects only the former.
 .TP
+.BR "\-\-search\-index < y | n >"
+Enable or disable indexed search for search actions. This option is
+enabled by default. The search index needs to be regenerated by
+\fBegencache\fR(1) after changes are made to a repository (see the
+\fB\-\-update\-pkg\-desc\-index\fR action). This setting can be added
+to \fBEMERGE_DEFAULT_OPTS\fR (see \fBmake.conf\fR(5)) and later
+overridden via the command line.
+.TP
 .BR "\-\-select [ y | n ] (\-w short option)"
 Add specified packages to the world set (inverse of
 \fB\-\-oneshot\fR). This is useful if you want to

diff --git a/pym/_emerge/actions.py b/pym/_emerge/actions.py
index 434fc47..c7246a9 100644
--- a/pym/_emerge/actions.py
+++ b/pym/_emerge/actions.py
@@ -1901,7 +1901,8 @@ def action_search(root_config, myopts, myfiles, spinner):
 		searchinstance = search(root_config,
 			spinner, "--searchdesc" in myopts,
 			"--quiet" not in myopts, "--usepkg" in myopts,
-			"--usepkgonly" in myopts)
+			"--usepkgonly" in myopts,
+			search_index = myopts.get("--search-index", "y") != "n")
 		for mysearch in myfiles:
 			try:
 				searchinstance.execute(mysearch)

diff --git a/pym/_emerge/depgraph.py b/pym/_emerge/depgraph.py
index 3455b6b..df1e266 100644
--- a/pym/_emerge/depgraph.py
+++ b/pym/_emerge/depgraph.py
@@ -8730,7 +8730,7 @@ def ambiguous_package_name(arg, atoms, root_config, spinner, myopts):
 
 	s = search(root_config, spinner, "--searchdesc" in myopts,
 		"--quiet" not in myopts, "--usepkg" in myopts,
-		"--usepkgonly" in myopts)
+		"--usepkgonly" in myopts, search_index = False)
 	null_cp = portage.dep_getkey(insert_category_into_atom(
 		arg, "null"))
 	cat, atom_pn = portage.catsplit(null_cp)

diff --git a/pym/_emerge/main.py b/pym/_emerge/main.py
index 1506207..66d4891 100644
--- a/pym/_emerge/main.py
+++ b/pym/_emerge/main.py
@@ -617,6 +617,11 @@ def parse_opts(tmpcmdline, silent=False):
 			"choices" :("True", "rdeps")
 		},
 
+		"--search-index": {
+			"help": "Enable or disable indexed search (enabled by default)",
+			"choices": y_or_n
+		},
+
 		"--select": {
 			"shortopt" : "-w",
 			"help"    : "add specified packages to the world set " + \

diff --git a/pym/_emerge/search.py b/pym/_emerge/search.py
index e51d206..4b1833a 100644
--- a/pym/_emerge/search.py
+++ b/pym/_emerge/search.py
@@ -5,9 +5,12 @@ import re
 import portage
 from portage import os
 from portage.dbapi.porttree import _parse_uri_map
+from portage.dbapi.IndexedPortdb import IndexedPortdb
+from portage.dbapi.IndexedVardb import IndexedVardb
 from portage.localization import localized_size
 from portage.output import  bold, bold as white, darkgreen, green, red
 from portage.util import writemsg_stdout
+from portage.util.iterators.MultiIterGroupBy import MultiIterGroupBy
 
 from _emerge.Package import Package
 
@@ -23,7 +26,7 @@ class search(object):
 	# public interface
 	#
 	def __init__(self, root_config, spinner, searchdesc,
-		verbose, usepkg, usepkgonly):
+		verbose, usepkg, usepkgonly, search_index=True):
 		"""Searches the available and installed packages for the supplied search key.
 		The list of available and installed packages is created at object instantiation.
 		This makes successive searches faster."""
@@ -45,6 +48,10 @@ class search(object):
 		bindb = root_config.trees["bintree"].dbapi
 		vardb = root_config.trees["vartree"].dbapi
 
+		if search_index:
+			portdb = IndexedPortdb(portdb)
+			vardb = IndexedVardb(vardb)
+
 		if not usepkgonly and portdb._have_root_eclass_dir:
 			self._dbs.append(portdb)
 
@@ -60,10 +67,16 @@ class search(object):
 			self.spinner.update()
 
 	def _cp_all(self):
-		cp_all = set()
+		iterators = []
 		for db in self._dbs:
-			cp_all.update(db.cp_all())
-		return list(sorted(cp_all))
+			i = db.cp_all()
+			try:
+				i = iter(i)
+			except TypeError:
+				pass
+			iterators.append(i)
+		for group in MultiIterGroupBy(iterators):
+			yield group[0]
 
 	def _aux_get(self, *args, **kwargs):
 		for db in self._dbs:
@@ -154,9 +167,17 @@ class search(object):
 						result = cpv
 				else:
 					db_keys = list(db._aux_cache_keys)
+					matches = db.match(atom)
+					try:
+						db.match_unordered
+					except AttributeError:
+						pass
+					else:
+						db._cpv_sort_ascending(matches)
+
 					# break out of this loop with highest visible
 					# match, checked in descending order
-					for cpv in reversed(db.match(atom)):
+					for cpv in reversed(matches):
 						if portage.cpv_getkey(cpv) != cp:
 							continue
 						metadata = zip(db_keys,