[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     51d563b2df46ecab2b9be0a7cfd98aacfdd16b2a
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Jan  9 22:23:28 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Jan  9 22:23:28 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=51d563b2

_fetch_digester_exit: fix NoneType AttributeError

---
 pym/portage/_emirrordist/FetchTask.py |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 65fd672..0853197 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -475,8 +475,8 @@ class FetchTask(CompositeTask):
 		else:
 			bad_digest = self._find_bad_digest(digester.digests)
 			if bad_digest is not None:
-				msg = "%s %s has bad %s digest: expected %s, got %s" % \
-					(self.distfile, self._current_mirror.name, bad_digest,
+				msg = "%s has bad %s digest: expected %s, got %s" % \
+					(self.distfile, bad_digest,
 					self.digests[bad_digest], digester.digests[bad_digest])
 				self.scheduler.output(msg + '\n', background=True,
 					log_path=self._log_path)

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     243080ecc5e867a4ab96d093b9cd3542c7337633
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 03:57:09 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 03:57:09 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=243080ec

FetchIterator: use RepoConfig.load_manifest()

---
 pym/portage/_emirrordist/FetchIterator.py |    8 +++++---
 1 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py
index cc5d4be..58044ac 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -4,7 +4,6 @@
 from portage import os
 from portage.dep import use_reduce
 from portage.exception import PortageException
-from portage.manifest import Manifest
 from .FetchTask import FetchTask
 
 class FetchIterator(object):
@@ -24,6 +23,7 @@ class FetchIterator(object):
 	def __iter__(self):
 
 		portdb = self._config.portdb
+		get_repo_for_location = portdb.repositories.get_repo_for_location
 		file_owners = self._config.file_owners
 		file_failures = self._config.file_failures
 		restrict_mirror_exemptions = self._config.restrict_mirror_exemptions
@@ -35,6 +35,7 @@ class FetchIterator(object):
 				# Reset state so the Manifest is pulled once
 				# for this cp / tree combination.
 				digests = None
+				repo_config = get_repo_for_location(tree)
 
 				for cpv in portdb.cp_list(cp, mytree=tree):
 
@@ -95,8 +96,9 @@ class FetchIterator(object):
 					# Parse Manifest for this cp if we haven't yet.
 					if digests is None:
 						try:
-							digests = Manifest(os.path.join(
-								tree, cp)).getTypeDigests("DIST")
+							digests = repo_config.load_manifest(
+								os.path.join(repo_config.location, cp)
+								).getTypeDigests("DIST")
 						except (EnvironmentError, PortageException) as e:
 							for filename in uri_map:
 								self._log_failure(

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     f14f173934f11415889c114af24685bb2b19de22
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 07:54:26 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 07:54:26 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=f14f1739

FetchIterator: support PORTAGE_CHECKSUM_FILTER

---
 pym/portage/_emirrordist/FetchIterator.py |   13 +++++++++++++
 1 files changed, 13 insertions(+), 0 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py
index 58044ac..16a0b04 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -2,6 +2,8 @@
 # Distributed under the terms of the GNU General Public License v2
 
 from portage import os
+from portage.checksum import (_apply_hash_filter,
+	_filter_unaccelarated_hashes, _hash_filter)
 from portage.dep import use_reduce
 from portage.exception import PortageException
 from .FetchTask import FetchTask
@@ -28,6 +30,11 @@ class FetchIterator(object):
 		file_failures = self._config.file_failures
 		restrict_mirror_exemptions = self._config.restrict_mirror_exemptions
 
+		hash_filter = _hash_filter(
+			portdb.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
+		if hash_filter.transparent:
+			hash_filter = None
+
 		for cp in self._iter_every_cp():
 
 			for tree in portdb.porttrees:
@@ -125,6 +132,12 @@ class FetchIterator(object):
 							continue
 						file_owners[filename] = cpv
 
+						file_digests = \
+							_filter_unaccelarated_hashes(file_digests)
+						if hash_filter is not None:
+							file_digests = _apply_hash_filter(
+								file_digests, hash_filter)
+
 						yield FetchTask(cpv=cpv,
 							background=True,
 							digests=file_digests,

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     ebb6a97eb926c030eb933e84041b7673335cab41
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 08:41:01 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 08:41:01 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=ebb6a97e

FetchTask: log_failure for distfiles digester

---
 pym/portage/_emirrordist/FetchTask.py |    7 +++++++
 1 files changed, 7 insertions(+), 0 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 0853197..61b567b 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -150,11 +150,18 @@ class FetchTask(CompositeTask):
 			return
 
 		if self._default_exit(digester) != os.EX_OK:
+			# IOError reading file in our main distfiles directory? This
+			# is a bad situation which normally does not occur, so
+			# skip this file and report it, in order to draw attention
+			# from the administrator.
 			msg = "%s distfiles digester failed unexpectedly" % \
 				(self.distfile,)
 			self.scheduler.output(msg + '\n', background=True,
 				log_path=self._log_path)
 			logging.error(msg)
+			self.config.log_failure("%s\t%s\t%s" %
+				(self.cpv, self.distfile, msg))
+			self.config.file_failures[self.distfile] = self.cpv
 			self.wait()
 			return
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     7d00427d74450a6ebf3e6bfd0a258392a0f65121
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 09:05:40 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 09:05:40 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=7d00427d

FetchTask: randomize thirdpartymirrors

---
 pym/portage/_emirrordist/FetchTask.py |    8 +++++++-
 1 files changed, 7 insertions(+), 1 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 61b567b..50e4242 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -4,6 +4,7 @@
 import collections
 import errno
 import logging
+import random
 import stat
 import subprocess
 import sys
@@ -206,7 +207,12 @@ class FetchTask(CompositeTask):
 		slash_index = uri.find("/", 9)
 		if slash_index != -1:
 			mirror_name = uri[9:slash_index].strip("/")
-			for mirror in mirrors_dict.get(mirror_name, []):
+			mirrors = mirrors_dict.get(mirror_name)
+			if not mirrors:
+				return
+			mirrors = list(mirrors)
+			while mirrors:
+				mirror = mirrors.pop(random.randint(0, len(mirrors) - 1))
 				yield mirror.rstrip("/") + "/" + uri[slash_index+1:]
 
 	def _try_next_mirror(self):

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     9fc0204ad78b998e589d2f232eceda5e018584d6
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 09:17:46 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 09:17:46 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=9fc0204a

FetchTask: avoid possible duplicate URI

---
 pym/portage/_emirrordist/FetchTask.py |    6 ++++--
 1 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 50e4242..107db56 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -255,8 +255,10 @@ class FetchTask(CompositeTask):
 					if uri not in self._tried_uris:
 						return uri
 
-			if self._primaryuri_stack:
-				return self._primaryuri_stack.pop()
+			while self._primaryuri_stack:
+				uri = self._primaryuri_stack.pop()
+				if uri not in self._tried_uris:
+					return uri
 
 		return None
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     799ba15b69b4aebc08a4f21870d22340ef910d14
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 09:40:38 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 09:40:38 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=799ba15b

FetchTask: use umask for copied files

---
 pym/portage/_emirrordist/FetchTask.py |   14 ++------------
 1 files changed, 2 insertions(+), 12 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 107db56..c9e0433 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -12,7 +12,6 @@ import sys
 import portage
 from portage import _encodings, _unicode_encode
 from portage import os
-from portage.exception import PortageException
 from portage.util._async.FileCopier import FileCopier
 from portage.util._async.FileDigester import FileDigester
 from portage.util._async.PipeLogger import PipeLogger
@@ -368,17 +367,8 @@ class FetchTask(CompositeTask):
 			logging.debug("copy '%s' from %s to distfiles" %
 				(self.distfile, current_mirror.name))
 
-			try:
-				portage.util.apply_stat_permissions(
-					copier.dest_path, self._current_stat)
-			except (OSError, PortageException) as e:
-				msg = ("%s %s apply_stat_permissions "
-					"failed unexpectedly: %s") % \
-					(self.distfile, current_mirror.name, e)
-				self.scheduler.output(msg + '\n', background=True,
-					log_path=self._log_path)
-				logging.error(msg)
-
+			# Apply the timestamp from the source file, but
+			# just rely on umask for permissions.
 			try:
 				if sys.hexversion >= 0x3030000:
 					os.utime(copier.dest_path,

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     48caf2e819725a6caaf26c453f88e0c8460a5b4c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jan 10 10:34:49 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan 10 10:34:49 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=48caf2e8

_emirrordist/main.py: remove extra Config import

---
 pym/portage/_emirrordist/main.py |    1 -
 1 files changed, 0 insertions(+), 1 deletions(-)

diff --git a/pym/portage/_emirrordist/main.py b/pym/portage/_emirrordist/main.py
index 6b7c22f..da9f3f3 100644
--- a/pym/portage/_emirrordist/main.py
+++ b/pym/portage/_emirrordist/main.py
@@ -11,7 +11,6 @@ from portage.util import normalize_path
 from portage.util._async.run_main_scheduler import run_main_scheduler
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.util._eventloop.global_event_loop import global_event_loop
-from portage._emirrordist.Config import Config
 from .Config import Config
 from .MirrorDistTask import MirrorDistTask
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     3ce6cfd1c41532afeed9d5f8bbbb0c5b652a9099
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Jun 20 01:23:21 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jun 20 01:23:21 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=3ce6cfd1

emirrordist: wget --no-check-certificate

Use --no-check-certificate since Manifest digests should provide
enough security, and certificates can be self-signed or whatnot.

---
 pym/portage/_emirrordist/FetchTask.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index c9e0433..46412e3 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -20,7 +20,9 @@ from _emerge.CompositeTask import CompositeTask
 
 default_hash_name = portage.const.MANIFEST2_REQUIRED_HASH
 
-default_fetchcommand = "wget -c -v -t 1 --passive-ftp --timeout=60 -O \"${DISTDIR}/${FILE}\" \"${URI}\""
+# Use --no-check-certificate since Manifest digests should provide
+# enough security, and certificates can be self-signed or whatnot.
+default_fetchcommand = "wget -c -v -t 1 --passive-ftp --no-check-certificate --timeout=60 -O \"${DISTDIR}/${FILE}\" \"${URI}\""
 
 class FetchTask(CompositeTask):
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     12f541490c80c1d6cdc1c71d87eef98d715fa089
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Fri Aug  2 23:27:17 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Aug  2 23:27:17 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=12f54149

emirrordist: portage.util._argparse

---
 pym/portage/_emirrordist/main.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/pym/portage/_emirrordist/main.py b/pym/portage/_emirrordist/main.py
index f4824f6..becd11b 100644
--- a/pym/portage/_emirrordist/main.py
+++ b/pym/portage/_emirrordist/main.py
@@ -2,12 +2,12 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
-import optparse
 import sys
 
 import portage
 from portage import os
 from portage.util import normalize_path, writemsg_level
+from portage.util._argparse import ArgumentParser
 from portage.util._async.run_main_scheduler import run_main_scheduler
 from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.util._eventloop.global_event_loop import global_event_loop
@@ -47,18 +47,21 @@ common_options = (
 	{
 		"longopt"  : "--jobs",
 		"shortopt" : "-j",
-		"help"     : "number of concurrent jobs to run"
+		"help"     : "number of concurrent jobs to run",
+		"type"     : int
 	},
 	{
 		"longopt"  : "--load-average",
 		"shortopt" : "-l",
 		"help"     : "load average limit for spawning of new concurrent jobs",
-		"metavar"  : "LOAD"
+		"metavar"  : "LOAD",
+		"type"     : float
 	},
 	{
 		"longopt"  : "--tries",
 		"help"     : "maximum number of tries per file, 0 means unlimited (default is 10)",
-		"default"  : 10
+		"default"  : 10,
+		"type"     : int
 	},
 	{
 		"longopt"  : "--repo",
@@ -87,7 +90,6 @@ common_options = (
 		"help"     : "manually override \"strict\" FEATURES setting",
 		"choices"  : ("y", "n"),
 		"metavar"  : "<y|n>",
-		"type"     : "choice"
 	},
 	{
 		"longopt"  : "--failure-log",
@@ -178,7 +180,8 @@ common_options = (
 			"recycle dir, measured in seconds (defaults to "
 			"the equivalent of 60 days)",
 		"default"  : 60 * seconds_per_day,
-		"metavar"  : "SECONDS"
+		"metavar"  : "SECONDS",
+		"type"     : int
 	},
 	{
 		"longopt"  : "--fetch-log-dir",
@@ -198,18 +201,17 @@ def parse_args(args):
 	description = "emirrordist - a fetch tool for mirroring " \
 		"of package distfiles"
 	usage = "emirrordist [options] <action>"
-	parser = optparse.OptionParser(description=description, usage=usage)
+	parser = ArgumentParser(description=description, usage=usage)
 
-	actions = optparse.OptionGroup(parser, 'Actions')
-	actions.add_option("--version",
+	actions = parser.add_argument_group('Actions')
+	actions.add_argument("--version",
 		action="store_true",
 		help="display portage version and exit")
-	actions.add_option("--mirror",
+	actions.add_argument("--mirror",
 		action="store_true",
 		help="mirror distfiles for the selected repository")
-	parser.add_option_group(actions)
 
-	common = optparse.OptionGroup(parser, 'Common options')
+	common = parser.add_argument_group('Common options')
 	for opt_info in common_options:
 		opt_pargs = [opt_info["longopt"]]
 		if opt_info.get("shortopt"):
@@ -218,10 +220,9 @@ def parse_args(args):
 		for k in ("action", "choices", "default", "metavar", "type"):
 			if k in opt_info:
 				opt_kwargs[k] = opt_info[k]
-		common.add_option(*opt_pargs, **opt_kwargs)
-	parser.add_option_group(common)
+		common.add_argument(*opt_pargs, **opt_kwargs)
 
-	options, args = parser.parse_args(args)
+	options, args = parser.parse_known_args(args)
 
 	return (parser, options, args)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     5626f1f7e4a586ee3d3776dee07501cb379adefb
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Aug 12 23:08:12 2013 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Aug 12 23:08:12 2013 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=5626f1f7

emirrordist: allow missing --repo if exactly one

---
 pym/portage/_emirrordist/main.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/pym/portage/_emirrordist/main.py b/pym/portage/_emirrordist/main.py
index becd11b..3c7b81a 100644
--- a/pym/portage/_emirrordist/main.py
+++ b/pym/portage/_emirrordist/main.py
@@ -244,9 +244,6 @@ def emirrordist_main(args):
 
 	config_root = options.config_root
 
-	if options.repo is None:
-		parser.error("--repo option is required")
-
 	if options.portdir is not None:
 		writemsg_level("emirrordist: warning: --portdir option is deprecated in favor of --repositories-configuration option\n",
 			level=logging.WARNING, noiselevel=-1)
@@ -275,6 +272,16 @@ def emirrordist_main(args):
 		settings = portage.config(config_root=config_root,
 			local_config=False, env=env)
 
+	if options.repo is None:
+		if len(settings.repositories.prepos) == 2:
+			for repo in settings.repositories:
+				if repo.name != "DEFAULT":
+					options.repo = repo.name
+					break
+
+		if options.repo is None:
+			parser.error("--repo option is required")
+
 	repo_path = settings.repositories.treemap.get(options.repo)
 	if repo_path is None:
 		parser.error("Unable to locate repository named '%s'" % (options.repo,))

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Arfrever Frehtes Taifersar Arahesis]

commit:     d2c0a27401ad6d2ccdb6e5af293c830c7abf0ad0
Author:     Arfrever Frehtes Taifersar Arahesis <Arfrever <AT> Apache <DOT> Org>
AuthorDate: Sun Feb  2 03:13:26 2014 +0000
Commit:     Arfrever Frehtes Taifersar Arahesis <arfrever <AT> apache <DOT> org>
CommitDate: Sun Feb  2 03:13:26 2014 +0000
URL:        http://git.overlays.gentoo.org/gitweb/?p=proj/portage.git;a=commit;h=d2c0a274

Bug #500030: emirrordist: Support directory in --whitelist-from argument.

---
 pym/portage/_emirrordist/main.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/pym/portage/_emirrordist/main.py b/pym/portage/_emirrordist/main.py
index f28aad7..139f24f 100644
--- a/pym/portage/_emirrordist/main.py
+++ b/pym/portage/_emirrordist/main.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2014 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
@@ -6,7 +6,7 @@ import sys
 
 import portage
 from portage import os
-from portage.util import normalize_path, writemsg_level
+from portage.util import normalize_path, writemsg_level, _recursive_file_list
 from portage.util._argparse import ArgumentParser
 from portage.util._async.run_main_scheduler import run_main_scheduler
 from portage.util._async.SchedulerInterface import SchedulerInterface
@@ -414,10 +414,17 @@ def emirrordist_main(args):
 		normalized_paths = []
 		for x in options.whitelist_from:
 			path = normalize_path(os.path.abspath(x))
-			normalized_paths.append(path)
-			if not (os.access(path, os.R_OK) and os.path.isfile(path)):
-				parser.error(
-					"--whitelist-from '%s' is not a readable file" % x)
+			if not os.access(path, os.R_OK):
+				parser.error("--whitelist-from '%s' is not readable" % x)
+			if os.path.isfile(path):
+				normalized_paths.append(path)
+			elif os.path.isdir(path):
+				for file in _recursive_file_list(path):
+					if not os.access(file, os.R_OK):
+						parser.error("--whitelist-from '%s' directory contains not readable file '%s'" % (x, file))
+					normalized_paths.append(file)
+			else:
+				parser.error("--whitelist-from '%s' is not a regular file or a directory" % x)
 		options.whitelist_from = normalized_paths
 
 	if options.strict_manifests is not None:

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     7c94014a32d173ae61919b762140ac1c32d3b522
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Mon Jan  4 03:05:24 2016 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Jan  7 16:49:56 2016 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c94014a

emirrordist: support bsddb3 module (bug 570798)

If shelve.open raises ImportError for bsddb, then use bsddb3 instead.

X-Gentoo-Bug: 570798
X-Gentoo-Bug-url: https://bugs.gentoo.org/show_bug.cgi?id=570798
Acked-by: Alexander Berntsen <bernalex <AT> gentoo.org>

 pym/portage/_emirrordist/Config.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/Config.py b/pym/portage/_emirrordist/Config.py
index f884a49..574a835 100644
--- a/pym/portage/_emirrordist/Config.py
+++ b/pym/portage/_emirrordist/Config.py
@@ -109,7 +109,15 @@ class Config(object):
 		if self.options.dry_run and not os.path.exists(db_file):
 			db = {}
 		else:
-			db = shelve.open(db_file, flag=open_flag)
+			try:
+				db = shelve.open(db_file, flag=open_flag)
+			except ImportError as e:
+				# ImportError has different attributes for python2 vs. python3
+				if (getattr(e, 'name', None) == 'bsddb' or
+					getattr(e, 'message', None) == 'No module named bsddb'):
+					from bsddb3 import dbshelve
+					db = dbshelve.open(db_file, flags=open_flag)
+
 			if sys.hexversion < 0x3000000:
 				db = ShelveUnicodeWrapper(db)
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     f70db92cba82535c8f65d385bd08a40207e24ec1
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 21 23:34:18 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 24 20:32:25 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=f70db92c

FetchIterator: add terminate method

Add a terminate method to FetchIterator so that it doesn't
delay termination of emirrordist via SIGINT. Otherwise it
it is possible for the __iter__ method to loop for a very
long time after SIGINT has been delivered. For example,
this could happen if there are many ebuilds with stale
cache and RESTRICT=mirror. This issue was discovered
during testing of changes to the MirrorDistTask.terminate
implementation.

 pym/portage/_emirrordist/FetchIterator.py  | 21 +++++++++++++++++++++
 pym/portage/_emirrordist/MirrorDistTask.py |  6 +++++-
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py
index 16a0b04c9..38419799d 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -1,6 +1,8 @@
 # Copyright 2013 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
+import threading
+
 from portage import os
 from portage.checksum import (_apply_hash_filter,
 	_filter_unaccelarated_hashes, _hash_filter)
@@ -13,6 +15,19 @@ class FetchIterator(object):
 	def __init__(self, config):
 		self._config = config
 		self._log_failure = config.log_failure
+		self._terminated = threading.Event()
+
+	def terminate(self):
+		"""
+		Schedules early termination of the __iter__ method, which is
+		useful because under some conditions it's possible for __iter__
+		to loop for a long time without yielding to the caller. For
+		example, it's useful when there are many ebuilds with stale
+		cache and RESTRICT=mirror.
+
+		This method is thread-safe (and safe for signal handlers).
+		"""
+		self._terminated.set()
 
 	def _iter_every_cp(self):
 		# List categories individually, in order to start yielding quicker,
@@ -37,6 +52,9 @@ class FetchIterator(object):
 
 		for cp in self._iter_every_cp():
 
+			if self._terminated.is_set():
+				return
+
 			for tree in portdb.porttrees:
 
 				# Reset state so the Manifest is pulled once
@@ -46,6 +64,9 @@ class FetchIterator(object):
 
 				for cpv in portdb.cp_list(cp, mytree=tree):
 
+					if self._terminated.is_set():
+						return
+
 					try:
 						restrict, = portdb.aux_get(cpv, ("RESTRICT",),
 							mytree=tree)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py
index d6b3decc0..a34f2c061 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -32,9 +32,11 @@ class MirrorDistTask(CompositeTask):
 		self._config = config
 		self._term_rlock = threading.RLock()
 		self._term_callback_handle = None
+		self._fetch_iterator = None
 
 	def _start(self):
-		fetch = TaskScheduler(iter(FetchIterator(self._config)),
+		self._fetch_iterator = FetchIterator(self._config)
+		fetch = TaskScheduler(iter(self._fetch_iterator),
 			max_jobs=self._config.options.jobs,
 			max_load=self._config.options.load_average,
 			event_loop=self._config.event_loop)
@@ -226,6 +228,8 @@ class MirrorDistTask(CompositeTask):
 					self._term_callback)
 
 	def _term_callback(self):
+		if self._fetch_iterator is not None:
+			self._fetch_iterator.terminate()
 		self.cancel()
 		self.wait()
 

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     61878e4fbdfef5f8512b34640089e954a14e6d12
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Tue Mar 21 07:06:12 2017 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Fri Mar 24 20:32:25 2017 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=61878e4f

MirrorDistTask: terminate via call_soon for asyncio compat

These changes are analogous to the PollScheduler changes in
the previous commit.

 pym/portage/_emirrordist/MirrorDistTask.py | 39 +++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 12 deletions(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py
index e23a11b3c..d6b3decc0 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -24,15 +24,16 @@ if sys.hexversion >= 0x3000000:
 
 class MirrorDistTask(CompositeTask):
 
-	__slots__ = ('_config', '_terminated', '_term_check_id')
+	__slots__ = ('_config', '_fetch_iterator', '_term_rlock',
+		'_term_callback_handle')
 
 	def __init__(self, config):
 		CompositeTask.__init__(self, scheduler=config.event_loop)
 		self._config = config
-		self._terminated = threading.Event()
+		self._term_rlock = threading.RLock()
+		self._term_callback_handle = None
 
 	def _start(self):
-		self._term_check_id = self.scheduler.idle_add(self._termination_check)
 		fetch = TaskScheduler(iter(FetchIterator(self._config)),
 			max_jobs=self._config.options.jobs,
 			max_load=self._config.options.load_average,
@@ -203,17 +204,31 @@ class MirrorDistTask(CompositeTask):
 		logging.info("added %i files" % added_file_count)
 		logging.info("added %i bytes total" % added_byte_count)
 
+	def _cleanup(self):
+		"""
+		Cleanup any callbacks that have been registered with the global
+		event loop.
+		"""
+		# The self._term_callback_handle attribute requires locking
+		# since it's modified by the thread safe terminate method.
+		with self._term_rlock:
+			if self._term_callback_handle not in (None, False):
+				self._term_callback_handle.cancel()
+			# This prevents the terminate method from scheduling
+			# any more callbacks (since _cleanup must eliminate all
+			# callbacks in order to ensure complete cleanup).
+			self._term_callback_handle = False
+
 	def terminate(self):
-		self._terminated.set()
+		with self._term_rlock:
+			if self._term_callback_handle is None:
+				self._term_callback_handle = self.scheduler.call_soon_threadsafe(
+					self._term_callback)
 
-	def _termination_check(self):
-		if self._terminated.is_set():
-			self.cancel()
-			self.wait()
-		return True
+	def _term_callback(self):
+		self.cancel()
+		self.wait()
 
 	def _wait(self):
 		CompositeTask._wait(self)
-		if self._term_check_id is not None:
-			self.scheduler.source_remove(self._term_check_id)
-			self._term_check_id = None
+		self._cleanup()

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     a7c8ddd9968741fd51ba21500a9f6b9ef27bef15
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 25 07:28:44 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Wed Apr 25 07:28:44 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=a7c8ddd9

FetchTask._fetch_uri: use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion in
emirrordist --dry-run mode.

Bug: https://bugs.gentoo.org/591760

 pym/portage/_emirrordist/FetchTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/FetchTask.py b/pym/portage/_emirrordist/FetchTask.py
index 47908cb6b..1440b697c 100644
--- a/pym/portage/_emirrordist/FetchTask.py
+++ b/pym/portage/_emirrordist/FetchTask.py
@@ -404,7 +404,7 @@ class FetchTask(CompositeTask):
 				(self.distfile, uri))
 			self._success()
 			self.returncode = os.EX_OK
-			self.wait()
+			self._async_wait()
 			return
 
 		if self.config.options.temp_dir:

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     391daef6fce981acd5a01e41a0f7238044c48877
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Thu Apr 26 08:29:04 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Thu Apr 26 08:43:53 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=391daef6

MirrorDistTask._term_callback(): use _async_wait() (bug 591760)

Use _async_wait() to avoid event loop recursion, but don't call
it prematurely, since since that could trigger event loop recursion
if the current (cancelled) task's exit callback does not set the
returncode first.

Bug: https://bugs.gentoo.org/591760

 pym/portage/_emirrordist/MirrorDistTask.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py
index a34f2c061..48d0f7cf2 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -231,7 +231,15 @@ class MirrorDistTask(CompositeTask):
 		if self._fetch_iterator is not None:
 			self._fetch_iterator.terminate()
 		self.cancel()
-		self.wait()
+		if self.returncode is None:
+			# In this case, the exit callback for self._current_task will
+			# trigger notification of exit listeners. Don't call _async_wait()
+			# yet, since that could trigger event loop recursion if the
+			# current (cancelled) task's exit callback does not set the
+			# returncode first.
+			pass
+		else:
+			self._async_wait()
 
 	def _wait(self):
 		CompositeTask._wait(self)

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     7c652d1b967f9c4c6a7fbd9fc5e46a0e57438a16
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Wed Apr 25 06:42:10 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 13:56:42 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=7c652d1b

FetchIterator: fix event loop recursion (bug 654038)

Since construction of FetchTask instances requires results from
aux_get calls that would trigger event loop recursion when executed
synchronously, add an _async_fetch_tasks function to construct
FetchTask instances asynchronously and return a Future. Use an
_EbuildFetchTasks class to wait for the FetchTask instances to
become available, and then execute them.

Bug: https://bugs.gentoo.org/654038

 pym/portage/_emirrordist/FetchIterator.py | 324 ++++++++++++++++++++----------
 1 file changed, 218 insertions(+), 106 deletions(-)

diff --git a/pym/portage/_emirrordist/FetchIterator.py b/pym/portage/_emirrordist/FetchIterator.py
index 38419799d..366453c12 100644
--- a/pym/portage/_emirrordist/FetchIterator.py
+++ b/pym/portage/_emirrordist/FetchIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import threading
@@ -7,14 +7,18 @@ from portage import os
 from portage.checksum import (_apply_hash_filter,
 	_filter_unaccelarated_hashes, _hash_filter)
 from portage.dep import use_reduce
-from portage.exception import PortageException
+from portage.exception import PortageException, PortageKeyError
+from portage.util._async.AsyncTaskFuture import AsyncTaskFuture
+from portage.util._async.TaskScheduler import TaskScheduler
+from portage.util.futures.iter_completed import iter_gather
 from .FetchTask import FetchTask
+from _emerge.CompositeTask import CompositeTask
+
 
 class FetchIterator(object):
 
 	def __init__(self, config):
 		self._config = config
-		self._log_failure = config.log_failure
 		self._terminated = threading.Event()
 
 	def terminate(self):
@@ -41,9 +45,6 @@ class FetchIterator(object):
 
 		portdb = self._config.portdb
 		get_repo_for_location = portdb.repositories.get_repo_for_location
-		file_owners = self._config.file_owners
-		file_failures = self._config.file_failures
-		restrict_mirror_exemptions = self._config.restrict_mirror_exemptions
 
 		hash_filter = _hash_filter(
 			portdb.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
@@ -59,110 +60,221 @@ class FetchIterator(object):
 
 				# Reset state so the Manifest is pulled once
 				# for this cp / tree combination.
-				digests = None
 				repo_config = get_repo_for_location(tree)
+				digests_future = portdb._event_loop.create_future()
 
 				for cpv in portdb.cp_list(cp, mytree=tree):
 
 					if self._terminated.is_set():
 						return
 
-					try:
-						restrict, = portdb.aux_get(cpv, ("RESTRICT",),
-							mytree=tree)
-					except (KeyError, PortageException) as e:
-						self._log_failure("%s\t\taux_get exception %s" %
-							(cpv, e))
-						continue
-
-					# Here we use matchnone=True to ignore conditional parts
-					# of RESTRICT since they don't apply unconditionally.
-					# Assume such conditionals only apply on the client side.
-					try:
-						restrict = frozenset(use_reduce(restrict,
-							flat=True, matchnone=True))
-					except PortageException as e:
-						self._log_failure("%s\t\tuse_reduce exception %s" %
-							(cpv, e))
-						continue
-
-					if "fetch" in restrict:
-						continue
-
-					try:
-						uri_map = portdb.getFetchMap(cpv)
-					except PortageException as e:
-						self._log_failure("%s\t\tgetFetchMap exception %s" %
-							(cpv, e))
-						continue
-
-					if not uri_map:
-						continue
-
-					if "mirror" in restrict:
-						skip = False
-						if restrict_mirror_exemptions is not None:
-							new_uri_map = {}
-							for filename, uri_tuple in uri_map.items():
-								for uri in uri_tuple:
-									if uri[:9] == "mirror://":
-										i = uri.find("/", 9)
-										if i != -1 and uri[9:i].strip("/") in \
-											restrict_mirror_exemptions:
-											new_uri_map[filename] = uri_tuple
-											break
-							if new_uri_map:
-								uri_map = new_uri_map
-							else:
-								skip = True
-						else:
-							skip = True
-
-						if skip:
-							continue
-
-					# Parse Manifest for this cp if we haven't yet.
-					if digests is None:
-						try:
-							digests = repo_config.load_manifest(
-								os.path.join(repo_config.location, cp)
-								).getTypeDigests("DIST")
-						except (EnvironmentError, PortageException) as e:
-							for filename in uri_map:
-								self._log_failure(
-									"%s\t%s\tManifest exception %s" %
-									(cpv, filename, e))
-								file_failures[filename] = cpv
-							continue
-
-					if not digests:
-						for filename in uri_map:
-							self._log_failure("%s\t%s\tdigest entry missing" %
-								(cpv, filename))
-							file_failures[filename] = cpv
-						continue
-
-					for filename, uri_tuple in uri_map.items():
-						file_digests = digests.get(filename)
-						if file_digests is None:
-							self._log_failure("%s\t%s\tdigest entry missing" %
-								(cpv, filename))
-							file_failures[filename] = cpv
-							continue
-						if filename in file_owners:
-							continue
-						file_owners[filename] = cpv
-
-						file_digests = \
-							_filter_unaccelarated_hashes(file_digests)
-						if hash_filter is not None:
-							file_digests = _apply_hash_filter(
-								file_digests, hash_filter)
-
-						yield FetchTask(cpv=cpv,
-							background=True,
-							digests=file_digests,
-							distfile=filename,
-							restrict=restrict,
-							uri_tuple=uri_tuple,
-							config=self._config)
+					yield _EbuildFetchTasks(
+						fetch_tasks_future=_async_fetch_tasks(
+							self._config,
+							hash_filter,
+							repo_config,
+							digests_future,
+							cpv,
+							portdb._event_loop)
+					)
+
+
+class _EbuildFetchTasks(CompositeTask):
+	"""
+	Executes FetchTask instances (which are asynchronously constructed)
+	for each of the files referenced by an ebuild.
+	"""
+	__slots__ = ('fetch_tasks_future',)
+	def _start(self):
+		self._start_task(AsyncTaskFuture(future=self.fetch_tasks_future),
+			self._start_fetch_tasks)
+
+	def _start_fetch_tasks(self, task):
+		if self._default_exit(task) != os.EX_OK:
+			self._async_wait()
+			return
+
+		self._start_task(
+			TaskScheduler(
+				iter(self.fetch_tasks_future.result()),
+				max_jobs=1,
+				event_loop=self.scheduler),
+			self._default_final_exit)
+
+
+def _async_fetch_tasks(config, hash_filter, repo_config, digests_future, cpv,
+	loop):
+	"""
+	Asynchronously construct FetchTask instances for each of the files
+	referenced by an ebuild.
+
+	@param config: emirrordist config
+	@type config: portage._emirrordist.Config.Config
+	@param hash_filter: PORTAGE_CHECKSUM_FILTER settings
+	@type hash_filter: portage.checksum._hash_filter
+	@param repo_config: repository configuration
+	@type repo_config: RepoConfig
+	@param digests_future: future that contains cached distfiles digests
+		for the current cp if available
+	@type digests_future: asyncio.Future
+	@param cpv: current ebuild cpv
+	@type cpv: portage.versions._pkg_str
+	@param loop: event loop
+	@type loop: EventLoop
+	@return: A future that results in a list containing FetchTask
+		instances for each of the files referenced by an ebuild.
+	@rtype: asyncio.Future (or compatible)
+	"""
+	loop = getattr(loop, '_asyncio_wrapper', loop)
+	result = loop.create_future()
+	fetch_tasks = []
+
+	def aux_get_done(gather_result):
+		# All exceptions must be consumed from gather_result before this
+		# function returns, in order to avoid triggering the event loop's
+		# exception handler.
+		if not gather_result.cancelled():
+			list(future.exception() for future in gather_result.result()
+				if not future.cancelled())
+
+		if result.cancelled():
+			return
+
+		aux_get_result, fetch_map_result = gather_result.result()
+		try:
+			restrict, = aux_get_result.result()
+		except (PortageKeyError, PortageException) as e:
+			config.log_failure("%s\t\taux_get exception %s" %
+				(cpv, e))
+			result.set_result(fetch_tasks)
+			return
+
+		# Here we use matchnone=True to ignore conditional parts
+		# of RESTRICT since they don't apply unconditionally.
+		# Assume such conditionals only apply on the client side.
+		try:
+			restrict = frozenset(use_reduce(restrict,
+				flat=True, matchnone=True))
+		except PortageException as e:
+			config.log_failure("%s\t\tuse_reduce exception %s" %
+				(cpv, e))
+			result.set_result(fetch_tasks)
+			return
+
+		if "fetch" in restrict:
+			result.set_result(fetch_tasks)
+			return
+
+		try:
+			uri_map = fetch_map_result.result()
+		except PortageException as e:
+			config.log_failure("%s\t\tgetFetchMap exception %s" %
+				(cpv, e))
+			result.set_result(fetch_tasks)
+			return
+
+		if not uri_map:
+			result.set_result(fetch_tasks)
+			return
+
+		if "mirror" in restrict:
+			skip = False
+			if config.restrict_mirror_exemptions is not None:
+				new_uri_map = {}
+				for filename, uri_tuple in uri_map.items():
+					for uri in uri_tuple:
+						if uri[:9] == "mirror://":
+							i = uri.find("/", 9)
+							if i != -1 and uri[9:i].strip("/") in \
+								config.restrict_mirror_exemptions:
+								new_uri_map[filename] = uri_tuple
+								break
+				if new_uri_map:
+					uri_map = new_uri_map
+				else:
+					skip = True
+			else:
+				skip = True
+
+			if skip:
+				result.set_result(fetch_tasks)
+				return
+
+		# Parse Manifest for this cp if we haven't yet.
+		try:
+			if digests_future.done():
+				# If there's an exception then raise it.
+				digests = digests_future.result()
+			else:
+				digests = repo_config.load_manifest(
+					os.path.join(repo_config.location, cpv.cp)).\
+					getTypeDigests("DIST")
+		except (EnvironmentError, PortageException) as e:
+			digests_future.done() or digests_future.set_exception(e)
+			for filename in uri_map:
+				config.log_failure(
+					"%s\t%s\tManifest exception %s" %
+					(cpv, filename, e))
+				config.file_failures[filename] = cpv
+			result.set_result(fetch_tasks)
+			return
+		else:
+			digests_future.done() or digests_future.set_result(digests)
+
+		if not digests:
+			for filename in uri_map:
+				config.log_failure("%s\t%s\tdigest entry missing" %
+					(cpv, filename))
+				config.file_failures[filename] = cpv
+			result.set_result(fetch_tasks)
+			return
+
+		for filename, uri_tuple in uri_map.items():
+			file_digests = digests.get(filename)
+			if file_digests is None:
+				config.log_failure("%s\t%s\tdigest entry missing" %
+					(cpv, filename))
+				config.file_failures[filename] = cpv
+				continue
+			if filename in config.file_owners:
+				continue
+			config.file_owners[filename] = cpv
+
+			file_digests = \
+				_filter_unaccelarated_hashes(file_digests)
+			if hash_filter is not None:
+				file_digests = _apply_hash_filter(
+					file_digests, hash_filter)
+
+			fetch_tasks.append(FetchTask(
+				cpv=cpv,
+				background=True,
+				digests=file_digests,
+				distfile=filename,
+				restrict=restrict,
+				uri_tuple=uri_tuple,
+				config=config))
+
+		result.set_result(fetch_tasks)
+
+	def future_generator():
+		yield config.portdb.async_aux_get(cpv, ("RESTRICT",),
+			myrepo=repo_config.name, loop=loop)
+		yield config.portdb.async_fetch_map(cpv,
+			mytree=repo_config.location, loop=loop)
+
+	# Use iter_gather(max_jobs=1) to limit the number of processes per
+	# _EbuildFetchTask instance, and also to avoid spawning two bash
+	# processes for the same cpv simultaneously (the second one can
+	# use metadata cached by the first one).
+	gather_result = iter_gather(
+		future_generator(),
+		max_jobs=1,
+		loop=loop,
+	)
+	gather_result.add_done_callback(aux_get_done)
+	result.add_done_callback(lambda result:
+		gather_result.cancel() if result.cancelled() and
+		not gather_result.done() else None)
+
+	return result

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     27801746e9aa7e9ffc02408b2b005aa106c0d8a7
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 21:51:01 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 21:55:46 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=27801746

MirrorDistTask: fix deprecated _wait usage (bug 653856)

Override AsynchronousTask._async_wait() for cleanup, since
AsynchronousTask._wait() is deprecated.

Bug: https://bugs.gentoo.org/653856

 pym/portage/_emirrordist/MirrorDistTask.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py
index 48d0f7cf2..a0912d673 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2014 Gentoo Foundation
+# Copyright 2013-2018 Gentoo Foundation
 # Distributed under the terms of the GNU General Public License v2
 
 import errno
@@ -46,7 +46,7 @@ class MirrorDistTask(CompositeTask):
 
 		self._assert_current(fetch)
 		if self._was_cancelled():
-			self.wait()
+			self._async_wait()
 			return
 
 		if self._config.options.delete:
@@ -63,7 +63,7 @@ class MirrorDistTask(CompositeTask):
 
 		self._assert_current(deletion)
 		if self._was_cancelled():
-			self.wait()
+			self._async_wait()
 			return
 
 		self._post_deletion()
@@ -80,7 +80,7 @@ class MirrorDistTask(CompositeTask):
 
 		self.returncode = os.EX_OK
 		self._current_task = None
-		self.wait()
+		self._async_wait()
 
 	def _update_recycle_db(self):
 
@@ -242,5 +242,15 @@ class MirrorDistTask(CompositeTask):
 			self._async_wait()
 
 	def _wait(self):
+		"""
+		Deprecated. Use _async_wait() instead.
+		"""
 		CompositeTask._wait(self)
 		self._cleanup()
+
+	def _async_wait(self):
+		"""
+		Override _async_wait to call self._cleanup().
+		"""
+		self._cleanup()
+		super(CompositeTask, self)._async_wait()

[gentoo-commits] proj/portage:master commit in: pym/portage/_emirrordist/

[Zac Medico]

commit:     1b0a408fcd556b7d9912856b852a331260d22022
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Apr 28 22:48:51 2018 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Sat Apr 28 22:48:51 2018 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=1b0a408f

MirrorDistTask: fix super in _async_wait

Fixes: 27801746e9aa ("MirrorDistTask: fix deprecated _wait usage (bug 653856)")

 pym/portage/_emirrordist/MirrorDistTask.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pym/portage/_emirrordist/MirrorDistTask.py b/pym/portage/_emirrordist/MirrorDistTask.py
index a0912d673..b8cde0af4 100644
--- a/pym/portage/_emirrordist/MirrorDistTask.py
+++ b/pym/portage/_emirrordist/MirrorDistTask.py
@@ -253,4 +253,4 @@ class MirrorDistTask(CompositeTask):
 		Override _async_wait to call self._cleanup().
 		"""
 		self._cleanup()
-		super(CompositeTask, self)._async_wait()
+		super(MirrorDistTask, self)._async_wait()