[gentoo-commits] proj/portage:master commit in: lib/portage/tests/ebuild/, lib/portage/package/ebuild/

["Zac Medico" <zmedico@gentoo.org>]

commit:     b9ef191c74982b0e8d837aa7dd256dc3c52f7d2c
Author:     Zac Medico <zmedico <AT> gentoo <DOT> org>
AuthorDate: Sat Feb 20 23:11:46 2021 +0000
Commit:     Zac Medico <zmedico <AT> gentoo <DOT> org>
CommitDate: Mon Feb 22 11:48:41 2021 +0000
URL:        https://gitweb.gentoo.org/proj/portage.git/commit/?id=b9ef191c

MirrorLayoutConfig: content digest support (bug 756778)

In order to support mirror layouts that use content
digests, extend MirrorLayoutConfig validate_structure and
get_best_supported_layout methods to support an optional
filename parameter of type DistfileName which includes a digests
attribute. Use the new parameter to account for availablility
of specific distfile content digests when validating and selecting
mirror layouts which require those digests.

The DistfileName type represents a distfile name and associated
content digests, used by MirrorLayoutConfig and related layout
implementations.

The path of a distfile within a layout must be dependent on
nothing more than the distfile name and its associated content
digests. For filename-hash layout, path is dependent on distfile
name alone, and the get_filenames implementation yields strings
corresponding to distfile names. For content-hash layout, path is
dependent on content digest alone, and the get_filenames
implementation yields DistfileName instances whose names are equal
to content digest values. The content-hash layout simply lacks
the filename-hash layout's innate ability to translate a distfile
path to a distfile name, and instead caries an innate ability
to translate a distfile path to a content digest.

In order to prepare for a migration from filename-hash to
content-hash layout, all consumers of the layout get_filenames
method need to be updated to work with content digests as a
substitute for distfile names. For example, in order to prepare
emirrordist for content-hash, a key-value store needs to be
added as a means to associate distfile names with content
digest values yielded by the content-hash get_filenames
implementation.

Bug: https://bugs.gentoo.org/756778
Signed-off-by: Zac Medico <zmedico <AT> gentoo.org>

 lib/portage/package/ebuild/fetch.py    | 98 ++++++++++++++++++++++++++++++----
 lib/portage/tests/ebuild/test_fetch.py | 33 +++++++++---
 2 files changed, 114 insertions(+), 17 deletions(-)

diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index e0fecaf23..af9edd91e 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -1,4 +1,4 @@
-# Copyright 2010-2020 Gentoo Authors
+# Copyright 2010-2021 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = ['fetch']
@@ -344,6 +344,57 @@ _size_suffix_map = {
 }
 
 
+class DistfileName(str):
+	"""
+	The DistfileName type represents a distfile name and associated
+	content digests, used by MirrorLayoutConfig and related layout
+	implementations.
+
+	The path of a distfile within a layout must be dependent on
+	nothing more than the distfile name and its associated content
+	digests. For filename-hash layout, path is dependent on distfile
+	name alone, and the get_filenames implementation yields strings
+	corresponding to distfile names. For content-hash layout, path is
+	dependent on content digest alone, and the get_filenames
+	implementation yields DistfileName instances whose names are equal
+	to content digest values. The content-hash layout simply lacks
+	the filename-hash layout's innate ability to translate a distfile
+	path to a distfile name, and instead caries an innate ability
+	to translate a distfile path to a content digest.
+
+	In order to prepare for a migration from filename-hash to
+	content-hash layout, all consumers of the layout get_filenames
+	method need to be updated to work with content digests as a
+	substitute for distfile names. For example, in order to prepare
+	emirrordist for content-hash, a key-value store needs to be
+	added as a means to associate distfile names with content
+	digest values yielded by the content-hash get_filenames
+	implementation.
+	"""
+	def __new__(cls, s, digests=None):
+		return str.__new__(cls, s)
+
+	def __init__(self, s, digests=None):
+		super().__init__()
+		self.digests = {} if digests is None else digests
+
+	def digests_equal(self, other):
+		"""
+		Test if digests compare equal to those of another instance.
+		"""
+		if not isinstance(other, DistfileName):
+			return False
+		matches = []
+		for algo, digest in self.digests.items():
+			other_digest = other.digests.get(algo)
+			if other_digest is not None:
+				if other_digest == digest:
+					matches.append(algo)
+				else:
+					return False
+		return bool(matches)
+
+
 class FlatLayout:
 	def get_path(self, filename):
 		return filename
@@ -439,19 +490,36 @@ class MirrorLayoutConfig:
 		self.structure = data
 
 	@staticmethod
-	def validate_structure(val):
+	def validate_structure(val, filename=None):
+		"""
+		If the filename argument is given, then supported hash
+		algorithms are constrained by digests available in the filename
+		digests attribute.
+
+		@param val: layout.conf entry args
+		@param filename: filename with digests attribute
+		@return: True if args are valid for available digest algorithms,
+			and False otherwise
+		"""
 		if val[0] == 'flat':
 			return FlatLayout.verify_args(val)
-		if val[0] == 'filename-hash':
+		elif val[0] == 'filename-hash':
 			return FilenameHashLayout.verify_args(val)
 		return False
 
-	def get_best_supported_layout(self):
+	def get_best_supported_layout(self, filename=None):
+		"""
+		If the filename argument is given, then acceptable hash
+		algorithms are constrained by digests available in the filename
+		digests attribute.
+
+		@param filename: filename with digests attribute
+		"""
 		for val in self.structure:
-			if self.validate_structure(val):
+			if self.validate_structure(val, filename=filename):
 				if val[0] == 'flat':
 					return FlatLayout(*val[1:])
-				if val[0] == 'filename-hash':
+				elif val[0] == 'filename-hash':
 					return FilenameHashLayout(*val[1:])
 		# fallback
 		return FlatLayout()
@@ -515,7 +583,7 @@ def get_mirror_url(mirror_url, filename, mysettings, cache_path=None):
 
 	# For some protocols, urlquote is required for correct behavior,
 	# and it must not be used for other protocols like rsync and sftp.
-	path = mirror_conf.get_best_supported_layout().get_path(filename)
+	path = mirror_conf.get_best_supported_layout(filename=filename).get_path(filename)
 	if urlparse(mirror_url).scheme in ('ftp', 'http', 'https'):
 		path = urlquote(path)
 	return mirror_url + "/distfiles/" + path
@@ -722,15 +790,23 @@ def fetch(myuris, mysettings, listonly=0, fetchonly=0,
 	if hasattr(myuris, 'items'):
 		for myfile, uri_set in myuris.items():
 			for myuri in uri_set:
-				file_uri_tuples.append((myfile, myuri))
+				file_uri_tuples.append(
+					(DistfileName(myfile, digests=mydigests.get(myfile)), myuri)
+				)
 			if not uri_set:
-				file_uri_tuples.append((myfile, None))
+				file_uri_tuples.append(
+					(DistfileName(myfile, digests=mydigests.get(myfile)), None)
+				)
 	else:
 		for myuri in myuris:
 			if urlparse(myuri).scheme:
-				file_uri_tuples.append((os.path.basename(myuri), myuri))
+				file_uri_tuples.append(
+					(DistfileName(myfile, digests=mydigests.get(myfile)), myuri)
+				)
 			else:
-				file_uri_tuples.append((os.path.basename(myuri), None))
+				file_uri_tuples.append(
+					(DistfileName(myfile, digests=mydigests.get(myfile)), None)
+				)
 
 	filedict = OrderedDict()
 	primaryuri_dict = {}

diff --git a/lib/portage/tests/ebuild/test_fetch.py b/lib/portage/tests/ebuild/test_fetch.py
index c5ea8253b..b88ae3efb 100644
--- a/lib/portage/tests/ebuild/test_fetch.py
+++ b/lib/portage/tests/ebuild/test_fetch.py
@@ -7,7 +7,8 @@ import tempfile
 
 import portage
 from portage import shutil, os
-from portage.const import BASH_BINARY, PORTAGE_PYM_PATH
+from portage.checksum import checksum_str
+from portage.const import BASH_BINARY, MANIFEST2_HASH_DEFAULTS, PORTAGE_PYM_PATH
 from portage.tests import TestCase
 from portage.tests.resolver.ResolverPlayground import ResolverPlayground
 from portage.tests.util.test_socks5 import AsyncHTTPServer
@@ -18,8 +19,14 @@ from portage.util._async.SchedulerInterface import SchedulerInterface
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage.package.ebuild.config import config
 from portage.package.ebuild.digestgen import digestgen
-from portage.package.ebuild.fetch import (_download_suffix, fetch, FlatLayout,
-		FilenameHashLayout, MirrorLayoutConfig)
+from portage.package.ebuild.fetch import (
+	DistfileName,
+	_download_suffix,
+	fetch,
+	FilenameHashLayout,
+	FlatLayout,
+	MirrorLayoutConfig,
+)
 from _emerge.EbuildFetcher import EbuildFetcher
 from _emerge.Package import Package
 
@@ -142,9 +149,14 @@ class EbuildFetchTestCase(TestCase):
 				content["/distfiles/layout.conf"] = layout_data.encode("utf8")
 
 				for k, v in distfiles.items():
+					filename = DistfileName(
+						k,
+						digests=dict((algo, checksum_str(v, hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS),
+					)
+
 					# mirror path
 					for layout in layouts:
-						content["/distfiles/" + layout.get_path(k)] = v
+						content["/distfiles/" + layout.get_path(filename)] = v
 					# upstream path
 					content["/distfiles/{}.txt".format(k)] = v
 
@@ -499,6 +511,10 @@ class EbuildFetchTestCase(TestCase):
 				io.StringIO(conf))
 
 	def test_filename_hash_layout_get_filenames(self):
+		filename = DistfileName(
+			'foo-1.tar.gz',
+			digests=dict((algo, checksum_str(b'', hashname=algo)) for algo in MANIFEST2_HASH_DEFAULTS),
+		)
 		layouts = (
 			FlatLayout(),
 			FilenameHashLayout('SHA1', '4'),
@@ -506,7 +522,6 @@ class EbuildFetchTestCase(TestCase):
 			FilenameHashLayout('SHA1', '8:16'),
 			FilenameHashLayout('SHA1', '8:16:24'),
 		)
-		filename = 'foo-1.tar.gz'
 
 		for layout in layouts:
 			distdir = tempfile.mkdtemp()
@@ -520,6 +535,12 @@ class EbuildFetchTestCase(TestCase):
 				with open(path, 'wb') as f:
 					pass
 
-				self.assertEqual([filename], list(layout.get_filenames(distdir)))
+				file_list = list(layout.get_filenames(distdir))
+				self.assertTrue(len(file_list) > 0)
+				for filename_result in file_list:
+					if isinstance(filename_result, DistfileName):
+						self.assertTrue(filename_result.digests_equal(filename))
+					else:
+						self.assertEqual(filename_result, str(filename))
 			finally:
 				shutil.rmtree(distdir)