From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id 8AB63138334 for ; Wed, 9 Oct 2019 06:00:57 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id CDF83E082B; Wed, 9 Oct 2019 06:00:56 +0000 (UTC) Received: from smtp.gentoo.org (woodpecker.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id A56ACE082B for ; Wed, 9 Oct 2019 06:00:56 +0000 (UTC) Received: from oystercatcher.gentoo.org (oystercatcher.gentoo.org [148.251.78.52]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 2511834BA34 for ; Wed, 9 Oct 2019 06:00:55 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id EF0127FF for ; Wed, 9 Oct 2019 06:00:52 +0000 (UTC) From: "Michał Górny" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Michał Górny" Message-ID: <1570600835.0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c.mgorny@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/_emirrordist/ X-VCS-Repository: proj/portage X-VCS-Files: lib/portage/_emirrordist/Config.py lib/portage/_emirrordist/DeletionIterator.py lib/portage/_emirrordist/DeletionTask.py lib/portage/_emirrordist/FetchTask.py lib/portage/_emirrordist/main.py lib/portage/package/ebuild/fetch.py X-VCS-Directories: lib/portage/package/ebuild/ lib/portage/_emirrordist/ X-VCS-Committer: mgorny X-VCS-Committer-Name: Michał Górny X-VCS-Revision: 0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c X-VCS-Branch: master Date: Wed, 9 Oct 2019 06:00:52 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply X-Archives-Salt: dac60730-8909-4929-ace6-c9322f999bf6 X-Archives-Hash: cf2760b1f994b6b1345a8345e7f513c0 commit: 0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c Author: Michał Górny gentoo org> AuthorDate: Sun Oct 6 19:44:55 2019 +0000 Commit: Michał Górny gentoo org> CommitDate: Wed Oct 9 06:00:35 2019 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d34d89d emirrordist: Implement mirror layout.conf support Read mirror layout from layout.conf in distfiles directory. When multiple layouts are used, fetch according to the first one and then hardlink according to the remaining layouts. Similarly, when deleting recycle or delete according to the first layout, then unlink according to all remaining layouts. Reviewed-by: Zac Medico gentoo.org> Closes: https://github.com/gentoo/portage/pull/463 Signed-off-by: Michał Górny gentoo.org> lib/portage/_emirrordist/Config.py | 8 +++- lib/portage/_emirrordist/DeletionIterator.py | 31 +++++++++----- lib/portage/_emirrordist/DeletionTask.py | 37 ++++++++++++----- lib/portage/_emirrordist/FetchTask.py | 60 ++++++++++++++++++++++------ lib/portage/_emirrordist/main.py | 6 +++ lib/portage/package/ebuild/fetch.py | 31 +++++++++++++- 6 files changed, 139 insertions(+), 34 deletions(-) diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py index 574a83559..ace3b0a6a 100644 --- a/lib/portage/_emirrordist/Config.py +++ b/lib/portage/_emirrordist/Config.py @@ -1,4 +1,4 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import copy @@ -10,6 +10,7 @@ import time import portage from portage import os +from portage.package.ebuild.fetch import MirrorLayoutConfig from portage.util import grabdict, grablines from portage.util._ShelveUnicodeWrapper import ShelveUnicodeWrapper @@ -72,6 +73,11 @@ class Config(object): self.deletion_db = self._open_shelve( options.deletion_db, 'deletion') + self.layout_conf = MirrorLayoutConfig() + self.layout_conf.read_from_file( + os.path.join(self.distfiles, 'layout.conf')) + self.layouts = self.layout_conf.get_all_layouts() + def _open_log(self, log_desc, log_path, mode): if log_path is None or self.options.dry_run: diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py index dff52c042..cd773b3c8 100644 --- a/lib/portage/_emirrordist/DeletionIterator.py +++ b/lib/portage/_emirrordist/DeletionIterator.py @@ -1,4 +1,4 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import logging @@ -20,17 +20,28 @@ class DeletionIterator(object): deletion_db = self._config.deletion_db deletion_delay = self._config.options.deletion_delay start_time = self._config.start_time - distfiles_set = set(os.listdir(self._config.options.distfiles)) + distfiles_set = set() + for layout in self._config.layouts: + distfiles_set.update(layout.get_filenames(distdir)) for filename in distfiles_set: - try: - st = os.stat(os.path.join(distdir, filename)) - except OSError as e: - logging.error("stat failed on '%s' in distfiles: %s\n" % - (filename, e)) - continue - if not stat.S_ISREG(st.st_mode): + # require at least one successful stat() + first_exception = None + for layout in reversed(self._config.layouts): + try: + st = os.stat( + os.path.join(distdir, layout.get_path(filename))) + except OSError as e: + first_exception = e + else: + if stat.S_ISREG(st.st_mode): + break + else: + if first_exception is not None: + logging.error("stat failed on '%s' in distfiles: %s\n" % + (filename, first_exception)) continue - elif filename in file_owners: + + if filename in file_owners: if deletion_db is not None: try: del deletion_db[filename] diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py index 7d10957fa..db5ac5ffb 100644 --- a/lib/portage/_emirrordist/DeletionTask.py +++ b/lib/portage/_emirrordist/DeletionTask.py @@ -1,4 +1,4 @@ -# Copyright 2013 Gentoo Foundation +# Copyright 2013-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 import errno @@ -15,10 +15,10 @@ class DeletionTask(CompositeTask): def _start(self): distfile_path = os.path.join( - self.config.options.distfiles, self.distfile) + self.config.options.distfiles, + self.config.layouts[0].get_path(self.distfile)) if self.config.options.recycle_dir is not None: - distfile_path = os.path.join(self.config.options.distfiles, self.distfile) recycle_path = os.path.join( self.config.options.recycle_dir, self.distfile) if self.config.options.dry_run: @@ -28,13 +28,14 @@ class DeletionTask(CompositeTask): logging.debug(("move '%s' from " "distfiles to recycle") % self.distfile) try: - os.rename(distfile_path, recycle_path) + # note: distfile_path can be a symlink here + os.rename(os.path.realpath(distfile_path), recycle_path) except OSError as e: if e.errno != errno.EXDEV: logging.error(("rename %s from distfiles to " "recycle failed: %s") % (self.distfile, e)) else: - self.returncode = os.EX_OK + self._delete_links() self._async_wait() return @@ -62,8 +63,7 @@ class DeletionTask(CompositeTask): success = False if success: - self._success() - self.returncode = os.EX_OK + self._delete_links() else: self.returncode = 1 @@ -93,14 +93,33 @@ class DeletionTask(CompositeTask): success = False if success: - self._success() - self.returncode = os.EX_OK + self._delete_links() else: self.returncode = 1 self._current_task = None self.wait() + def _delete_links(self): + success = True + for layout in self.config.layouts[1:]: + distfile_path = os.path.join( + self.config.options.distfiles, + layout.get_path(self.distfile)) + try: + os.unlink(distfile_path) + except OSError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + logging.error("%s unlink failed in distfiles: %s" % + (self.distfile, e)) + success = False + + if success: + self._success() + self.returncode = os.EX_OK + else: + self.returncode = 1 + def _success(self): cpv = "unknown" diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py index 1440b697c..7b68b7d3e 100644 --- a/lib/portage/_emirrordist/FetchTask.py +++ b/lib/portage/_emirrordist/FetchTask.py @@ -1,4 +1,4 @@ -# Copyright 2013-2014 Gentoo Foundation +# Copyright 2013-2019 Gentoo Authors # Distributed under the terms of the GNU General Public License v2 from __future__ import division @@ -14,6 +14,7 @@ import sys import portage from portage import _encodings, _unicode_encode from portage import os +from portage.util import ensure_dirs from portage.util._async.FileCopier import FileCopier from portage.util._async.FileDigester import FileDigester from portage.util._async.PipeLogger import PipeLogger @@ -61,7 +62,8 @@ class FetchTask(CompositeTask): return distfile_path = os.path.join( - self.config.options.distfiles, self.distfile) + self.config.options.distfiles, + self.config.layouts[0].get_path(self.distfile)) st = None size_ok = False @@ -332,7 +334,8 @@ class FetchTask(CompositeTask): return else: src = os.path.join(current_mirror.location, self.distfile) - dest = os.path.join(self.config.options.distfiles, self.distfile) + dest = os.path.join(self.config.options.distfiles, + self.config.layouts[0].get_path(self.distfile)) if self._hardlink_atomic(src, dest, "%s to %s" % (current_mirror.name, "distfiles")): logging.debug("hardlink '%s' from %s to distfiles" % @@ -501,7 +504,9 @@ class FetchTask(CompositeTask): except OSError: pass else: - dest = os.path.join(self.config.options.distfiles, self.distfile) + dest = os.path.join(self.config.options.distfiles, + self.config.layouts[0].get_path(self.distfile)) + ensure_dirs(os.path.dirname(dest)) try: os.rename(self._fetch_tmp_file, dest) except OSError: @@ -514,9 +519,7 @@ class FetchTask(CompositeTask): self._fetch_copier_exit) return else: - self._success() - self.returncode = os.EX_OK - self.wait() + self._make_layout_links() return self._try_next_mirror() @@ -535,9 +538,7 @@ class FetchTask(CompositeTask): return if copier.returncode == os.EX_OK: - self._success() - self.returncode = os.EX_OK - self.wait() + self._make_layout_links() else: # out of space? msg = "%s %s copy failed unexpectedly" % \ @@ -551,6 +552,38 @@ class FetchTask(CompositeTask): self.returncode = 1 self.wait() + def _make_layout_links(self): + dist_path = None + success = True + for layout in self.config.layouts[1:]: + if dist_path is None: + dist_path = os.path.join(self.config.options.distfiles, + self.config.layouts[0].get_path(self.distfile)) + link_path = os.path.join(self.config.options.distfiles, + layout.get_path(self.distfile)) + ensure_dirs(os.path.dirname(link_path)) + src_path = dist_path + if self.config.options.symlinks: + src_path = os.path.relpath(dist_path, + os.path.dirname(link_path)) + + if not self._hardlink_atomic(src_path, link_path, + "%s -> %s" % (link_path, src_path), + self.config.options.symlinks): + success = False + break + + if success: + self._success() + self.returncode = os.EX_OK + else: + self.config.log_failure("%s\t%s\t%s" % + (self.cpv, self.distfile, msg)) + self.config.file_failures[self.distfile] = self.cpv + self.returncode = 1 + + self.wait() + def _unlink_file(self, file_path, dir_info): try: os.unlink(file_path) @@ -595,7 +628,7 @@ class FetchTask(CompositeTask): else: return st1.st_dev == st2.st_dev - def _hardlink_atomic(self, src, dest, dir_info): + def _hardlink_atomic(self, src, dest, dir_info, symlink=False): head, tail = os.path.split(dest) hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \ @@ -603,7 +636,10 @@ class FetchTask(CompositeTask): try: try: - os.link(src, hardlink_tmp) + if symlink: + os.symlink(src, hardlink_tmp) + else: + os.link(src, hardlink_tmp) except OSError as e: if e.errno != errno.EXDEV: msg = "hardlink %s from %s failed: %s" % \ diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py index b63837b2a..0ae45ab6f 100644 --- a/lib/portage/_emirrordist/main.py +++ b/lib/portage/_emirrordist/main.py @@ -187,6 +187,12 @@ common_options = ( "action" : "append", "metavar" : "FILE" }, + { + "longopt" : "--symlinks", + "help" : "use symlinks rather than hardlinks for linking " + "distfiles between layouts", + "action" : "store_true" + }, ) def parse_args(args): diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py index 4458796fc..5d0bc7355 100644 --- a/lib/portage/package/ebuild/fetch.py +++ b/lib/portage/package/ebuild/fetch.py @@ -27,13 +27,14 @@ except ImportError: import portage portage.proxy.lazyimport.lazyimport(globals(), + 'glob:glob', 'portage.package.ebuild.config:check_config_instance,config', 'portage.package.ebuild.doebuild:doebuild_environment,' + \ '_doebuild_spawn', 'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs', 'portage.util:atomic_ofstream', 'portage.util.configparser:SafeConfigParser,read_configs,' + - 'NoOptionError,ConfigParserError', + 'ConfigParserError', 'portage.util._urlopen:urlopen', ) @@ -267,6 +268,9 @@ class FlatLayout(object): def get_path(self, filename): return filename + def get_filenames(self, distdir): + return iter(os.listdir(distdir)) + @staticmethod def verify_args(args): return len(args) == 1 @@ -287,6 +291,16 @@ class FilenameHashLayout(object): fnhash = fnhash[c:] return ret + filename + def get_filenames(self, distdir): + pattern = '' + for c in self.cutoffs: + assert c % 4 == 0 + c = c // 4 + pattern += c * '[0-9a-f]' + '/' + pattern += '*' + return (x.rsplit('/', 1)[1] + for x in glob(os.path.join(distdir, pattern))) + @staticmethod def verify_args(args): if len(args) != 3: @@ -322,7 +336,7 @@ class MirrorLayoutConfig(object): for i in itertools.count(): try: vals.append(tuple(cp.get('structure', '%d' % i).split())) - except NoOptionError: + except ConfigParserError: break self.structure = tuple(vals) @@ -351,6 +365,19 @@ class MirrorLayoutConfig(object): # fallback return FlatLayout() + def get_all_layouts(self): + ret = [] + for val in self.structure: + if not self.validate_structure(val): + raise ValueError("Unsupported structure: {}".format(val)) + if val[0] == 'flat': + ret.append(FlatLayout(*val[1:])) + elif val[0] == 'filename-hash': + ret.append(FilenameHashLayout(*val[1:])) + if not ret: + ret.append(FlatLayout()) + return ret + def get_mirror_url(mirror_url, filename, cache_path=None): """