* [gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/_emirrordist/
@ 2019-10-09 6:00 Michał Górny
0 siblings, 0 replies; only message in thread
From: Michał Górny @ 2019-10-09 6:00 UTC (permalink / raw
To: gentoo-commits
commit: 0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c
Author: Michał Górny <mgorny <AT> gentoo <DOT> org>
AuthorDate: Sun Oct 6 19:44:55 2019 +0000
Commit: Michał Górny <mgorny <AT> gentoo <DOT> org>
CommitDate: Wed Oct 9 06:00:35 2019 +0000
URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=0d34d89d
emirrordist: Implement mirror layout.conf support
Read mirror layout from layout.conf in distfiles directory. When
multiple layouts are used, fetch according to the first one and then
hardlink according to the remaining layouts. Similarly, when deleting
recycle or delete according to the first layout, then unlink according
to all remaining layouts.
Reviewed-by: Zac Medico <zmedico <AT> gentoo.org>
Closes: https://github.com/gentoo/portage/pull/463
Signed-off-by: Michał Górny <mgorny <AT> gentoo.org>
lib/portage/_emirrordist/Config.py | 8 +++-
lib/portage/_emirrordist/DeletionIterator.py | 31 +++++++++-----
lib/portage/_emirrordist/DeletionTask.py | 37 ++++++++++++-----
lib/portage/_emirrordist/FetchTask.py | 60 ++++++++++++++++++++++------
lib/portage/_emirrordist/main.py | 6 +++
lib/portage/package/ebuild/fetch.py | 31 +++++++++++++-
6 files changed, 139 insertions(+), 34 deletions(-)
diff --git a/lib/portage/_emirrordist/Config.py b/lib/portage/_emirrordist/Config.py
index 574a83559..ace3b0a6a 100644
--- a/lib/portage/_emirrordist/Config.py
+++ b/lib/portage/_emirrordist/Config.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import copy
@@ -10,6 +10,7 @@ import time
import portage
from portage import os
+from portage.package.ebuild.fetch import MirrorLayoutConfig
from portage.util import grabdict, grablines
from portage.util._ShelveUnicodeWrapper import ShelveUnicodeWrapper
@@ -72,6 +73,11 @@ class Config(object):
self.deletion_db = self._open_shelve(
options.deletion_db, 'deletion')
+ self.layout_conf = MirrorLayoutConfig()
+ self.layout_conf.read_from_file(
+ os.path.join(self.distfiles, 'layout.conf'))
+ self.layouts = self.layout_conf.get_all_layouts()
+
def _open_log(self, log_desc, log_path, mode):
if log_path is None or self.options.dry_run:
diff --git a/lib/portage/_emirrordist/DeletionIterator.py b/lib/portage/_emirrordist/DeletionIterator.py
index dff52c042..cd773b3c8 100644
--- a/lib/portage/_emirrordist/DeletionIterator.py
+++ b/lib/portage/_emirrordist/DeletionIterator.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import logging
@@ -20,17 +20,28 @@ class DeletionIterator(object):
deletion_db = self._config.deletion_db
deletion_delay = self._config.options.deletion_delay
start_time = self._config.start_time
- distfiles_set = set(os.listdir(self._config.options.distfiles))
+ distfiles_set = set()
+ for layout in self._config.layouts:
+ distfiles_set.update(layout.get_filenames(distdir))
for filename in distfiles_set:
- try:
- st = os.stat(os.path.join(distdir, filename))
- except OSError as e:
- logging.error("stat failed on '%s' in distfiles: %s\n" %
- (filename, e))
- continue
- if not stat.S_ISREG(st.st_mode):
+ # require at least one successful stat()
+ first_exception = None
+ for layout in reversed(self._config.layouts):
+ try:
+ st = os.stat(
+ os.path.join(distdir, layout.get_path(filename)))
+ except OSError as e:
+ first_exception = e
+ else:
+ if stat.S_ISREG(st.st_mode):
+ break
+ else:
+ if first_exception is not None:
+ logging.error("stat failed on '%s' in distfiles: %s\n" %
+ (filename, first_exception))
continue
- elif filename in file_owners:
+
+ if filename in file_owners:
if deletion_db is not None:
try:
del deletion_db[filename]
diff --git a/lib/portage/_emirrordist/DeletionTask.py b/lib/portage/_emirrordist/DeletionTask.py
index 7d10957fa..db5ac5ffb 100644
--- a/lib/portage/_emirrordist/DeletionTask.py
+++ b/lib/portage/_emirrordist/DeletionTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
import errno
@@ -15,10 +15,10 @@ class DeletionTask(CompositeTask):
def _start(self):
distfile_path = os.path.join(
- self.config.options.distfiles, self.distfile)
+ self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
if self.config.options.recycle_dir is not None:
- distfile_path = os.path.join(self.config.options.distfiles, self.distfile)
recycle_path = os.path.join(
self.config.options.recycle_dir, self.distfile)
if self.config.options.dry_run:
@@ -28,13 +28,14 @@ class DeletionTask(CompositeTask):
logging.debug(("move '%s' from "
"distfiles to recycle") % self.distfile)
try:
- os.rename(distfile_path, recycle_path)
+ # note: distfile_path can be a symlink here
+ os.rename(os.path.realpath(distfile_path), recycle_path)
except OSError as e:
if e.errno != errno.EXDEV:
logging.error(("rename %s from distfiles to "
"recycle failed: %s") % (self.distfile, e))
else:
- self.returncode = os.EX_OK
+ self._delete_links()
self._async_wait()
return
@@ -62,8 +63,7 @@ class DeletionTask(CompositeTask):
success = False
if success:
- self._success()
- self.returncode = os.EX_OK
+ self._delete_links()
else:
self.returncode = 1
@@ -93,14 +93,33 @@ class DeletionTask(CompositeTask):
success = False
if success:
- self._success()
- self.returncode = os.EX_OK
+ self._delete_links()
else:
self.returncode = 1
self._current_task = None
self.wait()
+ def _delete_links(self):
+ success = True
+ for layout in self.config.layouts[1:]:
+ distfile_path = os.path.join(
+ self.config.options.distfiles,
+ layout.get_path(self.distfile))
+ try:
+ os.unlink(distfile_path)
+ except OSError as e:
+ if e.errno not in (errno.ENOENT, errno.ESTALE):
+ logging.error("%s unlink failed in distfiles: %s" %
+ (self.distfile, e))
+ success = False
+
+ if success:
+ self._success()
+ self.returncode = os.EX_OK
+ else:
+ self.returncode = 1
+
def _success(self):
cpv = "unknown"
diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 1440b697c..7b68b7d3e 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2014 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division
@@ -14,6 +14,7 @@ import sys
import portage
from portage import _encodings, _unicode_encode
from portage import os
+from portage.util import ensure_dirs
from portage.util._async.FileCopier import FileCopier
from portage.util._async.FileDigester import FileDigester
from portage.util._async.PipeLogger import PipeLogger
@@ -61,7 +62,8 @@ class FetchTask(CompositeTask):
return
distfile_path = os.path.join(
- self.config.options.distfiles, self.distfile)
+ self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
st = None
size_ok = False
@@ -332,7 +334,8 @@ class FetchTask(CompositeTask):
return
else:
src = os.path.join(current_mirror.location, self.distfile)
- dest = os.path.join(self.config.options.distfiles, self.distfile)
+ dest = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
if self._hardlink_atomic(src, dest,
"%s to %s" % (current_mirror.name, "distfiles")):
logging.debug("hardlink '%s' from %s to distfiles" %
@@ -501,7 +504,9 @@ class FetchTask(CompositeTask):
except OSError:
pass
else:
- dest = os.path.join(self.config.options.distfiles, self.distfile)
+ dest = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
+ ensure_dirs(os.path.dirname(dest))
try:
os.rename(self._fetch_tmp_file, dest)
except OSError:
@@ -514,9 +519,7 @@ class FetchTask(CompositeTask):
self._fetch_copier_exit)
return
else:
- self._success()
- self.returncode = os.EX_OK
- self.wait()
+ self._make_layout_links()
return
self._try_next_mirror()
@@ -535,9 +538,7 @@ class FetchTask(CompositeTask):
return
if copier.returncode == os.EX_OK:
- self._success()
- self.returncode = os.EX_OK
- self.wait()
+ self._make_layout_links()
else:
# out of space?
msg = "%s %s copy failed unexpectedly" % \
@@ -551,6 +552,38 @@ class FetchTask(CompositeTask):
self.returncode = 1
self.wait()
+ def _make_layout_links(self):
+ dist_path = None
+ success = True
+ for layout in self.config.layouts[1:]:
+ if dist_path is None:
+ dist_path = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
+ link_path = os.path.join(self.config.options.distfiles,
+ layout.get_path(self.distfile))
+ ensure_dirs(os.path.dirname(link_path))
+ src_path = dist_path
+ if self.config.options.symlinks:
+ src_path = os.path.relpath(dist_path,
+ os.path.dirname(link_path))
+
+ if not self._hardlink_atomic(src_path, link_path,
+ "%s -> %s" % (link_path, src_path),
+ self.config.options.symlinks):
+ success = False
+ break
+
+ if success:
+ self._success()
+ self.returncode = os.EX_OK
+ else:
+ self.config.log_failure("%s\t%s\t%s" %
+ (self.cpv, self.distfile, msg))
+ self.config.file_failures[self.distfile] = self.cpv
+ self.returncode = 1
+
+ self.wait()
+
def _unlink_file(self, file_path, dir_info):
try:
os.unlink(file_path)
@@ -595,7 +628,7 @@ class FetchTask(CompositeTask):
else:
return st1.st_dev == st2.st_dev
- def _hardlink_atomic(self, src, dest, dir_info):
+ def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
@@ -603,7 +636,10 @@ class FetchTask(CompositeTask):
try:
try:
- os.link(src, hardlink_tmp)
+ if symlink:
+ os.symlink(src, hardlink_tmp)
+ else:
+ os.link(src, hardlink_tmp)
except OSError as e:
if e.errno != errno.EXDEV:
msg = "hardlink %s from %s failed: %s" % \
diff --git a/lib/portage/_emirrordist/main.py b/lib/portage/_emirrordist/main.py
index b63837b2a..0ae45ab6f 100644
--- a/lib/portage/_emirrordist/main.py
+++ b/lib/portage/_emirrordist/main.py
@@ -187,6 +187,12 @@ common_options = (
"action" : "append",
"metavar" : "FILE"
},
+ {
+ "longopt" : "--symlinks",
+ "help" : "use symlinks rather than hardlinks for linking "
+ "distfiles between layouts",
+ "action" : "store_true"
+ },
)
def parse_args(args):
diff --git a/lib/portage/package/ebuild/fetch.py b/lib/portage/package/ebuild/fetch.py
index 4458796fc..5d0bc7355 100644
--- a/lib/portage/package/ebuild/fetch.py
+++ b/lib/portage/package/ebuild/fetch.py
@@ -27,13 +27,14 @@ except ImportError:
import portage
portage.proxy.lazyimport.lazyimport(globals(),
+ 'glob:glob',
'portage.package.ebuild.config:check_config_instance,config',
'portage.package.ebuild.doebuild:doebuild_environment,' + \
'_doebuild_spawn',
'portage.package.ebuild.prepare_build_dirs:prepare_build_dirs',
'portage.util:atomic_ofstream',
'portage.util.configparser:SafeConfigParser,read_configs,' +
- 'NoOptionError,ConfigParserError',
+ 'ConfigParserError',
'portage.util._urlopen:urlopen',
)
@@ -267,6 +268,9 @@ class FlatLayout(object):
def get_path(self, filename):
return filename
+ def get_filenames(self, distdir):
+ return iter(os.listdir(distdir))
+
@staticmethod
def verify_args(args):
return len(args) == 1
@@ -287,6 +291,16 @@ class FilenameHashLayout(object):
fnhash = fnhash[c:]
return ret + filename
+ def get_filenames(self, distdir):
+ pattern = ''
+ for c in self.cutoffs:
+ assert c % 4 == 0
+ c = c // 4
+ pattern += c * '[0-9a-f]' + '/'
+ pattern += '*'
+ return (x.rsplit('/', 1)[1]
+ for x in glob(os.path.join(distdir, pattern)))
+
@staticmethod
def verify_args(args):
if len(args) != 3:
@@ -322,7 +336,7 @@ class MirrorLayoutConfig(object):
for i in itertools.count():
try:
vals.append(tuple(cp.get('structure', '%d' % i).split()))
- except NoOptionError:
+ except ConfigParserError:
break
self.structure = tuple(vals)
@@ -351,6 +365,19 @@ class MirrorLayoutConfig(object):
# fallback
return FlatLayout()
+ def get_all_layouts(self):
+ ret = []
+ for val in self.structure:
+ if not self.validate_structure(val):
+ raise ValueError("Unsupported structure: {}".format(val))
+ if val[0] == 'flat':
+ ret.append(FlatLayout(*val[1:]))
+ elif val[0] == 'filename-hash':
+ ret.append(FilenameHashLayout(*val[1:]))
+ if not ret:
+ ret.append(FlatLayout())
+ return ret
+
def get_mirror_url(mirror_url, filename, cache_path=None):
"""
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2019-10-09 6:00 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2019-10-09 6:00 [gentoo-commits] proj/portage:master commit in: lib/portage/package/ebuild/, lib/portage/_emirrordist/ Michał Górny
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox