From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by finch.gentoo.org (Postfix) with ESMTPS id F2E2C139083 for ; Thu, 17 Aug 2017 01:59:57 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 4BA6DE0CB9; Thu, 17 Aug 2017 01:59:57 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [IPv6:2001:470:ea4a:1:5054:ff:fec7:86e4]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id 136D5E0CB9 for ; Thu, 17 Aug 2017 01:59:57 +0000 (UTC) Received: from oystercatcher.gentoo.org (unknown [IPv6:2a01:4f8:202:4333:225:90ff:fed9:fc84]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id E2D0134187F for ; Thu, 17 Aug 2017 01:59:55 +0000 (UTC) Received: from localhost.localdomain (localhost [IPv6:::1]) by oystercatcher.gentoo.org (Postfix) with ESMTP id E1CB27860 for ; Thu, 17 Aug 2017 01:59:50 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1502925927.2d7ed7a1fdac828dc5f6b26ec279b2d1e4ee83da.dolsen@gentoo> Subject: [gentoo-commits] proj/portage:repoman commit in: repoman/pym/repoman/ X-VCS-Repository: proj/portage X-VCS-Files: repoman/pym/repoman/config.py X-VCS-Directories: repoman/pym/repoman/ X-VCS-Committer: dolsen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 2d7ed7a1fdac828dc5f6b26ec279b2d1e4ee83da X-VCS-Branch: repoman Date: Thu, 17 Aug 2017 01:59:50 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 0b2ed796-bcf0-412f-a1a8-950b3258875c X-Archives-Hash: b8e80d4ed7af6da7e27bd03fc1052549 commit: 2d7ed7a1fdac828dc5f6b26ec279b2d1e4ee83da Author: Brian Dolbec gentoo org> AuthorDate: Wed Aug 16 23:23:09 2017 +0000 Commit: Brian Dolbec gentoo org> CommitDate: Wed Aug 16 23:25:27 2017 +0000 URL: https://gitweb.gentoo.org/proj/portage.git/commit/?id=2d7ed7a1 repoman: config.py: Fix whitespace repoman/pym/repoman/config.py | 245 +++++++++++++++++++++--------------------- 1 file changed, 124 insertions(+), 121 deletions(-) diff --git a/repoman/pym/repoman/config.py b/repoman/pym/repoman/config.py index 2825aee04..f41945c13 100644 --- a/repoman/pym/repoman/config.py +++ b/repoman/pym/repoman/config.py @@ -11,133 +11,136 @@ import yaml class ConfigError(Exception): - """Raised when a config file fails to load""" - pass + """Raised when a config file fails to load""" + pass def merge_config(base, head): - """ - Merge two JSON or YAML documents into a single object. Arrays are - merged by extension. If dissimilar types are encountered, then the - head value overwrites the base value. - """ - - if isinstance(head, dict): - if not isinstance(base, dict): - return copy.deepcopy(head) - - result = {} - for k in itertools.chain(head, base): - try: - result[k] = merge_config(base[k], head[k]) - except KeyError: - try: - result[k] = copy.deepcopy(head[k]) - except KeyError: - result[k] = copy.deepcopy(base[k]) - - elif isinstance(head, list): - result = [] - if not isinstance(base, list): - result.extend(copy.deepcopy(x) for x in head) - else: - if any(isinstance(x, (dict, list)) for x in itertools.chain(head, base)): - # merge items with identical indexes - for x, y in zip(base, head): - if isinstance(x, (dict, list)): - result.append(merge_config(x, y)) - else: - # head overwrites base (preserving index) - result.append(copy.deepcopy(y)) - # copy remaining items from the longer list - if len(base) != len(head): - if len(base) > len(head): - result.extend(copy.deepcopy(x) for x in base[len(head):]) - else: - result.extend(copy.deepcopy(x) for x in head[len(base):]) - else: - result.extend(copy.deepcopy(x) for x in base) - result.extend(copy.deepcopy(x) for x in head) - - else: - result = copy.deepcopy(head) - - return result + """ + Merge two JSON or YAML documents into a single object. Arrays are + merged by extension. If dissimilar types are encountered, then the + head value overwrites the base value. + """ + + if isinstance(head, dict): + if not isinstance(base, dict): + return copy.deepcopy(head) + + result = {} + for k in itertools.chain(head, base): + try: + result[k] = merge_config(base[k], head[k]) + except KeyError: + try: + result[k] = copy.deepcopy(head[k]) + except KeyError: + result[k] = copy.deepcopy(base[k]) + + elif isinstance(head, list): + result = [] + if not isinstance(base, list): + result.extend(copy.deepcopy(x) for x in head) + else: + if any(isinstance(x, (dict, list)) for x in itertools.chain(head, base)): + # merge items with identical indexes + for x, y in zip(base, head): + if isinstance(x, (dict, list)): + result.append(merge_config(x, y)) + else: + # head overwrites base (preserving index) + result.append(copy.deepcopy(y)) + # copy remaining items from the longer list + if len(base) != len(head): + if len(base) > len(head): + result.extend(copy.deepcopy(x) for x in base[len(head):]) + else: + result.extend(copy.deepcopy(x) for x in head[len(base):]) + else: + result.extend(copy.deepcopy(x) for x in base) + result.extend(copy.deepcopy(x) for x in head) + + else: + result = copy.deepcopy(head) + + return result def _yaml_load(filename): - """ - Load filename as YAML and return a dict. Raise ConfigError if - it fails to load. - """ - with open(filename, 'rt') as f: - try: - return yaml.safe_load(f) - except yaml.parser.ParserError as e: - raise ConfigError("{}: {}".format(filename, e)) + """ + Load filename as YAML and return a dict. Raise ConfigError if + it fails to load. + """ + with open(filename, 'rt') as f: + try: + return yaml.safe_load(f) + except yaml.parser.ParserError as e: + raise ConfigError("{}: {}".format(filename, e)) def _json_load(filename): - """ - Load filename as JSON and return a dict. Raise ConfigError if - it fails to load. - """ - with open(filename, 'rt') as f: - try: - return json.load(f) #nosec - except ValueError as e: - raise ConfigError("{}: {}".format(filename, e)) + """ + Load filename as JSON and return a dict. Raise ConfigError if + it fails to load. + """ + with open(filename, 'rt') as f: + try: + return json.load(f) #nosec + except ValueError as e: + raise ConfigError("{}: {}".format(filename, e)) def iter_files(files_dirs): - """ - Iterate over nested file paths in lexical order. - """ - stack = list(reversed(files_dirs)) - while stack: - location = stack.pop() - try: - st = os.stat(location) - except FileNotFoundError: - continue - - if stat.S_ISDIR(st.st_mode): - stack.extend(os.path.join(location, x) - for x in sorted(os.listdir(location), reverse=True)) - - elif stat.S_ISREG(st.st_mode): - yield location - -def load_config(conf_dirs, file_extensions=None): - """ - Load JSON and/or YAML files from a directories, and merge them together - into a single object. - """ - - result = {} - for filename in iter_files(conf_dirs): - if file_extensions is not None and not filename.endswith(file_extensions): - continue - - loaders = [] - if filename.endswith('.json'): - loaders.append(_json_load) - elif filename.endswith('.yaml'): - loaders.append(_yaml_load) - else: - loaders.append(_yaml_load) - loaders.append(_json_load) - - config = None - for loader in loaders: - try: - config = loader(filename) or {} - except ConfigError as e: - exception = e - else: - break - - if config is None: - raise exception - - if config: - result = merge_config(result, config) - - return result + """ + Iterate over nested file paths in lexical order. + """ + stack = list(reversed(files_dirs)) + while stack: + location = stack.pop() + try: + st = os.stat(location) + except FileNotFoundError: + continue + + if stat.S_ISDIR(st.st_mode): + stack.extend(os.path.join(location, x) + for x in sorted(os.listdir(location), reverse=True)) + + elif stat.S_ISREG(st.st_mode): + yield location + +def load_config(conf_dirs, file_extensions=None, valid_versions=None): + """ + Load JSON and/or YAML files from a directories, and merge them together + into a single object. + """ + + result = {} + for filename in iter_files(conf_dirs): + if file_extensions is not None and not filename.endswith(file_extensions): + continue + + loaders = [] + if filename.endswith('.json'): + loaders.append(_json_load) + elif filename.endswith('.yaml'): + loaders.append(_yaml_load) + else: + loaders.append(_yaml_load) + loaders.append(_json_load) + + config = None + for loader in loaders: + try: + config = loader(filename) or {} + except ConfigError as e: + exception = e + else: + break + + if config is None: + raise exception + + if config: + if config['version'] not in valid_versions: + raise ConfigError("Invalid file version: %s in: %s\nPlease upgrade repoman: current valid versions: %s" + % (config['version'], filename, valid_versions)) + result = merge_config(result, config) + + return result