From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1Ryb9x-0006GJ-1i for garchives@archives.gentoo.org; Sat, 18 Feb 2012 03:40:29 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 743FDE087F; Sat, 18 Feb 2012 03:40:20 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 27D0EE087F for ; Sat, 18 Feb 2012 03:40:20 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 2CDBB1B4004 for ; Sat, 18 Feb 2012 03:40:19 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id DAB7BE5400 for ; Sat, 18 Feb 2012 03:40:17 +0000 (UTC) From: "Zac Medico" To: gentoo-commits@lists.gentoo.org Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Zac Medico" Message-ID: <1329536403.848da97a64b2d3d13c3fbc794c57dae714009854.zmedico@gentoo> Subject: [gentoo-commits] proj/portage:master commit in: pym/portage/util/ X-VCS-Repository: proj/portage X-VCS-Files: pym/portage/util/mtimedb.py X-VCS-Directories: pym/portage/util/ X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico X-VCS-Revision: 848da97a64b2d3d13c3fbc794c57dae714009854 X-VCS-Branch: master Date: Sat, 18 Feb 2012 03:40:17 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: c88d8c1c-70e5-4dda-a9c2-af050b14e6a1 X-Archives-Hash: 8a3438a3c995334bd36bd060063d3865 commit: 848da97a64b2d3d13c3fbc794c57dae714009854 Author: Zac Medico gentoo org> AuthorDate: Sat Feb 18 03:40:03 2012 +0000 Commit: Zac Medico gentoo org> CommitDate: Sat Feb 18 03:40:03 2012 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/portage.git;a= =3Dcommit;h=3D848da97a MtimeDB: add JSON read/write Support serialization as JSON instead of pickle, so that /var/cache/edb/mtimedb is human readable/writable, for those rare cases where it may be useful. Currently, pickle is still used for writes. The plan is to migrate to JSON after JSON read has been supported for some time. --- pym/portage/util/mtimedb.py | 69 ++++++++++++++++++++++++++++++++++---= ------ 1 files changed, 55 insertions(+), 14 deletions(-) diff --git a/pym/portage/util/mtimedb.py b/pym/portage/util/mtimedb.py index 0a27166..4ccea24 100644 --- a/pym/portage/util/mtimedb.py +++ b/pym/portage/util/mtimedb.py @@ -10,13 +10,32 @@ except ImportError: import pickle =20 import errno +import io +import json +import sys + import portage +from portage import _encodings +from portage import _unicode_decode from portage import _unicode_encode from portage.data import portage_gid, uid from portage.localization import _ from portage.util import apply_secpass_permissions, atomic_ofstream, wri= temsg =20 class MtimeDB(dict): + + # Enable this after JSON read has been supported for some time. + _json_write =3D False + + _json_write_opts =3D { + "ensure_ascii": False, + "indent": "\t", + "sort_keys": True + } + if sys.hexversion < 0x3020000: + # indent only supports int number of spaces + _json_write_opts["indent"] =3D 4 + def __init__(self, filename): dict.__init__(self) self.filename =3D filename @@ -24,28 +43,45 @@ class MtimeDB(dict): =20 def _load(self, filename): f =3D None + content =3D None try: f =3D open(_unicode_encode(filename), 'rb') - mypickle =3D pickle.Unpickler(f) - try: - mypickle.find_global =3D None - except AttributeError: - # TODO: If py3k, override Unpickler.find_class(). - pass - d =3D mypickle.load() - except (AttributeError, EOFError, EnvironmentError, ValueError, pickle= .UnpicklingError) as e: - if isinstance(e, EnvironmentError) and \ - getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): + content =3D f.read() + except EnvironmentError as e: + if getattr(e, 'errno', None) in (errno.ENOENT, errno.EACCES): pass else: writemsg(_("!!! Error loading '%s': %s\n") % \ - (filename, str(e)), noiselevel=3D-1) - del e - d =3D {} + (filename, e), noiselevel=3D-1) finally: if f is not None: f.close() =20 + d =3D None + if content: + try: + mypickle =3D pickle.Unpickler(io.BytesIO(content)) + try: + mypickle.find_global =3D None + except AttributeError: + # TODO: If py3k, override Unpickler.find_class(). + pass + d =3D mypickle.load() + except SystemExit: + raise + except Exception as e: + try: + d =3D json.loads(_unicode_decode(content, + encoding=3D_encodings['repo.content'], errors=3D'strict')) + except SystemExit: + raise + except Exception: + writemsg(_("!!! Error loading '%s': %s\n") % \ + (filename, e), noiselevel=3D-1) + + if d is None: + d =3D {} + if "old" in d: d["updates"] =3D d["old"] del d["old"] @@ -80,7 +116,12 @@ class MtimeDB(dict): except EnvironmentError: pass else: - pickle.dump(d, f, protocol=3D2) + if self._json_write: + f.write(_unicode_encode( + json.dumps(d, **self._json_write_opts), + encoding=3D_encodings['repo.content'], errors=3D'strict')) + else: + pickle.dump(d, f, protocol=3D2) f.close() apply_secpass_permissions(self.filename, uid=3Duid, gid=3Dportage_gid, mode=3D0o644)