From mboxrd@z Thu Jan 1 00:00:00 1970 Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org) by finch.gentoo.org with esmtp (Exim 4.60) (envelope-from ) id 1N1ahq-0001Jx-RQ for garchives@archives.gentoo.org; Sat, 24 Oct 2009 07:06:31 +0000 Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id CDEFEE0860; Sat, 24 Oct 2009 07:06:29 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) by pigeon.gentoo.org (Postfix) with ESMTP id 8D8BFE0860 for ; Sat, 24 Oct 2009 07:06:29 +0000 (UTC) Received: from stork.gentoo.org (stork.gentoo.org [64.127.104.133]) (using TLSv1 with cipher AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTP id 32EFB66F18 for ; Sat, 24 Oct 2009 07:06:29 +0000 (UTC) Received: from zmedico by stork.gentoo.org with local (Exim 4.69) (envelope-from ) id 1N1aho-0007YS-Nv for gentoo-commits@lists.gentoo.org; Sat, 24 Oct 2009 07:06:28 +0000 To: gentoo-commits@lists.gentoo.org From: "Zac Medico (zmedico)" Subject: [gentoo-commits] portage r14718 - main/branches/2.1.7/bin X-VCS-Repository: portage X-VCS-Revision: 14718 X-VCS-Files: main/branches/2.1.7/bin/repoman X-VCS-Directories: main/branches/2.1.7/bin X-VCS-Committer: zmedico X-VCS-Committer-Name: Zac Medico Content-Type: text/plain; charset=UTF-8 Message-Id: Sender: Zac Medico Date: Sat, 24 Oct 2009 07:06:28 +0000 Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: quoted-printable X-Archives-Salt: eee50bee-df77-4ff8-8c5b-501908565eb4 X-Archives-Hash: fe8587baad9cfd4336896bc608265e7d Author: zmedico Date: 2009-10-24 07:06:28 +0000 (Sat, 24 Oct 2009) New Revision: 14718 Modified: main/branches/2.1.7/bin/repoman Log: Use urllib.urlopen() instead of portage.fetch() for fetching metadata.dtd= . (trunk r14709) Modified: main/branches/2.1.7/bin/repoman =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D --- main/branches/2.1.7/bin/repoman 2009-10-24 07:06:07 UTC (rev 14717) +++ main/branches/2.1.7/bin/repoman 2009-10-24 07:06:28 UTC (rev 14718) @@ -25,6 +25,7 @@ import tempfile import time import platform +import urllib =20 from io import StringIO from itertools import chain @@ -440,6 +441,10 @@ "x11-misc/imake", ]) =20 +metadata_dtd_uri =3D 'http://www.gentoo.org/dtd/metadata.dtd' +# force refetch if the local copy creation time is older than this +metadata_dtd_ctime_interval =3D 60 * 60 * 24 * 7 # 7 days + # file.executable no_exec =3D frozenset(["Manifest","ChangeLog","metadata.xml"]) =20 @@ -765,59 +770,80 @@ for x in qacats: stats[x]=3D0 fails[x]=3D[] + xmllint_capable =3D False metadata_dtd =3D os.path.join(repoman_settings["DISTDIR"], 'metadata.dtd= ') -if options.mode =3D=3D "manifest": - pass -elif not find_binary('xmllint'): - print(red("!!! xmllint not found. Can't check metadata.xml.\n")) - if options.xml_parse or repolevel=3D=3D3: - print(red("!!!")+" sorry, xmllint is needed. failing\n") - sys.exit(1) -else: - #hardcoded paths/urls suck. :-/ - must_fetch=3D1 - backup_exists=3D0 + +def fetch_metadata_dtd(): + """ + Fetch metadata.dtd if it doesn't exist or the ctime is older than + metadata_dtd_ctime_interval. + @rtype: bool + @returns: True if successful, otherwise False + """ + + must_fetch =3D True + metadata_dtd_st =3D None + current_time =3D int(time.time()) try: - # if it's been over a week since fetching (or the system clock is fsck= ed), grab an updated copy of metadata.dtd=20 - # clock is fscked or it's been a week. time to grab a new one. - ct =3D os.stat(metadata_dtd)[ST_CTIME] - if abs(time.time() - ct) > (60*60*24*7): - # don't trap the exception, we're watching for errno 2 (file not foun= d), anything else is a bug. - backup_exists=3D1 - else: - must_fetch=3D0 + metadata_dtd_st =3D os.stat(metadata_dtd) + except EnvironmentError as e: + if e.errno not in (errno.ENOENT, errno.ESTALE): + raise + del e + else: + # Trigger fetch if metadata.dtd mtime is old or clock is wrong. + if abs(current_time - metadata_dtd_st.st_ctime) \ + < metadata_dtd_ctime_interval: + must_fetch =3D False =20 - except (OSError,IOError) as e: - if e.errno !=3D 2: - print(red("!!!")+" caught exception '%s' for %s/metadata.dtd, bailing= " % (str(e), portage.CACHE_PATH)) - sys.exit(1) - if must_fetch: - print()=20 - print(green("***")+" the local copy of metadata.dtd needs to be refetc= hed, doing that now") print() - val =3D 0 + print(green("***") + " the local copy of metadata.dtd " + \ + "needs to be refetched, doing that now") + print() try: + url_f =3D urllib.urlopen(metadata_dtd_uri) + last_modified =3D url_f.info().getdate('last-modified') + if last_modified is not None: + last_modified =3D time.mktime(last_modified) + + metadata_dtd_tmp =3D "%s.%s" % (metadata_dtd, os.getpid()) try: - os.unlink(metadata_dtd) - except OSError as e: - if e.errno !=3D errno.ENOENT: - raise - del e - val=3Dportage.fetch(['http://www.gentoo.org/dtd/metadata.dtd'],repoma= n_settings,fetchonly=3D0, \ - try_mirrors=3D0) + local_f =3D open(metadata_dtd_tmp, mode=3D'wb') + local_f.write(url_f.read()) + local_f.close() + if last_modified is not None: + os.utime(metadata_dtd_tmp, + (int(last_modified), int(last_modified))) + os.rename(metadata_dtd_tmp, metadata_dtd) + finally: + try: + os.unlink(metadata_dtd_tmp) + except OSError: + pass =20 - except SystemExit as e: - raise # Need to propogate this - except Exception as e: + url_f.close() + + except EnvironmentError as e: print() - print(red("!!!")+" attempting to fetch 'http://www.gentoo.org/dtd/met= adata.dtd', caught") - print(red("!!!")+" exception '%s' though." % str(e)) - val=3D0 - if not val: + print(red("!!!")+" attempting to fetch '%s', caught" % metadata_dtd_u= ri) + print(red("!!!")+" exception '%s' though." % (e,)) print(red("!!!")+" fetching new metadata.dtd failed, aborting") - sys.exit(1) + return False + + return True + +if options.mode =3D=3D "manifest": + pass +elif not find_binary('xmllint'): + print(red("!!! xmllint not found. Can't check metadata.xml.\n")) + if options.xml_parse or repolevel=3D=3D3: + print(red("!!!")+" sorry, xmllint is needed. failing\n") + sys.exit(1) +else: + if not fetch_metadata_dtd(): + sys.exit(1) #this can be problematic if xmllint changes their output xmllint_capable=3DTrue =20