From mboxrd@z Thu Jan 1 00:00:00 1970
Received: from pigeon.gentoo.org ([208.92.234.80] helo=lists.gentoo.org)
by finch.gentoo.org with esmtp (Exim 4.60)
(envelope-from <gentoo-commits+bounces-340754-garchives=archives.gentoo.org@lists.gentoo.org>)
id 1QGIn3-0003yy-SK
for garchives@archives.gentoo.org; Sat, 30 Apr 2011 22:37:30 +0000
Received: from pigeon.gentoo.org (localhost [127.0.0.1])
by pigeon.gentoo.org (Postfix) with SMTP id D390B1C043;
Sat, 30 Apr 2011 22:37:22 +0000 (UTC)
Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183])
by pigeon.gentoo.org (Postfix) with ESMTP id 928091C043
for <gentoo-commits@lists.gentoo.org>; Sat, 30 Apr 2011 22:37:22 +0000 (UTC)
Received: from pelican.gentoo.org (unknown [66.219.59.40])
(using TLSv1 with cipher ADH-CAMELLIA256-SHA (256/256 bits))
(No client certificate requested)
by smtp.gentoo.org (Postfix) with ESMTPS id E2DC51B402B
for <gentoo-commits@lists.gentoo.org>; Sat, 30 Apr 2011 22:37:21 +0000 (UTC)
Received: from localhost.localdomain (localhost [127.0.0.1])
by pelican.gentoo.org (Postfix) with ESMTP id 3FB5F80507
for <gentoo-commits@lists.gentoo.org>; Sat, 30 Apr 2011 22:37:21 +0000 (UTC)
From: "Brian Dolbec" <brian.dolbec@gmail.com>
To: gentoo-commits@lists.gentoo.org
Content-type: text/plain; charset=UTF-8
Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" <brian.dolbec@gmail.com>
Message-ID: <fcd35daad6e84fdaab69f4bd0c969b80daee590d.dol-sen@gentoo>
Subject: [gentoo-commits] proj/layman:master commit in: layman/
X-VCS-Repository: proj/layman
X-VCS-Files: layman/api.py layman/db.py
X-VCS-Directories: layman/
X-VCS-Committer: dol-sen
X-VCS-Committer-Name: Brian Dolbec
X-VCS-Revision: fcd35daad6e84fdaab69f4bd0c969b80daee590d
Date: Sat, 30 Apr 2011 22:37:21 +0000 (UTC)
Precedence: bulk
List-Post: <mailto:gentoo-commits@lists.gentoo.org>
List-Help: <mailto:gentoo-commits+help@lists.gentoo.org>
List-Unsubscribe: <mailto:gentoo-commits+unsubscribe@lists.gentoo.org>
List-Subscribe: <mailto:gentoo-commits+subscribe@lists.gentoo.org>
List-Id: Gentoo Linux mail <gentoo-commits.gentoo.org>
X-BeenThere: gentoo-commits@lists.gentoo.org
Content-Transfer-Encoding: quoted-printable
X-Archives-Salt:
X-Archives-Hash: c5e82e12bf842e0257da9a33f4a3c2d6
commit: fcd35daad6e84fdaab69f4bd0c969b80daee590d
Author: dol-sen <brian.dolbec <AT> gmail <DOT> com>
AuthorDate: Sat Apr 30 22:36:54 2011 +0000
Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com>
CommitDate: Sat Apr 30 22:36:54 2011 +0000
URL: http://git.overlays.gentoo.org/gitweb/?p=3Dproj/layman.git;a=3D=
commit;h=3Dfcd35daa
Add checks and code to fetch the new list using 'If-Modified-Since' proto=
col.
adds saving the 'last-modified' date of the same filename with a .timesta=
mp extension.
---
layman/api.py | 39 +++++++++++++++------------
layman/db.py | 80 ++++++++++++++++++++++++++++++++++++++++++++-------=
------
2 files changed, 84 insertions(+), 35 deletions(-)
diff --git a/layman/api.py b/layman/api.py
index cbb9d99..b71dd72 100644
--- a/layman/api.py
+++ b/layman/api.py
@@ -120,7 +120,7 @@ class LaymanAPI(object):
self._error(ERROR_INTERNAL_ERROR,
"Failed to disable repository '"+ovl+"':\n"+str(=
e))
results.append(False)
- self.get_installed(reload=3DTrue)
+ self.get_installed(dbreload=3DTrue)
if False in results:
return False
return True
@@ -151,7 +151,7 @@ class LaymanAPI(object):
self._error(ERROR_INTERNAL_ERROR,
"Failed to enable repository '"+ovl+"' : "+str(e=
))
results.append(False)
- self.get_installed(reload=3DTrue)
+ self.get_installed(dbreload=3DTrue)
if False in results:
return False
return True
@@ -373,47 +373,52 @@ class LaymanAPI(object):
def fetch_remote_list(self):
"""Fetches the latest remote overlay list"""
try:
- self._get_remote_db().cache()
+ dbreload =3D self._get_remote_db().cache()
+ self.output.debug(
+ 'LaymanAPI.fetch_remote_list(); cache updated =3D %s'
+ % str(dbreload),8)
except Exception as error:
self._error('Failed to fetch overlay list!\n Original Error =
was: '
+ str(error))
return False
- self.get_available(reload=3DTrue)
+ self.get_available(dbreload)
return True
=20
=20
- def get_available(self, reload=3DFalse):
+ def get_available(self, dbreload=3DFalse):
"""returns the list of available overlays"""
- if self._available_ids is None or reload:
- self._available_ids =3D self._get_remote_db(reload).list_ids=
()
+ self.output.info('LaymanAPI.get_available() dbreload =3D %s'
+ % str(dbreload))
+ if self._available_ids is None or dbreload:
+ self._available_ids =3D self._get_remote_db(dbreload).list_i=
ds()
return self._available_ids[:] or ['None']
=20
=20
- def get_installed(self, reload=3DFalse):
+ def get_installed(self, dbreload=3DFalse):
"""returns the list of installed overlays"""
- if self._installed_ids is None or reload:
- self._installed_ids =3D self._get_installed_db(reload).list_=
ids()
+ if self._installed_ids is None or dbreload:
+ self._installed_ids =3D self._get_installed_db(dbreload).lis=
t_ids()
return self._installed_ids[:]
=20
=20
- def _get_installed_db(self, reload=3DFalse):
+ def _get_installed_db(self, dbreload=3DFalse):
"""returns the list of installed overlays"""
- if not self._installed_db or reload:
+ if not self._installed_db or dbreload:
self._installed_db =3D DB(self.config)
return self._installed_db
=20
=20
- def _get_remote_db(self, reload=3DFalse):
+ def _get_remote_db(self, dbreload=3DFalse):
"""returns the list of installed overlays"""
- if self._available_db is None or reload:
+ if self._available_db is None or dbreload:
self._available_db =3D RemoteDB(self.config)
return self._available_db
=20
=20
def reload(self):
"""reloads the installed and remote db's to the data on disk"""
- result =3D self.get_available(reload=3DTrue)
- result =3D self.get_installed(reload=3DTrue)
+ result =3D self.get_available(dbreload=3DTrue)
+ result =3D self.get_installed(dbreload=3DTrue)
=20
=20
def _error(self, message):
@@ -423,7 +428,7 @@ class LaymanAPI(object):
"""
self._error_messages.append(message)
if self.report_errors:
- print(message, file=3Dstderr)
+ print(message, file=3Dself.config['stderr'])
=20
=20
def get_errors(self):
diff --git a/layman/db.py b/layman/db.py
index d078e46..3498223 100644
--- a/layman/db.py
+++ b/layman/db.py
@@ -24,7 +24,9 @@ __version__ =3D "$Id: db.py 309 2007-04-09 16:23:38Z wr=
obel $"
#
#-----------------------------------------------------------------------=
--------
=20
-import os, os.path, urllib2, hashlib
+import os, os.path
+import urllib2
+import hashlib
=20
from layman.utils import path, delete_empty_directory
from layman.dbbase import DbBase
@@ -243,7 +245,7 @@ class RemoteDB(DbBase):
=20
self.urls =3D [i.strip() for i in config['overlays'].split('\n'=
) if len(i)]
=20
- paths =3D [self.path(i) for i in self.urls]
+ paths =3D [self.filepath(i) + '.xml' for i in self.urls]
=20
if config['nocheck']:
ignore =3D 2
@@ -283,22 +285,45 @@ class RemoteDB(DbBase):
>>> a.overlays.keys()
[u'wrobel', u'wrobel-stable']
'''
+ has_updates =3D False
for url in self.urls:
=20
- mpath =3D self.path(url)
+ filepath =3D self.filepath(url)
+ mpath =3D filepath + '.xml'
+ tpath =3D filepath + '.timestamp'
=20
- # Check for sufficient privileges
- if os.path.exists(mpath) and not os.access(mpath, os.W_OK):
- self.output.warn('You do not have permission to update t=
he cache (%s).' % mpath)
- import getpass
- if getpass.getuser() !=3D 'root':
- self.output.warn('Hint: You are not root.\n')
+ # check when the cache was last updated
+ # and don't re-fetch it unless it has changed
+ request =3D urllib2.Request(url)
+ opener =3D urllib2.build_opener()
+ opener.addheaders =3D [('User-Agent', 'Layman-2.0-git')]
+
+ if os.path.exists(tpath):
+ with open(tpath,'r') as previous:
+ last_time =3D previous.read()
+ request.add_header('If-Modified-Since', last_time)
+
+ if not self.check_path([mpath]):
continue
=20
try:
-
+ connection =3D opener.open(request)
+ timestamp =3D connection.headers['last-modified']
+ except urllib2.HTTPError as e:
+ if e.getcode() =3D=3D 304:
+ self.output.info('Remote list already up to date: %s=
'
+ % url)
+ else:
+ self.output.info('RemoteDB.cache(); HTTPError was:\n=
%s'
+ % str(e))
+ continue
+ except IOError as error:
+ self.output.warn('Failed to update the overlay list from=
: '
+ + url + '\nError was:\n' + str(error))
+ else:
+ self.output.info('Fetching new list...')
# Fetch the remote list
- olist =3D urllib2.urlopen(url).read()
+ olist =3D connection.read()
=20
# Create our storage directory if it is missing
if not os.path.exists(os.path.dirname(mpath)):
@@ -326,24 +351,43 @@ class RemoteDB(DbBase):
out_file.write(olist)
out_file.close()
=20
+ out_file =3D open(tpath, 'w')
+ out_file.write(timestamp)
+ out_file.close()
+
+ has_updates =3D True
+
except Exception as error:
raise IOError('Failed to temporarily cache overlays =
list in'
' ' + mpath + '\nError was:\n' + str(e=
rror))
+ return has_updates
=20
=20
- except IOError as error:
- self.output.warn('Failed to update the overlay list from=
: '
- + url + '\nError was:\n' + str(error))
-
- def path(self, url):
+ def filepath(self, url):
'''Return a unique file name for the url.'''
=20
base =3D self.config['cache']
=20
self.output.debug('Generating cache path.', 6)
=20
- return base + '_' + hashlib.md5(url).hexdigest() + '.xml'
-
+ return base + '_' + hashlib.md5(url).hexdigest()
+
+
+ def check_path(self, paths, hint=3DTrue):
+ '''Check for sufficient privileges'''
+ self.output.debug('RemoteDB.check_path; paths =3D ' + str(paths)=
, 8)
+ is_ok =3D True
+ for path in paths:
+ if os.path.exists(path) and not os.access(path, os.W_OK):
+ if hint:
+ self.output.warn(
+ 'You do not have permission to update the cache =
(%s).'
+ % mpath)
+ import getpass
+ if getpass.getuser() !=3D 'root':
+ self.output.warn('Hint: You are not root.\n')
+ is_ok =3D False
+ return is_ok
=20
#=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=
=3D=3D=3D=3D=3D
#