From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from lists.gentoo.org (pigeon.gentoo.org [208.92.234.80]) by finch.gentoo.org (Postfix) with ESMTP id C561C1381F3 for ; Fri, 18 Oct 2013 14:26:26 +0000 (UTC) Received: from pigeon.gentoo.org (localhost [127.0.0.1]) by pigeon.gentoo.org (Postfix) with SMTP id 583D7E0BA2; Fri, 18 Oct 2013 14:26:26 +0000 (UTC) Received: from smtp.gentoo.org (smtp.gentoo.org [140.211.166.183]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by pigeon.gentoo.org (Postfix) with ESMTPS id D8012E0BA2 for ; Fri, 18 Oct 2013 14:26:25 +0000 (UTC) Received: from hornbill.gentoo.org (hornbill.gentoo.org [94.100.119.163]) (using TLSv1 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by smtp.gentoo.org (Postfix) with ESMTPS id 852A033EFBA for ; Fri, 18 Oct 2013 14:26:24 +0000 (UTC) Received: from localhost.localdomain (localhost [127.0.0.1]) by hornbill.gentoo.org (Postfix) with ESMTP id 366C9E5308 for ; Fri, 18 Oct 2013 14:26:23 +0000 (UTC) From: "Brian Dolbec" To: gentoo-commits@lists.gentoo.org Content-Transfer-Encoding: 8bit Content-type: text/plain; charset=UTF-8 Reply-To: gentoo-dev@lists.gentoo.org, "Brian Dolbec" Message-ID: <1382105682.6a703ce2d162d96df2a08124e3d3ca3d2f657f07.dol-sen@gentoo> Subject: [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/ X-VCS-Repository: proj/mirrorselect X-VCS-Files: mirrorselect/selectors.py X-VCS-Directories: mirrorselect/ X-VCS-Committer: dol-sen X-VCS-Committer-Name: Brian Dolbec X-VCS-Revision: 6a703ce2d162d96df2a08124e3d3ca3d2f657f07 X-VCS-Branch: master Date: Fri, 18 Oct 2013 14:26:23 +0000 (UTC) Precedence: bulk List-Post: List-Help: List-Unsubscribe: List-Subscribe: List-Id: Gentoo Linux mail X-BeenThere: gentoo-commits@lists.gentoo.org X-Archives-Salt: 6c2df78c-4221-4d7c-a2d5-c0f399c88451 X-Archives-Hash: bdb594102597b0b642c9b1244f96575f commit: 6a703ce2d162d96df2a08124e3d3ca3d2f657f07 Author: Brian Dolbec gentoo org> AuthorDate: Fri Oct 18 06:45:23 2013 +0000 Commit: Brian Dolbec gmail com> CommitDate: Fri Oct 18 14:14:42 2013 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=6a703ce2 refactor Deep classes deeptime() Thanks to Douglas Freed, fixed many errors, including dns errors on my system. Adds improved debug reporting. Adds total download failures output to the results. --- mirrorselect/selectors.py | 65 +++++++++++++++++++++++++++-------------------- 1 file changed, 37 insertions(+), 28 deletions(-) diff --git a/mirrorselect/selectors.py b/mirrorselect/selectors.py index 2c7bac8..4da6156 100644 --- a/mirrorselect/selectors.py +++ b/mirrorselect/selectors.py @@ -39,11 +39,13 @@ import hashlib if sys.version_info[0] >= 3: import urllib.request, urllib.parse, urllib.error url_parse = urllib.parse.urlparse + url_unparse = urllib.parse.urlunparse url_open = urllib.request.urlopen else: import urllib import urlparse url_parse = urlparse.urlparse + url_unparse = urlparse.urlunparse url_open = urllib.urlopen @@ -200,9 +202,7 @@ class Deep(object): elif options.ipv6: addr_families.append(socket.AF_INET6) else: - addr_families.append(socket.AF_INET) - if socket.has_ipv6: - addr_families.append(socket.AF_INET6) + addr_families.append(socket.AF_UNSPEC) self._addr_families = addr_families @@ -219,6 +219,7 @@ class Deep(object): maxtime = self._download_timeout hosts = [host[0] for host in self._hosts] num_hosts = len(hosts) + self.dl_failures = 0 for host in hosts: @@ -252,7 +253,9 @@ class Deep(object): #self.output.write('deeptest(): adding rethost %s, %s' % (key, top_hosts[key]), 2) rethosts.append(top_hosts[key]) - self.output.write('deeptest(): final rethost %s' % (rethosts), 2) + self.output.write('deeptest(): final rethost %s\n' % (rethosts), 2) + self.output.write('deeptest(): final md5 failures %s of %s\n' + % (self.dl_failures, num_hosts), 2) self.urls = rethosts @@ -280,40 +283,38 @@ class Deep(object): signal.signal(signal.SIGALRM, timeout_handler) ips = [] - for family in self._addr_families: - ipv6 = family == socket.AF_INET6 + for addr_family in self._addr_families: try: - try: - signal.alarm(self._dns_timeout) - for family, socktype, proto, canonname, sockaddr in \ - socket.getaddrinfo(url_parts.hostname, None, - family, socket.SOCK_STREAM): - ip = sockaddr[0] - if ipv6: - ip = "[%s]" % ip - ips.append(ip) - finally: - signal.alarm(0) + signal.alarm(self._dns_timeout) + for result in socket.getaddrinfo(url_parts.hostname, None, + addr_family, socket.SOCK_STREAM, 0, socket.AI_ADDRCONFIG): + family, _, __, ___, sockaddr = result + ip = sockaddr[0] + if family == socket.AF_INET6: + ip = "[%s]" % ip + ips.append(ip) except socket.error as e: - self.output.write('deeptime(): dns error for host %s: %s\n' % \ - (url_parts.hostname, e), 2) + self.output.write('deeptime(): dns error for host %s: %s\n' % (url_parts.hostname, e), 2) except TimeoutException: - self.output.write('deeptime(): dns timeout for host %s\n' % \ - (url_parts.hostname,), 2) + self.output.write('deeptime(): dns timeout for host %s\n' % url_parts.hostname, 2) + finally: + signal.alarm(0) if not ips: - self.output.write('deeptime(): unable to resolve ip for host %s\n' % \ - (url_parts.hostname,), 2) + self.output.write('deeptime(): unable to resolve ip for host %s\n' % url_parts.hostname, 2) return (None, True) delta = 0 f = None for ip in ips: + test_parts = url_parts._replace(netloc=ip) + test_url = url_unparse(test_parts) + self.output.write('deeptime(): testing url: %s\n' % test_url, 2) try: try: signal.alarm(self._connect_timeout) - f = url_open(url) + f = url_open(test_url) break finally: signal.alarm(0) @@ -324,7 +325,7 @@ class Deep(object): except TimeoutException: self.output.write(('deeptime(): connection to host %s ' + \ 'timed out for ip %s\n') % \ - (url_parts.hostname, ip), 2) + (test_parts.hostname, ip), 2) if f is None: self.output.write('deeptime(): unable to ' + \ @@ -355,13 +356,21 @@ class Deep(object): try: signal.alarm(int(math.ceil(maxtime))) stime = time.time() - f = url_open(url) + f = url_open(test_url) - if hashlib.md5(f.read()).hexdigest() != "bdf077b2e683c506bf9e8f2494eeb044": - return (None, True) + md5 = hashlib.md5(f.read()).hexdigest() delta = time.time() - stime f.close() + if md5 != self.test_md5: + self.output.write( + "deeptime(): md5sum error for file: %s\n" % self.test_file + + " expected: %s\n" % self.test_md5 + + " got.....: %s\n" % md5 + + " host....: %s, %s\n" % (url_parts.hostname, ip)) + self.dl_failures += 1 + return (None, True) + finally: signal.alarm(0)