From: "Brian Dolbec" <brian.dolbec@gmail.com> To: gentoo-commits@lists.gentoo.org Subject: [gentoo-commits] proj/mirrorselect:master commit in: mirrorselect/ Date: Sun, 2 Mar 2014 07:44:08 +0000 (UTC) [thread overview] Message-ID: <1390517588.a3b025b7f3d5e67b1735bb773b72b5766537225f.dol-sen@gentoo> (raw) commit: a3b025b7f3d5e67b1735bb773b72b5766537225f Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> AuthorDate: Thu Jan 23 22:53:08 2014 +0000 Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> CommitDate: Thu Jan 23 22:53:08 2014 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a3b025b7 Remove connections.py. Instead moved it to it's own ssl-fetch pkg. --- mirrorselect/connections.py | 182 -------------------------------------------- 1 file changed, 182 deletions(-) diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py deleted file mode 100644 index ca4aa88..0000000 --- a/mirrorselect/connections.py +++ /dev/null @@ -1,182 +0,0 @@ -#-*- coding:utf-8 -*- - -"""Mirrorselect 2.x - Tool for selecting Gentoo source and rsync mirrors. - -Copyright 2005-2012 Gentoo Foundation - - Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org> - Copyright (C) 2008 Zac Medico <zmedico@gentoo.org> - Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org> - Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org> - Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org> - -Distributed under the terms of the GNU General Public License v2 - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. - -""" - -import sys -import os - -VERIFY_SSL = False -VERIFY_MSGS = [] - -import requests -from requests.exceptions import SSLError - -# py3.2 -if sys.hexversion >= 0x30200f0: - VERIFY_SSL = True -else: - try: # import and enable SNI support for py2 - from requests.packages.urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() - VERIFY_SSL = True - VERIFY_MSGS = ["Successfully enabled ssl certificate verification."] - except ImportError as e: - VERIFY_MSGS = [ - "Failed to import and inject pyopenssl/SNI support into urllib3", - "Disabling certificate verification", - "Error was:" + e - ] - VERIFY_SSL = False - - -from mirrorselect.version import version - - -class Connector(object): - """Primary connection interface using the dev-python/requests package - """ - - def __init__(self, output, proxies): - self.output = output - self.proxies = proxies - self.headers = {'Accept-Charset': 'utf-8', - 'User-Agent': 'Mirrorselect-' + version} - - if VERIFY_MSGS: - for msg in VERIFY_MSGS: - self.output.write(msg + '\n', 2) - - - def add_timestamp(self, headers, tpath=None, timestamp=None): - """for possilble future caching of the list""" - if tpath and os.path.exists(tpath): - # fileopen is a layman comaptibility function not yet implemented here - with fileopen(tpath,'r') as previous: - timestamp = previous.read() - if timestamp: - headers['If-Modified-Since'] = timestamp - self.output.write('Current-modified: %s\n' % timestamp, 2) - return headers - - - def fetch_url(self, url, headers=None, timestamp=None): - """Fetches the url - - @param url: string - @param headers: dictionary, optional headers to use - @param tpath: string, optional filepath to a timestamp file - to use in the headers - @param timestamp: string, optional timestamp to use in the headers - - """ - - if not headers: - headers = self.headers - - if timestamp: - self.add_timestamp(headers, timestamp=timestamp) - - verify = 'https' in url and VERIFY_SSL - self.output.write("Enabled ssl certificate verification: %s, for: %s\n" - %(str(verify), url), 3) - - self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4) - self.output.write('Connector.fetch_url(); connecting to opener\n', 2) - - try: - connection = requests.get( - url, - headers=headers, - verify=verify, - proxies=self.proxies, - ) - except SSLError as error: - self.output.print_err('Connector.fetch_url(); Failed to update the ' - 'mirror list from: %s\nSSLError was:%s\n' - % (url, str(error))) - except Exception as error: - self.output.print_err('Connector.fetch_url(); Failed to retrieve ' - 'the content from: %s\nError was: %s\n' - % (url, str(error))) - - self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4) - self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2) - return connection - - - @staticmethod - def normalize_headers(headers, to_lower=True): - """ py2, py3 compatibility function, since only py2 returns keys as lower() - """ - if to_lower: - return dict((x.lower(), x) for x in list(headers)) - return dict((x.upper(), x) for x in list(headers)) - - - def fetch_content(self, url, tpath=None): - """Fetch the mirror list - - @param url: string of the content to fetch - @param headers: dictionary, optional headers to use - @param tpath: string, optional filepath to a timestamp file - to use in the headers - @returns (success bool, content fetched , timestamp of fetched content, - content headers returned) - """ - - fheaders = self.headers - - if tpath: - fheaders = self.add_timestamp(fheaders, tpath) - - connection = self.fetch_url(url, fheaders) - - headers = self.normalize_headers(connection.headers) - - if 'last-modified' in headers: - timestamp = headers['last-modified'] - elif 'date' in headers: - timestamp = headers['date'] - else: - timestamp = None - - if connection.status_code in [304]: - self.output.write('Content already up to date: %s\n' - % url, 4) - self.output.write('Last-modified: %s\n' % timestamp, 4) - elif connection.status_code not in [200]: - self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n' - 'url: %s\n%s' - % (url, str(connection.status_code))) - - if connection.status_code in [200]: - self.output.write('New content downloaded for: %s\n' - % url, 4) - return (True, connection.content, timestamp) - return (False, '', '') -
WARNING: multiple messages have this Message-ID (diff)
From: "Brian Dolbec" <brian.dolbec@gmail.com> To: gentoo-commits@lists.gentoo.org Subject: [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/ Date: Fri, 31 Jan 2014 15:44:51 +0000 (UTC) [thread overview] Message-ID: <1390517588.a3b025b7f3d5e67b1735bb773b72b5766537225f.dol-sen@gentoo> (raw) Message-ID: <20140131154451.I7kbd1YCtBgB-NOfLiHtTkyZhxJgtkz8UAbNvE-PMsY@z> (raw) commit: a3b025b7f3d5e67b1735bb773b72b5766537225f Author: Brian Dolbec <dolsen <AT> gentoo <DOT> org> AuthorDate: Thu Jan 23 22:53:08 2014 +0000 Commit: Brian Dolbec <brian.dolbec <AT> gmail <DOT> com> CommitDate: Thu Jan 23 22:53:08 2014 +0000 URL: http://git.overlays.gentoo.org/gitweb/?p=proj/mirrorselect.git;a=commit;h=a3b025b7 Remove connections.py. Instead moved it to it's own ssl-fetch pkg. --- mirrorselect/connections.py | 182 -------------------------------------------- 1 file changed, 182 deletions(-) diff --git a/mirrorselect/connections.py b/mirrorselect/connections.py deleted file mode 100644 index ca4aa88..0000000 --- a/mirrorselect/connections.py +++ /dev/null @@ -1,182 +0,0 @@ -#-*- coding:utf-8 -*- - -"""Mirrorselect 2.x - Tool for selecting Gentoo source and rsync mirrors. - -Copyright 2005-2012 Gentoo Foundation - - Copyright (C) 2005 Colin Kingsley <tercel@gentoo.org> - Copyright (C) 2008 Zac Medico <zmedico@gentoo.org> - Copyright (C) 2009 Sebastian Pipping <sebastian@pipping.org> - Copyright (C) 2009 Christian Ruppert <idl0r@gentoo.org> - Copyright (C) 2012 Brian Dolbec <dolsen@gentoo.org> - -Distributed under the terms of the GNU General Public License v2 - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. - -""" - -import sys -import os - -VERIFY_SSL = False -VERIFY_MSGS = [] - -import requests -from requests.exceptions import SSLError - -# py3.2 -if sys.hexversion >= 0x30200f0: - VERIFY_SSL = True -else: - try: # import and enable SNI support for py2 - from requests.packages.urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() - VERIFY_SSL = True - VERIFY_MSGS = ["Successfully enabled ssl certificate verification."] - except ImportError as e: - VERIFY_MSGS = [ - "Failed to import and inject pyopenssl/SNI support into urllib3", - "Disabling certificate verification", - "Error was:" + e - ] - VERIFY_SSL = False - - -from mirrorselect.version import version - - -class Connector(object): - """Primary connection interface using the dev-python/requests package - """ - - def __init__(self, output, proxies): - self.output = output - self.proxies = proxies - self.headers = {'Accept-Charset': 'utf-8', - 'User-Agent': 'Mirrorselect-' + version} - - if VERIFY_MSGS: - for msg in VERIFY_MSGS: - self.output.write(msg + '\n', 2) - - - def add_timestamp(self, headers, tpath=None, timestamp=None): - """for possilble future caching of the list""" - if tpath and os.path.exists(tpath): - # fileopen is a layman comaptibility function not yet implemented here - with fileopen(tpath,'r') as previous: - timestamp = previous.read() - if timestamp: - headers['If-Modified-Since'] = timestamp - self.output.write('Current-modified: %s\n' % timestamp, 2) - return headers - - - def fetch_url(self, url, headers=None, timestamp=None): - """Fetches the url - - @param url: string - @param headers: dictionary, optional headers to use - @param tpath: string, optional filepath to a timestamp file - to use in the headers - @param timestamp: string, optional timestamp to use in the headers - - """ - - if not headers: - headers = self.headers - - if timestamp: - self.add_timestamp(headers, timestamp=timestamp) - - verify = 'https' in url and VERIFY_SSL - self.output.write("Enabled ssl certificate verification: %s, for: %s\n" - %(str(verify), url), 3) - - self.output.write('Connector.fetch_url(); headers = %s\n' %str(headers), 4) - self.output.write('Connector.fetch_url(); connecting to opener\n', 2) - - try: - connection = requests.get( - url, - headers=headers, - verify=verify, - proxies=self.proxies, - ) - except SSLError as error: - self.output.print_err('Connector.fetch_url(); Failed to update the ' - 'mirror list from: %s\nSSLError was:%s\n' - % (url, str(error))) - except Exception as error: - self.output.print_err('Connector.fetch_url(); Failed to retrieve ' - 'the content from: %s\nError was: %s\n' - % (url, str(error))) - - self.output.write('Connector.fetch_url() HEADERS = %s\n' %str(connection.headers), 4) - self.output.write('Connector.fetch_url() Status_code = %i\n' % connection.status_code, 2) - return connection - - - @staticmethod - def normalize_headers(headers, to_lower=True): - """ py2, py3 compatibility function, since only py2 returns keys as lower() - """ - if to_lower: - return dict((x.lower(), x) for x in list(headers)) - return dict((x.upper(), x) for x in list(headers)) - - - def fetch_content(self, url, tpath=None): - """Fetch the mirror list - - @param url: string of the content to fetch - @param headers: dictionary, optional headers to use - @param tpath: string, optional filepath to a timestamp file - to use in the headers - @returns (success bool, content fetched , timestamp of fetched content, - content headers returned) - """ - - fheaders = self.headers - - if tpath: - fheaders = self.add_timestamp(fheaders, tpath) - - connection = self.fetch_url(url, fheaders) - - headers = self.normalize_headers(connection.headers) - - if 'last-modified' in headers: - timestamp = headers['last-modified'] - elif 'date' in headers: - timestamp = headers['date'] - else: - timestamp = None - - if connection.status_code in [304]: - self.output.write('Content already up to date: %s\n' - % url, 4) - self.output.write('Last-modified: %s\n' % timestamp, 4) - elif connection.status_code not in [200]: - self.output.print_err('Connector.fetch_content(); HTTP Status-Code was:\n' - 'url: %s\n%s' - % (url, str(connection.status_code))) - - if connection.status_code in [200]: - self.output.write('New content downloaded for: %s\n' - % url, 4) - return (True, connection.content, timestamp) - return (False, '', '') -
next reply other threads:[~2014-03-02 7:44 UTC|newest] Thread overview: 87+ messages / expand[flat|nested] mbox.gz Atom feed top 2014-03-02 7:44 Brian Dolbec [this message] 2014-01-31 15:44 ` [gentoo-commits] proj/mirrorselect:ssl commit in: mirrorselect/ Brian Dolbec -- strict thread matches above, loose matches on Subject: below -- 2024-05-28 3:48 [gentoo-commits] proj/mirrorselect:master " Sam James 2023-08-07 0:23 Sam James 2023-08-07 0:14 Sam James 2023-08-07 0:14 Sam James 2023-08-07 0:14 Sam James 2023-08-06 23:30 Sam James 2023-08-06 23:30 Sam James 2023-07-06 8:05 Sam James 2022-05-31 18:39 Brian Dolbec 2022-05-31 18:39 Brian Dolbec 2022-05-31 18:39 Brian Dolbec 2022-05-31 18:39 Brian Dolbec 2022-05-31 4:08 Brian Dolbec 2022-05-31 4:08 Brian Dolbec 2022-05-31 2:22 Brian Dolbec 2022-05-31 2:22 Brian Dolbec 2022-05-30 23:12 Brian Dolbec 2022-05-30 23:12 Brian Dolbec 2020-06-03 19:05 Brian Dolbec 2019-07-17 5:06 Zac Medico 2019-05-27 17:52 Zac Medico 2019-05-27 17:22 Zac Medico 2019-02-13 8:22 Zac Medico 2019-02-13 8:09 Zac Medico 2019-02-13 5:51 Zac Medico 2018-05-26 15:43 Brian Dolbec 2017-02-21 4:44 Zac Medico 2017-02-21 3:19 Brian Dolbec 2017-02-21 3:19 Brian Dolbec 2016-11-15 1:16 Zac Medico 2015-01-27 18:20 Brian Dolbec 2015-01-27 4:38 Brian Dolbec 2015-01-27 4:38 Brian Dolbec 2014-05-28 21:08 Brian Dolbec 2014-05-28 19:44 Brian Dolbec 2014-05-05 2:04 Brian Dolbec 2014-05-05 2:04 Brian Dolbec 2014-05-05 2:04 Brian Dolbec 2014-03-02 7:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec 2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-03-02 7:44 Brian Dolbec 2014-01-31 15:44 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec 2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec 2013-10-20 18:19 [gentoo-commits] proj/mirrorselect:ssl " Brian Dolbec 2014-03-02 7:44 ` [gentoo-commits] proj/mirrorselect:master " Brian Dolbec 2013-10-19 9:06 Brian Dolbec 2013-10-19 9:06 Brian Dolbec 2013-10-19 9:06 Brian Dolbec 2013-10-18 14:26 Brian Dolbec 2013-10-18 6:46 Brian Dolbec 2013-10-18 6:46 Brian Dolbec 2013-10-18 6:46 Brian Dolbec 2013-10-18 6:46 Brian Dolbec 2013-10-17 14:26 Brian Dolbec 2013-10-17 6:57 Brian Dolbec 2013-10-17 3:16 Brian Dolbec 2013-10-17 3:16 Brian Dolbec 2013-10-17 3:16 Brian Dolbec 2013-10-16 9:17 Brian Dolbec 2013-10-16 8:36 Brian Dolbec 2013-10-16 8:36 Brian Dolbec 2013-10-16 8:36 Brian Dolbec 2013-10-16 8:36 Brian Dolbec 2013-10-15 22:43 Brian Dolbec 2013-10-15 14:39 Brian Dolbec 2013-10-15 14:39 Brian Dolbec 2013-10-15 14:39 Brian Dolbec 2013-10-15 7:27 Brian Dolbec 2013-10-15 3:51 Brian Dolbec 2013-10-15 3:51 Brian Dolbec 2013-03-10 13:07 Brian Dolbec 2012-12-16 2:38 Brian Dolbec 2012-12-15 21:25 Brian Dolbec 2012-11-15 3:53 Brian Dolbec 2012-11-15 3:44 Brian Dolbec 2012-11-14 20:28 Paul Varner 2012-11-12 21:41 Brian Dolbec 2012-11-12 20:37 Brian Dolbec 2012-11-12 15:56 Brian Dolbec 2012-11-12 7:46 Brian Dolbec 2012-11-12 7:46 Brian Dolbec 2012-11-12 7:46 Brian Dolbec 2012-11-12 7:46 Brian Dolbec
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1390517588.a3b025b7f3d5e67b1735bb773b72b5766537225f.dol-sen@gentoo \ --to=brian.dolbec@gmail.com \ --cc=gentoo-commits@lists.gentoo.org \ --cc=gentoo-dev@lists.gentoo.org \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions for how to clone and mirror all data and code used for this inbox