Skip to content

Commit

Permalink
Internal refactoring: Distinguish 404 from other errors*
Browse files Browse the repository at this point in the history 
This change set builds on top of edcf561 to enhance validate_mirror()
by making a distinction between a confirmed HTTP 404 response versus
other error conditions which may be of a more transient nature.

The goal of this change is to preserve the semantics requested in
issue #1 and implemented in pull request #2 without needing the
additional HTTP request performed by can_connect_to_mirror().

Because validate_mirror() previously returned a boolean but now returns
an enumeration member this change is technically backwards incompatible,
then again validate_mirror() isn't specifically intended for callers
because it concerns internal logic of apt-mirror-updater. When I publish
this I will nevertheless bump the major version number.
  • Loading branch information
@xolox
xolox committed Nov 1, 2017
1 parent edcf561 commit c2bf8fd
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 63 deletions.
110 changes: 51 additions & 59 deletions apt_mirror_updater/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# Automated, robust apt-get mirror selection for Debian and Ubuntu.
#
# Author: Peter Odding <peter@peterodding.com>
# Last Change: October 31, 2017
# Last Change: November 1, 2017
# URL: https://apt-mirror-updater.readthedocs.io

"""
Expand All @@ -19,12 +19,12 @@
import os
import sys
import time

# Python 2.x / 3.x compatibility.
try:
# For Python 3.0 and later
from urllib.error import URLError
from enum import Enum
except ImportError:
# Fall back to Python 2's urllib2
from urllib2 import URLError
from flufl.enum import Enum

# External dependencies.
from capturer import CaptureOutput
Expand All @@ -42,7 +42,7 @@
from six.moves.urllib.parse import urlparse

# Modules included in our package.
from apt_mirror_updater.http import fetch_concurrent, fetch_url, get_default_concurrency
from apt_mirror_updater.http import NotFoundError, fetch_concurrent, fetch_url, get_default_concurrency

# Semi-standard module versioning.
__version__ = '4.0'
Expand Down Expand Up @@ -269,9 +269,7 @@ def release_is_eol(self):
logger.debug("Checking whether %s suite %s is EOL ..",
self.distributor_id.capitalize(),
self.distribution_codename.capitalize())
# only set EOL if we can connect but validation fails
# this prevents incorrect setting of EOL if network connection fails
release_is_eol = self.can_connect_to_mirror(self.security_url) and not self.validate_mirror(self.security_url)
release_is_eol = (self.validate_mirror(self.security_url) == MirrorStatus.MAYBE_EOL)
logger.debug("The %s suite %s is %s.",
self.distributor_id.capitalize(),
self.distribution_codename.capitalize(),
Expand Down Expand Up @@ -569,30 +567,23 @@ def smart_update(self, max_attempts=10, switch_mirrors=True):
output = session.get_text()
# Check for EOL releases. This somewhat peculiar way of
# checking is meant to ignore 404 responses from
# `secondary package mirrors' like PPAs.
maybe_end_of_life = any(
self.current_mirror in line and u'404' in line.split()
for line in output.splitlines()
)
# If the output of `apt-get update' implies that the
# release is EOL we need to verify our assumption.
if maybe_end_of_life:
logger.warning("It looks like the current release (%s) is EOL, verifying ..",
self.distribution_codename)
if not self.validate_mirror(self.current_mirror):
# `secondary package mirrors' like PPAs. If the output
# of `apt-get update' implies that the release is EOL
# we need to verify our assumption.
if any(self.current_mirror in line and u'404' in line.split() for line in output.splitlines()):
logger.warning("Current release (%s) may be EOL, checking ..", self.distribution_codename)
if self.release_is_eol:
if switch_mirrors:
logger.warning("Switching to old releases mirror because current release is EOL ..")
logger.warning("Switching to old releases mirror (release appears to be EOL) ..")
self.change_mirror(self.old_releases_url, update=False)
continue
else:
# When asked to do the impossible we abort
# with a clear error message :-).
raise Exception(compact("""
Failed to update package lists because the
current release ({release}) is end of life but
I'm not allowed to switch mirrors! (there's
no point in retrying so I'm not going to)
""", release=self.distribution_codename))
Failed to update package lists because it looks like
the current release (%s) is end of life but I'm not
allowed to switch mirrors! (there's no point in
retrying so I'm not going to)
""", self.distribution_codename))
# Check for `hash sum mismatch' errors.
if switch_mirrors and u'hash sum mismatch' in output.lower():
logger.warning("Detected 'hash sum mismatch' failure, switching to other mirror ..")
Expand All @@ -610,41 +601,17 @@ def smart_update(self, max_attempts=10, switch_mirrors=True):
backoff_time += backoff_time / 3
raise Exception("Failed to update package lists %i consecutive times?!" % max_attempts)

def can_connect_to_mirror(self, mirror_url):
"""
Make sure the mirror can be connected to
:param mirror_url: The base URL of the mirror (a string).
:returns: :data:`True` if the mirror can be connected to,
:data:`False` otherwise.
"""
mirror_url = normalize_mirror_url(mirror_url)
logger.info("Checking whether %s can be connected to.", mirror_url)
mirror = CandidateMirror(mirror_url=mirror_url, updater=self)
try:
response = fetch_url(mirror.release_gpg_url, retry=False)
mirror.release_gpg_contents = response.read()
except URLError as e:
if 'connection refused' in str(e).lower() or 'name or service not known' in str(e).lower():
logger.warning("Cannot connect to %s.", mirror_url)
return False
except Exception:
pass
logger.info("Can connect to %s.", mirror_url)
return True

def validate_mirror(self, mirror_url):
"""
Make sure a mirror serves :attr:`distribution_codename`.
:param mirror_url: The base URL of the mirror (a string).
:returns: :data:`True` if the mirror hosts the relevant release,
:data:`False` otherwise.
:returns: One of the values in the :class:`MirrorStatus` enumeration.
This method assumes that :attr:`old_releases_url` is always valid.
"""
if mirrors_are_equal(mirror_url, self.old_releases_url):
return True
return MirrorStatus.AVAILABLE
else:
mirror_url = normalize_mirror_url(mirror_url)
key = (mirror_url, self.distribution_codename)
Expand All @@ -653,13 +620,24 @@ def validate_mirror(self, mirror_url):
logger.info("Checking whether %s is a supported release for %s ..",
self.distribution_codename.capitalize(),
self.distributor_id.capitalize())
mirror = CandidateMirror(mirror_url=mirror_url, updater=self)
# Try to download the Release.gpg file, in the assumption that
# this file should always exist and is more or less guaranteed
# to be relatively small.
try:
data = fetch_url(mirror.release_gpg_url, retry=False)
mirror.release_gpg_contents = data
mirror = CandidateMirror(mirror_url=mirror_url, updater=self)
mirror.release_gpg_contents = fetch_url(mirror.release_gpg_url, retry=False)
value = (MirrorStatus.AVAILABLE if mirror.is_available else MirrorStatus.UNAVAILABLE)
except NotFoundError:
# When the mirror is serving 404 responses it can be an
# indication that the release has gone end of life. In any
# case the mirror is unavailable.
value = MirrorStatus.MAYBE_EOL
except Exception:
pass
self.validated_mirrors[key] = value = mirror.is_available
# When we get an unspecified error that is not a 404
# response we conclude that the mirror is unavailable.
value = MirrorStatus.UNAVAILABLE
# Cache the mirror status that we just determined.
self.validated_mirrors[key] = value
return value


Expand Down Expand Up @@ -799,6 +777,20 @@ def updater(self):
"""A reference to the :class:`AptMirrorUpdater` object that created the candidate."""


class MirrorStatus(Enum):

"""Enumeration for mirror statuses determined by :func:`AptMirrorUpdater.validate_mirror()`."""

AVAILABLE = 1
"""The mirror is accepting connections and serving the expected content."""

MAYBE_EOL = 2
"""The mirror is serving HTTP 404 "Not Found" responses instead of the expected content."""

UNAVAILABLE = 2
"""The mirror is not accepting connections or not serving the expected content."""


def find_current_mirror(sources_list):
"""
Find the URL of the main mirror that is currently in use by ``apt-get``.
Expand Down
65 changes: 61 additions & 4 deletions setup.py
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,29 @@
#!/usr/bin/env python

"""Setup script for the `apt-mirror-updater` package."""

# Setup script for the `apt-mirror-updater' package.
#
# Author: Peter Odding <peter@peterodding.com>
# Last Change: June 8, 2017
# Last Change: November 1, 2017
# URL: https://apt-mirror-updater.readthedocs.io

"""
Setup script for the `apt-mirror-updater` package.
**python setup.py install**
Install from the working directory into the current Python environment.
**python setup.py sdist**
Build a source distribution archive.
**python setup.py bdist_wheel**
Build a wheel distribution archive.
"""

# Standard library modules.
import codecs
import os
import re
import sys

# De-facto standard solution for Python packaging.
from setuptools import find_packages, setup
Expand All @@ -28,6 +42,31 @@ def get_version(*args):
return metadata['version']


def get_install_requires():
"""Get the conditional dependencies for source distributions."""
install_requires = get_requirements('requirements.txt')
if 'bdist_wheel' not in sys.argv:
if sys.version_info[:2] < (3, 4):
install_requires.append('flufl.enum >= 4.0.1')
return sorted(install_requires)


def get_extras_require():
"""Get the conditional dependencies for wheel distributions."""
extras_require = {}
if have_environment_marker_support():
expression = ':%s' % ' or '.join([
'python_version == "2.6"',
'python_version == "2.7"',
'python_version == "3.0"',
'python_version == "3.1"',
'python_version == "3.2"',
'python_version == "3.3"',
])
extras_require[expression] = ['flufl.enum >= 4.0.1']
return extras_require


def get_requirements(*args):
"""Get requirements from pip requirement files."""
requirements = set()
Expand All @@ -46,6 +85,21 @@ def get_absolute_path(*args):
return os.path.join(os.path.dirname(os.path.abspath(__file__)), *args)


def have_environment_marker_support():
"""
Check whether setuptools has support for PEP-426 environment marker support.
Based on the ``setup.py`` script of the ``pytest`` package:
https://bitbucket.org/pytest-dev/pytest/src/default/setup.py
"""
try:
from pkg_resources import parse_version
from setuptools import __version__
return parse_version(__version__) >= parse_version('0.7.2')
except Exception:
return False


setup(
name='apt-mirror-updater',
version=get_version('apt_mirror_updater', '__init__.py'),
Expand All @@ -55,17 +109,20 @@ def get_absolute_path(*args):
author='Peter Odding',
author_email='peter@peterodding.com',
packages=find_packages(),
install_requires=get_requirements('requirements.txt'),
install_requires=get_install_requires(),
extras_require=get_extras_require(),
entry_points=dict(console_scripts=[
'apt-mirror-updater = apt_mirror_updater.cli:main',
]),
classifiers=[
'Development Status :: 4 - Beta',
'Environment :: Console',
'Intended Audience :: Developers',
'Intended Audience :: Information Technology',
'Intended Audience :: System Administrators',
'License :: OSI Approved :: MIT License',
'Natural Language :: English',
'Operating System :: POSIX :: Linux',
'Programming Language :: Python',
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.6',
Expand Down

0 comments on commit c2bf8fd

Please sign in to comment.