summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2012-04-07 14:39:01 -0500
committerDan McGee <dan@archlinux.org>2012-04-07 14:54:52 -0500
commitb5ab5b1e218219b09857b06f88e522bccb4b5600 (patch)
treef13a9acdad6c46ccf314f2f36870220bd5caa821
parent1a2f117037fd8b01ec1e1e3cce5186d7bfac1a78 (diff)
downloadarchweb-b5ab5b1e218219b09857b06f88e522bccb4b5600.tar.gz
archweb-b5ab5b1e218219b09857b06f88e522bccb4b5600.zip
Choose an up-to-date mirror for download URLs
Given that we collect a lot of mirror status data, we can utilize it to ensure the download link on the website actually works and newly-added packages have actually been mirrored out. Add a method that attempts to use the mirror status data to determine a mirror we should redirect our download requests to. This can change on a regular basis, and falls back to the old method if no mirror status data is available. Signed-off-by: Dan McGee <dan@archlinux.org>
-rw-r--r--mirrors/utils.py34
-rw-r--r--packages/views/__init__.py13
2 files changed, 38 insertions, 9 deletions
diff --git a/mirrors/utils.py b/mirrors/utils.py
index 0f8fef84..619d5f5c 100644
--- a/mirrors/utils.py
+++ b/mirrors/utils.py
@@ -27,6 +27,7 @@ def annotate_url(url, delays):
url.delay = None
url.score = None
+
@cache_function(123)
def get_mirror_statuses(cutoff=default_cutoff):
cutoff_time = utc_now() - cutoff
@@ -81,6 +82,7 @@ def get_mirror_statuses(cutoff=default_cutoff):
'urls': urls,
}
+
@cache_function(117)
def get_mirror_errors(cutoff=default_cutoff):
cutoff_time = utc_now() - cutoff
@@ -96,4 +98,36 @@ def get_mirror_errors(cutoff=default_cutoff):
err['country'] = err['url__country'] or err['url__mirror__country']
return errors
+
+@cache_function(295)
+def get_mirror_url_for_download(cutoff=default_cutoff):
+ '''Find a good mirror URL to use for package downloads. If we have mirror
+ status data available, it is used to determine a good choice by looking at
+ the last batch of status rows.'''
+ cutoff_time = utc_now() - cutoff
+ status_data = MirrorLog.objects.filter(
+ check_time__gte=cutoff_time).aggregate(
+ Max('check_time'), Max('last_sync'))
+ if status_data:
+ min_check_time = status_data['check_time__max'] - timedelta(minutes=5)
+ min_sync_time = status_data['last_sync__max'] - timedelta(minutes=30)
+ best_logs = MirrorLog.objects.filter(is_success=True,
+ check_time__gte=min_check_time, last_sync__gte=min_sync_time,
+ url__mirror__public=True, url__mirror__active=True,
+ url__protocol__protocol__iexact='HTTP').order_by(
+ 'duration')[:1]
+ if best_logs:
+ return MirrorUrl.objects.get(id=best_logs[0].url_id)
+
+ mirror_urls = MirrorUrl.objects.filter(
+ mirror__public=True, mirror__active=True,
+ protocol__protocol__iexact='HTTP')
+ # look first for an 'Any' URL, then fall back to any HTTP URL
+ filtered_urls = mirror_urls.filter(mirror__country='Any')[:1]
+ if not filtered_urls:
+ filtered_urls = mirror_urls[:1]
+ if not filtered_urls:
+ return None
+ return filtered_urls[0]
+
# vim: set ts=4 sw=4 et:
diff --git a/packages/views/__init__.py b/packages/views/__init__.py
index 2798fff1..abc86d78 100644
--- a/packages/views/__init__.py
+++ b/packages/views/__init__.py
@@ -14,6 +14,7 @@ from django.views.generic.simple import direct_to_template
from main.models import Package, PackageFile, PackageDepend, Arch, Repo
from mirrors.models import MirrorUrl
+from mirrors.utils import get_mirror_url_for_download
from ..models import (PackageRelation, PackageGroup, License,
Conflict, Provision, Replacement)
from ..utils import (get_group_info, get_differences_info,
@@ -225,21 +226,15 @@ def files_json(request, name, repo, arch):
def download(request, name, repo, arch):
pkg = get_object_or_404(Package,
pkgname=name, repo__name__iexact=repo, arch__name=arch)
- mirror_urls = MirrorUrl.objects.filter(
- mirror__public=True, mirror__active=True,
- protocol__protocol__iexact='HTTP')
- # look first for an 'Any' URL, then fall back to any HTTP URL
- filtered_urls = mirror_urls.filter(mirror__country='Any')[:1]
- if not filtered_urls:
- filtered_urls = mirror_urls[:1]
- if not filtered_urls:
+ url = get_mirror_url_for_download()
+ if not url:
raise Http404
arch = pkg.arch.name
if pkg.arch.agnostic:
# grab the first non-any arch to fake the download path
arch = Arch.objects.exclude(agnostic=True)[0].name
values = {
- 'host': filtered_urls[0].url,
+ 'host': url.url,
'arch': arch,
'repo': pkg.repo.name.lower(),
'file': pkg.filename,