summaryrefslogtreecommitdiffstats
path: root/mirrors/views
diff options
context:
space:
mode:
authorDan McGee <dan@archlinux.org>2015-02-26 19:16:44 -0600
committerDan McGee <dan@archlinux.org>2015-02-26 19:16:44 -0600
commit17b6ce186c6e793417c73e955bfc01b4f4b96864 (patch)
tree482f3a280de28f22e6b042566c8136fda7548044 /mirrors/views
parent1c0f41ff1a150f72aa323f1f3ebc8179e3ec9ac6 (diff)
downloadarchweb-17b6ce186c6e793417c73e955bfc01b4f4b96864.tar.gz
archweb-17b6ce186c6e793417c73e955bfc01b4f4b96864.zip
Move mirrors views into subdirectory
We'll start splitting these up as we did in packages/. Signed-off-by: Dan McGee <dan@archlinux.org>
Diffstat (limited to 'mirrors/views')
-rw-r--r--mirrors/views/__init__.py368
1 files changed, 368 insertions, 0 deletions
diff --git a/mirrors/views/__init__.py b/mirrors/views/__init__.py
new file mode 100644
index 00000000..54c427b3
--- /dev/null
+++ b/mirrors/views/__init__.py
@@ -0,0 +1,368 @@
+from datetime import timedelta
+from itertools import groupby
+import json
+from operator import attrgetter, itemgetter
+
+from django import forms
+from django.forms.widgets import SelectMultiple, CheckboxSelectMultiple
+from django.core.serializers.json import DjangoJSONEncoder
+from django.db import connection
+from django.db.models import Q
+from django.http import Http404, HttpResponse
+from django.shortcuts import get_object_or_404, redirect, render
+from django.utils.timezone import now
+from django.views.decorators.cache import cache_page
+from django.views.decorators.csrf import csrf_exempt
+from django.views.decorators.http import condition
+from django_countries import countries
+from django_countries.fields import Country
+
+from ..models import (Mirror, MirrorUrl, MirrorProtocol, MirrorLog,
+ CheckLocation)
+from ..utils import get_mirror_statuses, get_mirror_errors, DEFAULT_CUTOFF
+
+
+class MirrorlistForm(forms.Form):
+ country = forms.MultipleChoiceField(required=False,
+ widget=SelectMultiple(attrs={'size': '12'}))
+ protocol = forms.MultipleChoiceField(required=False,
+ widget=CheckboxSelectMultiple)
+ ip_version = forms.MultipleChoiceField(required=False,
+ label="IP version", choices=(('4','IPv4'), ('6','IPv6')),
+ widget=CheckboxSelectMultiple)
+ use_mirror_status = forms.BooleanField(required=False)
+
+ def __init__(self, *args, **kwargs):
+ super(MirrorlistForm, self).__init__(*args, **kwargs)
+ fields = self.fields
+ fields['country'].choices = [('all','All')] + self.get_countries()
+ fields['country'].initial = ['all']
+ protos = [(p.protocol, p.protocol) for p in
+ MirrorProtocol.objects.filter(is_download=True)]
+ initial = MirrorProtocol.objects.filter(is_download=True, default=True)
+ fields['protocol'].choices = protos
+ fields['protocol'].initial = [p.protocol for p in initial]
+ fields['ip_version'].initial = ['4']
+
+ def get_countries(self):
+ country_codes = set()
+ country_codes.update(MirrorUrl.objects.filter(active=True,
+ mirror__active=True).exclude(country='').values_list(
+ 'country', flat=True).order_by().distinct())
+ code_list = [(code, countries.name(code)) for code in country_codes]
+ return sorted(code_list, key=itemgetter(1))
+
+ def as_div(self):
+ "Returns this form rendered as HTML <divs>s."
+ return self._html_output(
+ normal_row = u'<div%(html_class_attr)s>%(label)s %(field)s%(help_text)s</div>',
+ error_row = u'%s',
+ row_ender = '</div>',
+ help_text_html = u' <span class="helptext">%s</span>',
+ errors_on_separate_row = True)
+
+
+@csrf_exempt
+def generate_mirrorlist(request):
+ if request.method == 'POST' or len(request.GET) > 0:
+ form = MirrorlistForm(data=request.REQUEST)
+ if form.is_valid():
+ countries = form.cleaned_data['country']
+ protocols = form.cleaned_data['protocol']
+ use_status = form.cleaned_data['use_mirror_status']
+ ipv4 = '4' in form.cleaned_data['ip_version']
+ ipv6 = '6' in form.cleaned_data['ip_version']
+ return find_mirrors(request, countries, protocols,
+ use_status, ipv4, ipv6)
+ else:
+ form = MirrorlistForm()
+
+ return render(request, 'mirrors/mirrorlist_generate.html',
+ {'mirrorlist_form': form})
+
+
+def status_filter(original_urls):
+ status_info = get_mirror_statuses()
+ scores = {u.id: u.score for u in status_info['urls']}
+ urls = []
+ for u in original_urls:
+ u.score = scores.get(u.id, None)
+ # also include mirrors that don't have an up to date score
+ # (as opposed to those that have been set with no score)
+ if (u.id not in scores) or (u.score and u.score < 100.0):
+ urls.append(u)
+ # if a url doesn't have a score, treat it as the highest possible
+ return sorted(urls, key=lambda x: x.score or 100.0)
+
+
+def find_mirrors(request, countries=None, protocols=None, use_status=False,
+ ipv4_supported=True, ipv6_supported=True):
+ if not protocols:
+ protocols = MirrorProtocol.objects.filter(is_download=True)
+ elif hasattr(protocols, 'model') and protocols.model == MirrorProtocol:
+ # we already have a queryset, no need to query again
+ pass
+ else:
+ protocols = MirrorProtocol.objects.filter(protocol__in=protocols)
+ qset = MirrorUrl.objects.select_related().filter(
+ protocol__in=protocols, active=True,
+ mirror__public=True, mirror__active=True)
+ if countries and 'all' not in countries:
+ qset = qset.filter(country__in=countries)
+
+ ip_version = Q()
+ if ipv4_supported:
+ ip_version |= Q(has_ipv4=True)
+ if ipv6_supported:
+ ip_version |= Q(has_ipv6=True)
+ qset = qset.filter(ip_version)
+
+ if not use_status:
+ sort_key = attrgetter('country.name', 'mirror.name', 'url')
+ urls = sorted(qset, key=sort_key)
+ template = 'mirrors/mirrorlist.txt'
+ else:
+ urls = status_filter(qset)
+ template = 'mirrors/mirrorlist_status.txt'
+
+ context = {
+ 'mirror_urls': urls,
+ }
+ return render(request, template, context, content_type='text/plain')
+
+
+def find_mirrors_simple(request, protocol):
+ if protocol == 'smart':
+ return redirect('mirrorlist_simple', 'http', permanent=True)
+ proto = get_object_or_404(MirrorProtocol, protocol=protocol)
+ return find_mirrors(request, protocols=[proto])
+
+
+def mirrors(request):
+ mirror_list = Mirror.objects.select_related().order_by('tier', 'name')
+ protos = MirrorUrl.objects.values_list(
+ 'mirror_id', 'protocol__protocol').order_by(
+ 'mirror_id', 'protocol__protocol').distinct()
+ countries = MirrorUrl.objects.values_list(
+ 'mirror_id', 'country').order_by(
+ 'mirror_id', 'country').distinct()
+
+ if not request.user.is_authenticated():
+ mirror_list = mirror_list.filter(public=True, active=True)
+ protos = protos.filter(
+ mirror__public=True, mirror__active=True, active=True)
+ countries = countries.filter(
+ mirror__public=True, mirror__active=True, active=True)
+
+ protos = {k: list(v) for k, v in groupby(protos, key=itemgetter(0))}
+ countries = {k: list(v) for k, v in groupby(countries, key=itemgetter(0))}
+
+ for mirror in mirror_list:
+ item_protos = protos.get(mirror.id, [])
+ mirror.protocols = [item[1] for item in item_protos]
+ mirror.country = None
+ item_countries = countries.get(mirror.id, [])
+ if len(item_countries) == 1:
+ mirror.country = Country(item_countries[0][1])
+
+ return render(request, 'mirrors/mirrors.html',
+ {'mirror_list': mirror_list})
+
+
+def mirror_details(request, name):
+ mirror = get_object_or_404(Mirror, name=name)
+ authorized = request.user.is_authenticated()
+ if not authorized and \
+ (not mirror.public or not mirror.active):
+ raise Http404
+ error_cutoff = timedelta(days=7)
+
+ status_info = get_mirror_statuses(mirror_id=mirror.id,
+ show_all=authorized)
+ checked_urls = {url for url in status_info['urls'] \
+ if url.mirror_id == mirror.id}
+ all_urls = mirror.urls.select_related('protocol')
+ if not authorized:
+ all_urls = all_urls.filter(active=True)
+ all_urls = set(all_urls)
+ # Add dummy data for URLs that we haven't checked recently
+ other_urls = all_urls.difference(checked_urls)
+ for url in other_urls:
+ for attr in ('last_sync', 'completion_pct', 'delay', 'duration_avg',
+ 'duration_stddev', 'score'):
+ setattr(url, attr, None)
+ all_urls = sorted(checked_urls.union(other_urls), key=attrgetter('url'))
+
+ error_logs = get_mirror_errors(mirror_id=mirror.id, cutoff=error_cutoff,
+ show_all=True)
+
+ context = {
+ 'mirror': mirror,
+ 'urls': all_urls,
+ 'cutoff': error_cutoff,
+ 'error_logs': error_logs,
+ }
+ return render(request, 'mirrors/mirror_details.html', context)
+
+
+def mirror_details_json(request, name):
+ authorized = request.user.is_authenticated()
+ mirror = get_object_or_404(Mirror, name=name)
+ if not authorized and (not mirror.public or not mirror.active):
+ raise Http404
+ status_info = get_mirror_statuses(mirror_id=mirror.id,
+ show_all=authorized)
+ data = status_info.copy()
+ data['version'] = 3
+ to_json = json.dumps(data, ensure_ascii=False,
+ cls=ExtendedMirrorStatusJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
+ return response
+
+
+def url_details(request, name, url_id):
+ url = get_object_or_404(MirrorUrl.objects.select_related(),
+ id=url_id, mirror__name=name)
+ mirror = url.mirror
+ authorized = request.user.is_authenticated()
+ if not authorized and \
+ (not mirror.public or not mirror.active or not url.active):
+ raise Http404
+ error_cutoff = timedelta(days=7)
+ cutoff_time = now() - error_cutoff
+ logs = MirrorLog.objects.select_related('location').filter(
+ url=url, check_time__gte=cutoff_time).order_by('-check_time')
+
+ context = {
+ 'url': url,
+ 'logs': logs,
+ }
+ return render(request, 'mirrors/url_details.html', context)
+
+
+def status_last_modified(request, *args, **kwargs):
+ cursor = connection.cursor()
+ cursor.execute("SELECT MAX(check_time) FROM mirrors_mirrorlog")
+ return cursor.fetchone()[0]
+
+
+@condition(last_modified_func=status_last_modified)
+def status(request, tier=None):
+ if tier is not None:
+ tier = int(tier)
+ if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+ raise Http404
+ bad_timedelta = timedelta(days=3)
+ status_info = get_mirror_statuses()
+
+ urls = status_info['urls']
+ good_urls = []
+ bad_urls = []
+ for url in urls:
+ # screen by tier if we were asked to
+ if tier is not None and url.mirror.tier != tier:
+ continue
+ # split them into good and bad lists based on delay
+ if url.completion_pct is None:
+ # skip URLs that have never been checked
+ continue
+ elif not url.delay or url.delay > bad_timedelta:
+ bad_urls.append(url)
+ else:
+ good_urls.append(url)
+
+ error_logs = get_mirror_errors()
+ if tier is not None:
+ error_logs = [log for log in error_logs
+ if log['url'].mirror.tier == tier]
+
+ context = status_info.copy()
+ context.update({
+ 'good_urls': sorted(good_urls, key=attrgetter('score')),
+ 'bad_urls': sorted(bad_urls, key=lambda u: u.delay or timedelta.max),
+ 'error_logs': error_logs,
+ 'tier': tier,
+ })
+ return render(request, 'mirrors/status.html', context)
+
+
+class MirrorStatusJSONEncoder(DjangoJSONEncoder):
+ '''Base JSONEncoder extended to handle datetime.timedelta and MirrorUrl
+ serialization. The base class takes care of datetime.datetime types.'''
+ url_attributes = ('url', 'protocol', 'last_sync', 'completion_pct',
+ 'delay', 'duration_avg', 'duration_stddev', 'score')
+
+ def default(self, obj):
+ if isinstance(obj, timedelta):
+ # always returned as integer seconds
+ return obj.days * 24 * 3600 + obj.seconds
+ if isinstance(obj, MirrorUrl):
+ data = {attr: getattr(obj, attr) for attr in self.url_attributes}
+ country = obj.country
+ data['country'] = unicode(country.name)
+ data['country_code'] = country.code
+ return data
+ if isinstance(obj, MirrorProtocol):
+ return unicode(obj)
+ return super(MirrorStatusJSONEncoder, self).default(obj)
+
+
+class ExtendedMirrorStatusJSONEncoder(MirrorStatusJSONEncoder):
+ '''Adds URL check history information.'''
+ log_attributes = ('check_time', 'last_sync', 'duration', 'is_success',
+ 'location_id')
+
+ def default(self, obj):
+ if isinstance(obj, MirrorUrl):
+ data = super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+ cutoff = now() - DEFAULT_CUTOFF
+ data['logs'] = list(obj.logs.filter(
+ check_time__gte=cutoff).order_by('check_time'))
+ return data
+ if isinstance(obj, MirrorLog):
+ return {attr: getattr(obj, attr) for attr in self.log_attributes}
+ return super(ExtendedMirrorStatusJSONEncoder, self).default(obj)
+
+
+@cache_page(67)
+@condition(last_modified_func=status_last_modified)
+def status_json(request, tier=None):
+ if tier is not None:
+ tier = int(tier)
+ if tier not in [t[0] for t in Mirror.TIER_CHOICES]:
+ raise Http404
+ status_info = get_mirror_statuses()
+ data = status_info.copy()
+ if tier is not None:
+ data['urls'] = [url for url in data['urls'] if url.mirror.tier == tier]
+ data['version'] = 3
+ to_json = json.dumps(data, ensure_ascii=False, cls=MirrorStatusJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
+ return response
+
+
+class LocationJSONEncoder(DjangoJSONEncoder):
+ '''Base JSONEncoder extended to handle CheckLocation objects.'''
+
+ def default(self, obj):
+ if isinstance(obj, CheckLocation):
+ return {
+ 'id': obj.pk,
+ 'hostname': obj.hostname,
+ 'source_ip': obj.source_ip,
+ 'country': unicode(obj.country.name),
+ 'country_code': obj.country.code,
+ 'ip_version': obj.ip_version,
+ }
+ return super(LocationJSONEncoder, self).default(obj)
+
+
+def locations_json(request):
+ data = {}
+ data['version'] = 1
+ data['locations'] = list(CheckLocation.objects.all().order_by('pk'))
+ to_json = json.dumps(data, ensure_ascii=False, cls=LocationJSONEncoder)
+ response = HttpResponse(to_json, content_type='application/json')
+ return response
+
+# vim: set ts=4 sw=4 et: