From 0776b440b522f2b01801c31e89b5cf4e8073f0f1 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Mon, 6 Sep 2010 12:37:52 -0500 Subject: Add new MirrorLog model and associated migration Signed-off-by: Dan McGee --- mirrors/migrations/0003_auto__add_mirrorlog.py | 72 ++++++++++++++++++++++++++ mirrors/models.py | 14 +++++ 2 files changed, 86 insertions(+) create mode 100644 mirrors/migrations/0003_auto__add_mirrorlog.py diff --git a/mirrors/migrations/0003_auto__add_mirrorlog.py b/mirrors/migrations/0003_auto__add_mirrorlog.py new file mode 100644 index 00000000..5b4c225b --- /dev/null +++ b/mirrors/migrations/0003_auto__add_mirrorlog.py @@ -0,0 +1,72 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'MirrorLog' + db.create_table('mirrors_mirrorlog', ( + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('url', self.gf('django.db.models.fields.related.ForeignKey')(related_name='logs', to=orm['mirrors.MirrorUrl'])), + ('check_time', self.gf('django.db.models.fields.DateTimeField')(db_index=True)), + ('last_sync', self.gf('django.db.models.fields.DateTimeField')(null=True)), + ('duration', self.gf('django.db.models.fields.FloatField')(null=True)), + ('is_success', self.gf('django.db.models.fields.BooleanField')(default=True)), + ('error', self.gf('django.db.models.fields.CharField')(default='', max_length=255, blank=True)), + )) + db.send_create_signal('mirrors', ['MirrorLog']) + + def backwards(self, orm): + # Deleting model 'MirrorLog' + db.delete_table('mirrors_mirrorlog') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'object_name': 'MirrorProtocol'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 94256a9c..5cab9db6 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -54,4 +54,18 @@ class MirrorRsync(models.Model): class Meta: verbose_name = 'Mirror Rsync IP' +class MirrorLog(models.Model): + url = models.ForeignKey(MirrorUrl, related_name="logs") + check_time = models.DateTimeField(db_index=True) + last_sync = models.DateTimeField(null=True) + duration = models.FloatField(null=True) + is_success = models.BooleanField(default=True) + error = models.CharField(max_length=255, blank=True, default='') + + def __unicode__(self): + return "Check of %s at %s" % (url.url, check_time) + + class Meta: + verbose_name = 'Mirror Check Log' + # vim: set ts=4 sw=4 et: -- cgit v1.2.3-55-g3dc8 From 7c88e3e1a2b5d36281b3981e2144c2dd16c7596c Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Fri, 10 Sep 2010 18:42:59 -0500 Subject: Rename generate mirrorlist view Signed-off-by: Dan McGee --- mirrors/views.py | 2 +- urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mirrors/views.py b/mirrors/views.py index ddc42cbb..b4c2c2c7 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -21,7 +21,7 @@ class MirrorlistForm(forms.Form): self.fields['protocol'].initial = [t[0] for t in protos] @csrf_exempt -def generate(request): +def generate_mirrorlist(request): if request.REQUEST.get('country', ''): form = MirrorlistForm(data=request.REQUEST) if form.is_valid(): diff --git a/urls.py b/urls.py index 00d0986f..f2ef3418 100644 --- a/urls.py +++ b/urls.py @@ -73,7 +73,7 @@ urlpatterns = patterns('', (r'^mirrors/$', 'devel.views.mirrorlist', {}, 'mirrors-list'), - (r'^mirrorlist/$', 'mirrors.views.generate', {}, 'mirrorlist'), + (r'^mirrorlist/$', 'mirrors.views.generate_mirrorlist', {}, 'mirrorlist'), (r'^mirrorlist/all/$', 'mirrors.views.find_mirrors', {'countries': ['all']}), (r'^mirrorlist/all/ftp/$', 'mirrors.views.find_mirrors', {'countries': ['all'], 'protocols': ['ftp']}), -- cgit v1.2.3-55-g3dc8 From 3d8bc07622561028dbca9c709470accf79c95bd6 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 16 Sep 2010 16:45:30 -0500 Subject: Add 'mirrorcheck' command This does the actual work of going out and checking the mirror status. In short, it polls every active mirror URL for the 'lastsync' file and then records the appropriate details. These include the contents of that file, how long the total time to retrieve took, and any errors encountered. In order to finish up a bit faster, we spawn several threads to do the actual work. This parallelization allows the whole check process to take around 30 seconds rather than several minutes. Signed-off-by: Dan McGee --- mirrors/management/__init__.py | 0 mirrors/management/commands/__init__.py | 0 mirrors/management/commands/mirrorcheck.py | 149 +++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 mirrors/management/__init__.py create mode 100644 mirrors/management/commands/__init__.py create mode 100644 mirrors/management/commands/mirrorcheck.py diff --git a/mirrors/management/__init__.py b/mirrors/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mirrors/management/commands/__init__.py b/mirrors/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py new file mode 100644 index 00000000..4a933ca7 --- /dev/null +++ b/mirrors/management/commands/mirrorcheck.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +""" +mirrorcheck command + +Poll every active mirror URL we have in the database, grab the 'lastsync' file, +and record details about how long it took and how up to date the mirror is. If +we encounter errors, record those as well. + +Usage: ./manage.py mirrorcheck +""" + +from django.core.management.base import NoArgsCommand +from django.db.models import Q + +from datetime import datetime, timedelta +import logging +import re +import socket +import sys +import time +import thread +from threading import Thread +from Queue import Queue, Empty +import urllib2 + +from logging import ERROR, WARNING, INFO, DEBUG + +from mirrors.models import Mirror, MirrorUrl, MirrorLog + +logging.basicConfig( + level=WARNING, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + +class Command(NoArgsCommand): + help = "Runs a check on all known mirror URLs to determine their up-to-date status." + + def handle_noargs(self, **options): + v = int(options.get('verbosity', 0)) + if v == 0: + logger.level = ERROR + elif v == 1: + logger.level = INFO + elif v == 2: + logger.level = DEBUG + + import signal, traceback + handler = lambda sig, stack: traceback.print_stack(stack) + signal.signal(signal.SIGQUIT, handler) + signal.signal(signal.SIGUSR1, handler) + + return check_current_mirrors() + +def parse_rfc3339_datetime(time): + # '2010-09-02 11:05:06+02:00' + m = re.match('^(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})([-+])(\d{2}):(\d{2})', time) + if m: + vals = m.groups() + parsed = datetime(int(vals[0]), int(vals[1]), int(vals[2]), + int(vals[3]), int(vals[4]), int(vals[5])) + # now account for time zone offset + sign = vals[6] + offset = timedelta(hours=int(sign + vals[7]), + minutes=int(sign + vals[8])) + # subtract the offset, e.g. '-04:00' should be moved up 4 hours + return parsed - offset + return None + +def check_mirror_url(mirror_url): + url = mirror_url.url + 'lastsync' + logger.info("checking URL %s" % url) + log = MirrorLog(url=mirror_url, check_time=datetime.utcnow()) + try: + start = time.time() + result = urllib2.urlopen(url, timeout=10) + data = result.read() + result.close() + end = time.time() + # lastsync should be an epoch value, but some mirrors + # are creating their own in RFC-3339 format: + # '2010-09-02 11:05:06+02:00' + try: + parsed_time = datetime.utcfromtimestamp(int(data)) + except ValueError: + parsed_time = parse_rfc3339_datetime(data) + + log.last_sync = parsed_time + log.duration = end - start + logger.debug("success: %s, %.2f" % (url, log.duration)) + except urllib2.HTTPError, e: + log.is_success = False + log.error =str(e) + logger.debug("failed: %s, %s" % (url, log.error)) + except urllib2.URLError, e: + log.is_success=False + log.error = e.reason + if isinstance(e.reason, socket.timeout): + log.error = "Connection timed out." + elif isinstance(e.reason, socket.error): + log.error = e.reason.args[1] + logger.debug("failed: %s, %s" % (url, log.error)) + + log.save() + return log + +def mirror_url_worker(queue): + while True: + try: + item = queue.get(block=False) + check_mirror_url(item) + queue.task_done() + except Empty: + return 0 + +class MirrorCheckPool(object): + def __init__(self, work, num_threads=10): + self.tasks = Queue() + for i in work: + self.tasks.put(i) + self.threads = [] + for i in range(num_threads): + thread = Thread(target=mirror_url_worker, args=(self.tasks,)) + thread.daemon = True + self.threads.append(thread) + + def run_and_join(self): + logger.debug("starting threads") + for t in self.threads: + t.start() + logger.debug("joining on all threads") + self.tasks.join() + +def check_current_mirrors(): + urls = MirrorUrl.objects.filter( + Q(protocol__protocol__iexact='HTTP') | + Q(protocol__protocol__iexact='FTP'), + mirror__active=True, mirror__public=True) + + pool = MirrorCheckPool(urls) + pool.run_and_join() + return 0 + +# For lack of a better place to put it, here is a query to get latest check +# result joined with mirror details: +# SELECT mu.*, m.*, ml.* FROM mirrors_mirrorurl mu JOIN mirrors_mirror m ON mu.mirror_id = m.id JOIN mirrors_mirrorlog ml ON mu.id = ml.url_id LEFT JOIN mirrors_mirrorlog ml2 ON ml.url_id = ml2.url_id AND ml.id < ml2.id WHERE ml2.id IS NULL AND m.active = 1 AND m.public = 1; + +# vim: set ts=4 sw=4 et: -- cgit v1.2.3-55-g3dc8 From 40ac4818aa7812a5399a0d4c176137984d5cfd30 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sun, 12 Sep 2010 10:14:34 -0500 Subject: Add mirror status view Signed-off-by: Dan McGee --- mirrors/templatetags/__init__.py | 0 mirrors/templatetags/mirror_status.py | 15 ++++++++++ mirrors/views.py | 50 +++++++++++++++++++++++++++++++++ templates/mirrors/status.html | 53 +++++++++++++++++++++++++++++++++++ templates/mirrors/status_table.html | 29 +++++++++++++++++++ urls.py | 1 + 6 files changed, 148 insertions(+) create mode 100644 mirrors/templatetags/__init__.py create mode 100644 mirrors/templatetags/mirror_status.py create mode 100644 templates/mirrors/status.html create mode 100644 templates/mirrors/status_table.html diff --git a/mirrors/templatetags/__init__.py b/mirrors/templatetags/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/mirrors/templatetags/mirror_status.py b/mirrors/templatetags/mirror_status.py new file mode 100644 index 00000000..09c5b331 --- /dev/null +++ b/mirrors/templatetags/mirror_status.py @@ -0,0 +1,15 @@ +from django import template + +register = template.Library() + +@register.filter +def duration(value): + if not value: + return u'\u221e' + # does not take microseconds into account + total_secs = value.seconds + value.days * 24 * 3600 + mins, secs = divmod(total_secs, 60) + hrs, mins = divmod(mins, 60) + return '%d:%02d' % (hrs, mins) + +# vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index b4c2c2c7..a31c1371 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -1,9 +1,14 @@ from django import forms +from django.db.models import Avg, Count, Max, Min, StdDev +from django.db.models import Q from django.views.decorators.csrf import csrf_exempt from django.views.generic.simple import direct_to_template from main.utils import make_choice from .models import Mirror, MirrorUrl, MirrorProtocol +from .models import MirrorLog + +import datetime class MirrorlistForm(forms.Form): country = forms.MultipleChoiceField(required=False) @@ -49,4 +54,49 @@ def find_mirrors(request, countries=None, protocols=None): }, mimetype='text/plain') +def status(request): + cutoff_time = datetime.datetime.utcnow() - datetime.timedelta(hours=24) + bad_timedelta = datetime.timedelta(days=3) + + protocols = MirrorProtocol.objects.exclude(protocol__iexact='rsync') + # I swear, this actually has decent performance... + urls = MirrorUrl.objects.select_related( + 'mirror', 'protocol').filter( + mirror__active=True, mirror__public=True, + protocol__in=protocols).filter( + logs__check_time__gte=cutoff_time).annotate( + check_count=Count('logs'), last_sync=Max('logs__last_sync'), + last_check=Max('logs__check_time'), + duration_avg=Avg('logs__duration'), duration_min=Min('logs__duration'), + duration_max=Max('logs__duration'), duration_stddev=StdDev('logs__duration') + ).order_by('mirror__country', 'url') + # errors during check process go in another table + error_logs = MirrorLog.objects.filter( + is_success=False, check_time__gte=cutoff_time).values( + 'url__url', 'url__protocol__protocol', 'url__mirror__country', + 'error').annotate(Count('error'), Max('check_time')) + + good_urls = [] + bad_urls = [] + for url in urls: + if url.last_check and url.last_sync: + d = url.last_check - url.last_sync + url.delay = d + url.score = d.days * 24 + d.seconds / 3600 + url.duration_avg + url.duration_stddev + else: + url.delay = None + url.score = None + # split them into good and bad lists based on delay + if not url.delay or url.delay > bad_timedelta: + bad_urls.append(url) + else: + good_urls.append(url) + + context = { + 'good_urls': good_urls, + 'bad_urls': bad_urls, + 'error_logs': error_logs, + } + return direct_to_template(request, 'mirrors/status.html', context) + # vim: set ts=4 sw=4 et: diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html new file mode 100644 index 00000000..8dd6fa11 --- /dev/null +++ b/templates/mirrors/status.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} + +{% block title %}Arch Linux - Mirror Status{% endblock %} + +{% block content %} +
+

Mirror Status

+ +

Out of Sync Mirrors

+ {% with bad_urls as urls %} + {% include "mirrors/status_table.html" %} + {% endwith %} + +

Successfully Syncing Mirrors

+ {% with good_urls as urls %} + {% include "mirrors/status_table.html" %} + {% endwith %} + +

Mirror Syncing Error Log

+ + + + + + + + + + + + + {% for log in error_logs %} + + + + + + + + + {% endfor %} + +
Mirror URLProtocolCountryError MessageLast OccurredOccurrences (last 24 hours)
{{ log.url__url }}{{ log.url__protocol__protocol }}{{ log.url__mirror__country }}{{ log.error }}{{ log.check_time__max|date:'Y-m-d H:i' }}{{ log.error__count }}
+ +
+{% load cdn %}{% jquery %} + + +{% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html new file mode 100644 index 00000000..d7f5b0ec --- /dev/null +++ b/templates/mirrors/status_table.html @@ -0,0 +1,29 @@ +{% load mirror_status %} + + + + + + + + + + + + + + + {% for m_url in urls %} + + + + + + + + + + + {% endfor %} + +
Mirror URLProtocolCountryLast SyncDelay (hh:mm)μ Duration (secs)σ Duration (secs)Mirror Score
{{ m_url.url }}{{ m_url.protocol }}{{ m_url.mirror.country }}{{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }}{{ m_url.delay|duration|default:'unknown' }}{{ m_url.duration_avg|floatformat:2|default:'unknown' }}{{ m_url.duration_stddev|floatformat:2|default:'unknown' }}{{ m_url.score|floatformat:1|default:'unknown' }}
diff --git a/urls.py b/urls.py index f2ef3418..7cc60cfe 100644 --- a/urls.py +++ b/urls.py @@ -72,6 +72,7 @@ urlpatterns = patterns('', (r'^news/$', 'news.views.news_list', {}, 'news-list'), (r'^mirrors/$', 'devel.views.mirrorlist', {}, 'mirrors-list'), + (r'^mirrors/status/$', 'mirrors.views.status', {}, 'mirrors-status'), (r'^mirrorlist/$', 'mirrors.views.generate_mirrorlist', {}, 'mirrorlist'), (r'^mirrorlist/all/$', 'mirrors.views.find_mirrors', {'countries': ['all']}), -- cgit v1.2.3-55-g3dc8 From 1f44788e1e701f78f71ba526183313bd0c4593aa Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 16 Sep 2010 17:16:18 -0500 Subject: Add debugger when having to parse lastsync file Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index 4a933ca7..c2ab077f 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -84,6 +84,8 @@ def check_mirror_url(mirror_url): try: parsed_time = datetime.utcfromtimestamp(int(data)) except ValueError: + logger.debug("attempting to parse generated lastsync file" + " from mirror %s, value %s" % (url, data)) parsed_time = parse_rfc3339_datetime(data) log.last_sync = parsed_time -- cgit v1.2.3-55-g3dc8 From b8a78408ff194ca2f822979fec5598778eff5826 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Sep 2010 17:29:45 -0500 Subject: Small updates to mirrorcheck command Signed-off-by: Dan McGee --- mirrors/management/commands/mirrorcheck.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mirrors/management/commands/mirrorcheck.py b/mirrors/management/commands/mirrorcheck.py index c2ab077f..1662b15c 100644 --- a/mirrors/management/commands/mirrorcheck.py +++ b/mirrors/management/commands/mirrorcheck.py @@ -42,7 +42,7 @@ class Command(NoArgsCommand): if v == 0: logger.level = ERROR elif v == 1: - logger.level = INFO + logger.level = WARNING elif v == 2: logger.level = DEBUG @@ -84,8 +84,10 @@ def check_mirror_url(mirror_url): try: parsed_time = datetime.utcfromtimestamp(int(data)) except ValueError: - logger.debug("attempting to parse generated lastsync file" - " from mirror %s, value %s" % (url, data)) + # it is bad news to try logging the lastsync value; + # sometimes we get a crazy-encoded web page. + logger.info("attempting to parse generated lastsync file" + " from mirror %s" % url) parsed_time = parse_rfc3339_datetime(data) log.last_sync = parsed_time -- cgit v1.2.3-55-g3dc8 From 2a296af10d34c65e0f94d1a5b70c84ba31596ba4 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Sep 2010 17:30:14 -0500 Subject: Add ordering, sorting, and a lot more info to mirror status page This should get this to the point where it is releasable to the general public for their use and pleasure. Still not sure on how often the check should be run, and we probably want to incorporate this mined data into some other things like the mirror list generator. Signed-off-by: Dan McGee --- mirrors/views.py | 9 ++++-- templates/mirrors/status.html | 56 ++++++++++++++++++++++++++++++++++--- templates/mirrors/status_table.html | 2 +- 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/mirrors/views.py b/mirrors/views.py index a31c1371..59d6337b 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -69,12 +69,16 @@ def status(request): last_check=Max('logs__check_time'), duration_avg=Avg('logs__duration'), duration_min=Min('logs__duration'), duration_max=Max('logs__duration'), duration_stddev=StdDev('logs__duration') - ).order_by('mirror__country', 'url') + ).order_by('-last_sync', '-duration_avg') # errors during check process go in another table error_logs = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time).values( 'url__url', 'url__protocol__protocol', 'url__mirror__country', - 'error').annotate(Count('error'), Max('check_time')) + 'error').annotate( + error_count=Count('error'), last_occurred=Max('check_time') + ).order_by('-last_occurred', '-error_count') + + last_check = max([u.last_check for u in urls]) good_urls = [] bad_urls = [] @@ -93,6 +97,7 @@ def status(request): good_urls.append(url) context = { + 'last_check': last_check, 'good_urls': good_urls, 'bad_urls': bad_urls, 'error_logs': error_logs, diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 8dd6fa11..5743e47b 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -5,19 +5,64 @@ {% block content %}

Mirror Status

+

This page reports the status of all known, public, and active Arch Linux + mirrors. All data on this page reflects the status of the mirrors within + the last 24 hours. All listed times are UTC. The check script runs + on a regular basis and polls for the lastsync file in the root of + our repository layout. This file is regularly updated on the central + repository, so checking the value within allows one to see if the mirror + has synced recently. This page contains several pieces of information about + each mirror.

+
    +
  • Mirror URL: Mirrors are checked on a per-URL basis. If + both FTP and HTTP access are provided, both will be listed here.
    • +
  • Last Sync: The timestamp retrieved from the + lastsync file on the mirror. If this file could not be + retrieved or contained data we didn't recognize, this column will show + 'unknown'.
    • +
  • Delay: The calculated mirroring delay; e.g. last + check − last sync.
    • +
  • μ Duration: The average (mean) time it took to connect and + retrieve the lastsync file from the given URL. Note that this + connection time is from the location of the Arch server; your geography + may product different results.
    • +
  • σ Duration: The standard deviation of the connect and + retrieval time. A high standard deviation can indicate an unstable or + overloaded mirror.
    • +
  • Mirror Score: A very rough calculation for ranking + mirrors. It is currently calculated as hours delay + average + duration + standard deviation. Lower is better.
    • +
+

The final table on this page is an error log, which shows any errors + that occurred while contacting mirrors. This only shows errors that + occurred within the last 24 hours.

+ +

The last mirror check ran at {{ last_check|date:'Y-m-d H:i' }} UTC.

+ +

Out of Sync Mirrors

{% with bad_urls as urls %} + {% with 'outofsync_mirrors' as table_id %} {% include "mirrors/status_table.html" %} {% endwith %} + {% endwith %} +

Successfully Syncing Mirrors

{% with good_urls as urls %} + {% with 'successful_mirrors' as table_id %} {% include "mirrors/status_table.html" %} {% endwith %} + {% endwith %} +

Mirror Syncing Error Log

- +
@@ -35,8 +80,8 @@ - - + + {% endfor %} @@ -47,7 +92,10 @@ {% endblock %} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index d7f5b0ec..90bbf6b6 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -1,5 +1,5 @@ {% load mirror_status %} -
Mirror URL{{ log.url__protocol__protocol }} {{ log.url__mirror__country }} {{ log.error }}{{ log.check_time__max|date:'Y-m-d H:i' }}{{ log.error__count }}{{ log.last_occurred|date:'Y-m-d H:i' }}{{ log.error_count }}
+
-- cgit v1.2.3-55-g3dc8 From 8ff8190c5ca29473cbcc398fb12b33b4430cc050 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 21 Sep 2010 17:39:39 -0500 Subject: Add link to mirror status tool Signed-off-by: Dan McGee --- templates/public/index.html | 13 ++++++++++--- urls.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/templates/public/index.html b/templates/public/index.html index 5a87787d..cc7aa21b 100644 --- a/templates/public/index.html +++ b/templates/public/index.html @@ -132,17 +132,24 @@ title="T-shirts">Schwag via Freewear -

Development

+

Tools

+ +

Development

+ +
Mirror URL