From 78a553e558217640c0efa233ac7a7037be3f34b5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Tue, 12 Apr 2011 18:36:20 -0500 Subject: Add optional country override for individual mirror URLs This allows a named top-level mirror to have geographically distributed URLs, e.g. kernel.org and the geo-DNS setup. Signed-off-by: Dan McGee --- .../0008_auto__add_field_mirrorurl_country.py | 67 ++++++++++++++++++++++ mirrors/models.py | 6 ++ mirrors/utils.py | 9 ++- mirrors/views.py | 12 ++-- templates/mirrors/mirrorlist.txt | 2 +- templates/mirrors/mirrorlist_status.txt | 2 +- templates/mirrors/status.html | 2 +- templates/mirrors/status_table.html | 2 +- 8 files changed, 90 insertions(+), 12 deletions(-) create mode 100644 mirrors/migrations/0008_auto__add_field_mirrorurl_country.py diff --git a/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py b/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py new file mode 100644 index 00000000..660ac080 --- /dev/null +++ b/mirrors/migrations/0008_auto__add_field_mirrorurl_country.py @@ -0,0 +1,67 @@ +# encoding: utf-8 +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding field 'MirrorUrl.country' + db.add_column('mirrors_mirrorurl', 'country', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True), keep_default=False) + + def backwards(self, orm): + # Deleting field 'MirrorUrl.country' + db.delete_column('mirrors_mirrorurl', 'country') + + models = { + 'mirrors.mirror': { + 'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'}, + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}), + 'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}), + 'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}), + 'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'}) + }, + 'mirrors.mirrorlog': { + 'Meta': {'object_name': 'MirrorLog'}, + 'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}), + 'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}), + 'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"}) + }, + 'mirrors.mirrorprotocol': { + 'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'mirrors.mirrorrsync': { + 'Meta': {'object_name': 'MirrorRsync'}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"}) + }, + 'mirrors.mirrorurl': { + 'Meta': {'object_name': 'MirrorUrl'}, + 'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), + 'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + } + } + + complete_apps = ['mirrors'] diff --git a/mirrors/models.py b/mirrors/models.py index 80808e2e..bcde210c 100644 --- a/mirrors/models.py +++ b/mirrors/models.py @@ -58,6 +58,8 @@ class MirrorUrl(models.Model): protocol = models.ForeignKey(MirrorProtocol, related_name="urls", editable=False, on_delete=models.PROTECT) mirror = models.ForeignKey(Mirror, related_name="urls") + country = models.CharField(max_length=255, blank=True, null=True, + db_index=True) has_ipv4 = models.BooleanField("IPv4 capable", default=True, editable=False) has_ipv6 = models.BooleanField("IPv6 capable", default=False, @@ -73,6 +75,10 @@ class MirrorUrl(models.Model): def hostname(self): return urlparse(self.url).hostname + @property + def real_country(self): + return self.country or self.mirror.country + def clean(self): try: # Auto-map the protocol field by looking at the URL diff --git a/mirrors/utils.py b/mirrors/utils.py index 124b66e6..2d88f125 100644 --- a/mirrors/utils.py +++ b/mirrors/utils.py @@ -82,10 +82,13 @@ def get_mirror_errors(cutoff=default_cutoff): errors = MirrorLog.objects.filter( is_success=False, check_time__gte=cutoff_time, url__mirror__active=True, url__mirror__public=True).values( - 'url__url', 'url__protocol__protocol', 'url__mirror__country', - 'error').annotate( + 'url__url', 'url__country', 'url__protocol__protocol', + 'url__mirror__country', 'error').annotate( error_count=Count('error'), last_occurred=Max('check_time') ).order_by('-last_occurred', '-error_count') - return list(errors) + errors = list(errors) + for err in errors: + err['country'] = err['url__country'] or err['url__mirror__country'] + return errors # vim: set ts=4 sw=4 et: diff --git a/mirrors/views.py b/mirrors/views.py index 032a4700..69592146 100644 --- a/mirrors/views.py +++ b/mirrors/views.py @@ -23,10 +23,10 @@ class MirrorlistForm(forms.Form): def __init__(self, *args, **kwargs): super(MirrorlistForm, self).__init__(*args, **kwargs) - mirrors = Mirror.objects.filter(active=True).values_list( + countries = Mirror.objects.filter(active=True).values_list( 'country', flat=True).distinct().order_by('country') self.fields['country'].choices = [('all','All')] + make_choice( - mirrors) + countries) self.fields['country'].initial = ['all'] protos = make_choice( MirrorProtocol.objects.filter(is_download=True)) @@ -61,7 +61,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, mirror__public=True, mirror__active=True, mirror__isos=True ) if countries and 'all' not in countries: - qset = qset.filter(mirror__country__in=countries) + qset = qset.filter(Q(country__in=countries) | + Q(mirror__country__in=countries)) ip_version = Q() if ipv4_supported: @@ -71,7 +72,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False, qset = qset.filter(ip_version) if not use_status: - urls = qset.order_by('mirror__country', 'mirror__name', 'url') + urls = qset.order_by('mirror__name', 'url') + urls = sorted(urls, key=lambda x: x.real_country) template = 'mirrors/mirrorlist.txt' else: status_info = get_mirror_statuses() @@ -158,7 +160,7 @@ class MirrorStatusJSONEncoder(DjangoJSONEncoder): for attr in self.url_attributes: data[attr] = getattr(obj, attr) # separate because it isn't on the URL directly - data['country'] = obj.mirror.country + data['country'] = obj.real_country return data if isinstance(obj, MirrorProtocol): return unicode(obj) diff --git a/templates/mirrors/mirrorlist.txt b/templates/mirrors/mirrorlist.txt index 2eedbd71..24b52fa1 100644 --- a/templates/mirrors/mirrorlist.txt +++ b/templates/mirrors/mirrorlist.txt @@ -8,6 +8,6 @@ content right, and then go back later to fix it all up. ## Generated on {% now "Y-m-d" %} ##{% for mirror_url in mirror_urls %}{% ifchanged %} -## {{ mirror_url.mirror.country }}{% endifchanged %} +## {{ mirror_url.real_country }}{% endifchanged %} #Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %} {% endautoescape %} diff --git a/templates/mirrors/mirrorlist_status.txt b/templates/mirrors/mirrorlist_status.txt index e2fbc1e6..e3504395 100644 --- a/templates/mirrors/mirrorlist_status.txt +++ b/templates/mirrors/mirrorlist_status.txt @@ -8,6 +8,6 @@ content right, and then go back later to fix it all up. ## Sorted by mirror score from mirror status page ## Generated on {% now "Y-m-d" %} {% for mirror_url in mirror_urls %} -## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.mirror.country }} +## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.real_country }} #Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %} {% endautoescape %} diff --git a/templates/mirrors/status.html b/templates/mirrors/status.html index 5724c71c..046c4196 100644 --- a/templates/mirrors/status.html +++ b/templates/mirrors/status.html @@ -91,7 +91,7 @@ {{ log.url__url }} {{ log.url__protocol__protocol }} - {{ log.url__mirror__country }} + {{ log.country }} {{ log.error }} {{ log.last_occurred|date:'Y-m-d H:i' }} {{ log.error_count }} diff --git a/templates/mirrors/status_table.html b/templates/mirrors/status_table.html index 240a5452..72de25dc 100644 --- a/templates/mirrors/status_table.html +++ b/templates/mirrors/status_table.html @@ -18,7 +18,7 @@ {{ m_url.url }} {{ m_url.protocol }} - {{ m_url.mirror.country }} + {{ m_url.real_country }} {{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }} {{ m_url.completion_pct|percentage:1 }} {{ m_url.delay|duration|default:'unknown' }} -- cgit v1.2.3-55-g3dc8