Martinvlba/evorepo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add optional country override for individual mirror URLs

Browse Source 
This allows a named top-level mirror to have geographically distributed
URLs, e.g. kernel.org and the geo-DNS setup.

Signed-off-by: Dan McGee <dan@archlinux.org>
This commit is contained in:
Dan McGee 2011-04-12 18:36:20 -05:00
parent 064813560c
commit 78a553e558
8 changed files with 90 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
mirrors/migrations/0008_auto__add_field_mirrorurl_country.py Normal file
View File

@ -0,0 +1,67 @@
# encoding: utf-8
import datetime
from south.db import db
from south.v2 import SchemaMigration
from django.db import models
class Migration(SchemaMigration):
def forwards(self, orm):
# Adding field 'MirrorUrl.country'
db.add_column('mirrors_mirrorurl', 'country', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=255, null=True, blank=True), keep_default=False)
def backwards(self, orm):
# Deleting field 'MirrorUrl.country'
db.delete_column('mirrors_mirrorurl', 'country')
models = {
'mirrors.mirror': {
'Meta': {'ordering': "('country', 'name')", 'object_name': 'Mirror'},
'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}),
'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}),
'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
'public': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'rsync_password': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
'rsync_user': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '50', 'blank': 'True'}),
'tier': ('django.db.models.fields.SmallIntegerField', [], {'default': '2'}),
'upstream': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['mirrors.Mirror']", 'null': 'True'})
},
'mirrors.mirrorlog': {
'Meta': {'object_name': 'MirrorLog'},
'check_time': ('django.db.models.fields.DateTimeField', [], {'db_index': 'True'}),
'duration': ('django.db.models.fields.FloatField', [], {'null': 'True'}),
'error': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '255', 'blank': 'True'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_success': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'last_sync': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
'url': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'logs'", 'to': "orm['mirrors.MirrorUrl']"})
},
'mirrors.mirrorprotocol': {
'Meta': {'ordering': "('protocol',)", 'object_name': 'MirrorProtocol'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'is_download': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'})
},
'mirrors.mirrorrsync': {
'Meta': {'object_name': 'MirrorRsync'},
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}),
'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['mirrors.Mirror']"})
},
'mirrors.mirrorurl': {
'Meta': {'object_name': 'MirrorUrl'},
'country': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '255', 'null': 'True', 'blank': 'True'}),
'has_ipv4': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
'has_ipv6': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.Mirror']"}),
'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['mirrors.MirrorProtocol']"}),
'url': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'})
}
}
complete_apps = ['mirrors']

6
mirrors/models.py
View File

@ -58,6 +58,8 @@ class MirrorUrl(models.Model):
protocol = models.ForeignKey(MirrorProtocol, related_name="urls",
editable=False, on_delete=models.PROTECT)
mirror = models.ForeignKey(Mirror, related_name="urls")
country = models.CharField(max_length=255, blank=True, null=True,
db_index=True)
has_ipv4 = models.BooleanField("IPv4 capable", default=True,
editable=False)
has_ipv6 = models.BooleanField("IPv6 capable", default=False,
@ -73,6 +75,10 @@ def address_families(self):
def hostname(self):
return urlparse(self.url).hostname
@property
def real_country(self):
return self.country or self.mirror.country
def clean(self):
try:
# Auto-map the protocol field by looking at the URL

9
mirrors/utils.py
View File

@ -82,10 +82,13 @@ def get_mirror_errors(cutoff=default_cutoff):
errors = MirrorLog.objects.filter(
is_success=False, check_time__gte=cutoff_time,
url__mirror__active=True, url__mirror__public=True).values(
'url__url', 'url__protocol__protocol', 'url__mirror__country',
'error').annotate(
'url__url', 'url__country', 'url__protocol__protocol',
'url__mirror__country', 'error').annotate(
error_count=Count('error'), last_occurred=Max('check_time')
).order_by('-last_occurred', '-error_count')
return list(errors)
errors = list(errors)
for err in errors:
err['country'] = err['url__country'] or err['url__mirror__country']
return errors
# vim: set ts=4 sw=4 et:

12
mirrors/views.py
View File

@ -23,10 +23,10 @@ class MirrorlistForm(forms.Form):
def __init__(self, *args, **kwargs):
super(MirrorlistForm, self).__init__(*args, **kwargs)
mirrors = Mirror.objects.filter(active=True).values_list(
countries = Mirror.objects.filter(active=True).values_list(
'country', flat=True).distinct().order_by('country')
self.fields['country'].choices = [('all','All')] + make_choice(
mirrors)
countries)
self.fields['country'].initial = ['all']
protos = make_choice(
MirrorProtocol.objects.filter(is_download=True))
@ -61,7 +61,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
mirror__public=True, mirror__active=True, mirror__isos=True
)
if countries and 'all' not in countries:
qset = qset.filter(mirror__country__in=countries)
qset = qset.filter(Q(country__in=countries) |
Q(mirror__country__in=countries))
ip_version = Q()
if ipv4_supported:
@ -71,7 +72,8 @@ def find_mirrors(request, countries=None, protocols=None, use_status=False,
qset = qset.filter(ip_version)
if not use_status:
urls = qset.order_by('mirror__country', 'mirror__name', 'url')
urls = qset.order_by('mirror__name', 'url')
urls = sorted(urls, key=lambda x: x.real_country)
template = 'mirrors/mirrorlist.txt'
else:
status_info = get_mirror_statuses()
@ -158,7 +160,7 @@ def default(self, obj):
for attr in self.url_attributes:
data[attr] = getattr(obj, attr)
# separate because it isn't on the URL directly
data['country'] = obj.mirror.country
data['country'] = obj.real_country
return data
if isinstance(obj, MirrorProtocol):
return unicode(obj)

2
templates/mirrors/mirrorlist.txt
View File

@ -8,6 +8,6 @@ content right, and then go back later to fix it all up.
## Generated on {% now "Y-m-d" %}
##{% for mirror_url in mirror_urls %}{% ifchanged %}
## {{ mirror_url.mirror.country }}{% endifchanged %}
## {{ mirror_url.real_country }}{% endifchanged %}
#Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %}
{% endautoescape %}

2
templates/mirrors/mirrorlist_status.txt
View File

@ -8,6 +8,6 @@ content right, and then go back later to fix it all up.
## Sorted by mirror score from mirror status page
## Generated on {% now "Y-m-d" %}
{% for mirror_url in mirror_urls %}
## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.mirror.country }}
## Score: {{ mirror_url.score|floatformat:1|default:'unknown' }}, {{ mirror_url.real_country }}
#Server = {{ mirror_url.url}}$repo/os/$arch{% endfor %}
{% endautoescape %}

2
templates/mirrors/status.html
View File

@ -91,7 +91,7 @@ <h3>Mirror Syncing Error Log</h3>
<tr class="{% cycle 'odd' 'even' %}">
<td>{{ log.url__url }}</td>
<td>{{ log.url__protocol__protocol }}</td>
<td>{{ log.url__mirror__country }}</td>
<td>{{ log.country }}</td>
<td>{{ log.error }}</td>
<td>{{ log.last_occurred|date:'Y-m-d H:i' }}</td>
<td>{{ log.error_count }}</td>

2
templates/mirrors/status_table.html
View File

@ -18,7 +18,7 @@
<tr class="{% cycle 'odd' 'even' %}">
<td>{{ m_url.url }}</td>
<td>{{ m_url.protocol }}</td>
<td>{{ m_url.mirror.country }}</td>
<td>{{ m_url.real_country }}</td>
<td>{{ m_url.last_sync|date:'Y-m-d H:i'|default:'unknown' }}</td>
<td>{{ m_url.completion_pct|percentage:1 }}</td>
<td>{{ m_url.delay|duration|default:'unknown' }}</td>