From a0cd93c1d528fbb1c5919fed0a52b09bbe84933d Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 11 Feb 2010 22:15:13 -0600 Subject: Bring package files view up to speed Move it away from the numeric pkgid-based view of old to the new pretty URL format. This does nothing to actually make the view show files (or even provide a link to it), but that will come in future commits. Signed-off-by: Dan McGee --- packages/views.py | 9 +++++---- urls.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/views.py b/packages/views.py index 97929ed3..acd71b2d 100644 --- a/packages/views.py +++ b/packages/views.py @@ -60,7 +60,7 @@ def update(request): def details(request, name='', repo='', arch=''): if all([name, repo, arch]): - pkg= get_object_or_404(Package, + pkg = get_object_or_404(Package, pkgname=name, repo__name__iexact=repo, arch__name=arch) return render_to_response('packages/details.html', RequestContext( request, {'pkg': pkg, })) @@ -178,9 +178,10 @@ def search(request, page=None): template_object_name="package", extra_context=page_dict) -def files(request, pkgid): - pkg = get_object_or_404(Package, id=pkgid) - files = PackageFile.objects.filter(pkg=pkgid) +def files(request, name='', repo='', arch=''): + pkg = get_object_or_404(Package, + pkgname=name, repo__name__iexact=repo, arch__name=arch) + files = PackageFile.objects.filter(pkg=pkg) return render_to_response('packages/files.html', RequestContext(request, {'pkg':pkg,'files':files})) @permission_required('main.change_package') diff --git a/urls.py b/urls.py index ec04edce..e53d918a 100644 --- a/urls.py +++ b/urls.py @@ -27,7 +27,6 @@ urlpatterns = patterns('', (r'^packages/flag/(\d+)/$', 'packages.views.flag'), (r'^packages/flaghelp/$', 'packages.views.flaghelp'), (r'^packages/unflag/(\d+)/$', 'packages.views.unflag'), - (r'^packages/files/(\d+)/$', 'packages.views.files'), (r'^packages/signoffs/$', 'packages.views.signoffs'), (r'^packages/signoff_package/(?P[A-z0-9]+)/(?P[A-z0-9\-+.]+)/$', 'packages.views.signoff_package'), @@ -46,6 +45,8 @@ urlpatterns = patterns('', 'packages.views.details'), (r'^packages/(?P[A-z0-9\-]+)/(?P[A-z0-9]+)/(?P[A-z0-9\-+.]+)/$', 'packages.views.details'), + (r'^packages/(?P[A-z0-9\-]+)/(?P[A-z0-9]+)/(?P[A-z0-9\-+.]+)/files/$', + 'packages.views.files'), (r'^packages/(?P[A-z0-9\-]+)/(?P[A-z0-9]+)/(?P[A-z0-9\-+.]+)/maintainer/$', 'packages.views.getmaintainer'), -- cgit v1.2.3-55-g3dc8 From 08f67821d1c2002711f519b2f7a09c7a8b5c8c2e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 11 Feb 2010 22:16:58 -0600 Subject: Make the package files view look better Make it look more like the dependencies and required-by panes on the main package details page. Some day you might even find it shows up below there too via an AJAX call or something. Signed-off-by: Dan McGee --- templates/packages/files.html | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/templates/packages/files.html b/templates/packages/files.html index 145bcf24..dc0c64b7 100644 --- a/templates/packages/files.html +++ b/templates/packages/files.html @@ -1,11 +1,16 @@ {% extends "base.html" %} -{% block title %}Pkg: {{ pkg.pkgname }} - Arch Linux Package File List{% endblock %} +{% block title %}Arch Linux - Package File List - {{ pkg.pkgname }}{% endblock %} {% block content %}
-

Viewing Files: {{ pkg.pkgname }} {{ pkg.pkgver }}-{{ pkg.pkgrel }}

- {% for file in files %} - {{ file.path }}
- {% endfor %} +

{{ pkg.pkgname }} {{ pkg.pkgver }}-{{ pkg.pkgrel }}

+
+

Files:

+
    + {% for file in files %} +
  • {{ file.path }}
    • + {% endfor %} +
+
{% endblock %} -- cgit v1.2.3-55-g3dc8 From e119c75838a0c91fd63c9b2aa2bb3940cd6eaee5 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 11 Feb 2010 22:18:02 -0600 Subject: Re-add link to package files Put the link that has been commented back on the page, and point it at the new URL for package files. Also fix the page title to be more in line with all the other pages on the site. Signed-off-by: Dan McGee --- templates/packages/details.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/templates/packages/details.html b/templates/packages/details.html index 5c3c9497..f696231a 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -1,6 +1,5 @@ {% extends "base.html" %} -{% load package_extras %} -{% block title %}Pkg: {{ pkg.pkgname }} - Arch Linux Package Details{% endblock %} +{% block title %}Arch Linux - Package Details - {{ pkg.pkgname }}{% endblock %} {% block content %}

{{ pkg.pkgname }} {{ pkg.pkgver }}-{{ pkg.pkgrel }}

@@ -9,7 +8,7 @@
  • SVN Entries ({{pkg.repo|lower}}-{{pkg.arch}})
  • SVN Entries (trunk)
  • Bug Reports
    • - +
  • View File List
  • {% if pkg.needupdate %} This package has been flagged out-of-date -- cgit v1.2.3-55-g3dc8 From 1cea5fc32e5fe213137cac0323d27bcc9cbc7d8b Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Thu, 11 Feb 2010 22:23:47 -0600 Subject: Order the package files when viewing Signed-off-by: Dan McGee --- packages/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/views.py b/packages/views.py index acd71b2d..9956b6ce 100644 --- a/packages/views.py +++ b/packages/views.py @@ -181,7 +181,7 @@ def search(request, page=None): def files(request, name='', repo='', arch=''): pkg = get_object_or_404(Package, pkgname=name, repo__name__iexact=repo, arch__name=arch) - files = PackageFile.objects.filter(pkg=pkg) + files = PackageFile.objects.filter(pkg=pkg).order_by('path') return render_to_response('packages/files.html', RequestContext(request, {'pkg':pkg,'files':files})) @permission_required('main.change_package') -- cgit v1.2.3-55-g3dc8 From 8314777c5858b8f9dd954ef906e19de1322a61d0 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:08:04 -0600 Subject: Make files view AJAX if supported This will put the filelist inline on the package details page if using a capable browser. It should still fallback to a separate page if necessary (e.g. all those users using links on the site). Signed-off-by: Dan McGee --- packages/views.py | 5 ++++- templates/packages/details.html | 19 ++++++++++++++++++- templates/packages/files-ajax.html | 8 ++++++++ 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 templates/packages/files-ajax.html diff --git a/packages/views.py b/packages/views.py index 9956b6ce..c5f87865 100644 --- a/packages/views.py +++ b/packages/views.py @@ -182,7 +182,10 @@ def files(request, name='', repo='', arch=''): pkg = get_object_or_404(Package, pkgname=name, repo__name__iexact=repo, arch__name=arch) files = PackageFile.objects.filter(pkg=pkg).order_by('path') - return render_to_response('packages/files.html', RequestContext(request, {'pkg':pkg,'files':files})) + template = 'packages/files.html' + if request.is_ajax(): + template = 'packages/files-ajax.html' + return render_to_response(template, RequestContext(request, {'pkg':pkg,'files':files})) @permission_required('main.change_package') def unflag(request, pkgid): diff --git a/templates/packages/details.html b/templates/packages/details.html index f696231a..7e5216e3 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -8,7 +8,6 @@
  • SVN Entries ({{pkg.repo|lower}}-{{pkg.arch}})
  • SVN Entries (trunk)
  • Bug Reports
    • -
  • View File List
  • {% if pkg.needupdate %} This package has been flagged out-of-date @@ -102,6 +101,24 @@ +
    +

    Files:

    + +
    • + + {% endblock %} diff --git a/templates/packages/files-ajax.html b/templates/packages/files-ajax.html new file mode 100644 index 00000000..9b4c83ad --- /dev/null +++ b/templates/packages/files-ajax.html @@ -0,0 +1,8 @@ +
    +

    Files:

    +
      + {% for file in files %} +
    • {{ file.path }}
      • + {% endfor %} +
    +
    -- cgit v1.2.3-55-g3dc8 From 1c073bea62ce854448d15bac05362ec8ecba763e Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:20:17 -0600 Subject: Ensure our cache is correct with AJAX requests Since the same URLs serve two different responses based on the request being AJAX or not, we want to ensure we don't cache the wrong one and serve it up incorrectly. Signed-off-by: Dan McGee --- packages/views.py | 2 ++ todolists/views.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/views.py b/packages/views.py index c5f87865..e95d4a2b 100644 --- a/packages/views.py +++ b/packages/views.py @@ -7,6 +7,7 @@ from django.shortcuts import get_object_or_404 from django.contrib.auth.models import User from django.contrib.auth.decorators import permission_required from django.contrib.admin.widgets import AdminDateWidget +from django.views.decorators.vary import vary_on_headers from django.views.generic import list_detail from django.db.models import Q @@ -178,6 +179,7 @@ def search(request, page=None): template_object_name="package", extra_context=page_dict) +@vary_on_headers('X-Requested-With') def files(request, name='', repo='', arch=''): pkg = get_object_or_404(Package, pkgname=name, repo__name__iexact=repo, arch__name=arch) diff --git a/todolists/views.py b/todolists/views.py index a38ec0d7..8358e4c6 100644 --- a/todolists/views.py +++ b/todolists/views.py @@ -5,6 +5,7 @@ from django.template import RequestContext from django.core.mail import send_mail from django.shortcuts import get_object_or_404, render_to_response from django.contrib.auth.decorators import login_required, permission_required +from django.views.decorators.vary import vary_on_headers from django.views.generic.create_update import delete_object from django.template import Context, loader from django.utils import simplejson @@ -31,6 +32,7 @@ class TodoListForm(forms.Form): @login_required +@vary_on_headers('X-Requested-With') def flag(request, listid, pkgid): list = get_object_or_404(Todolist, id=listid) pkg = get_object_or_404(TodolistPkg, id=pkgid) -- cgit v1.2.3-55-g3dc8 From 7e1e5a5e8a2f3231d0878612508aba06f4397024 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 10:31:55 -0600 Subject: files: template reuse Get rid of the copy/paste by including the sub-template. Signed-off-by: Dan McGee --- packages/views.py | 2 +- templates/packages/files-ajax.html | 8 -------- templates/packages/files-list.html | 8 ++++++++ templates/packages/files.html | 9 +-------- 4 files changed, 10 insertions(+), 17 deletions(-) delete mode 100644 templates/packages/files-ajax.html create mode 100644 templates/packages/files-list.html diff --git a/packages/views.py b/packages/views.py index e95d4a2b..8a2b4ec3 100644 --- a/packages/views.py +++ b/packages/views.py @@ -186,7 +186,7 @@ def files(request, name='', repo='', arch=''): files = PackageFile.objects.filter(pkg=pkg).order_by('path') template = 'packages/files.html' if request.is_ajax(): - template = 'packages/files-ajax.html' + template = 'packages/files-list.html' return render_to_response(template, RequestContext(request, {'pkg':pkg,'files':files})) @permission_required('main.change_package') diff --git a/templates/packages/files-ajax.html b/templates/packages/files-ajax.html deleted file mode 100644 index 9b4c83ad..00000000 --- a/templates/packages/files-ajax.html +++ /dev/null @@ -1,8 +0,0 @@ -
    -

    Files:

    -
      - {% for file in files %} -
    • {{ file.path }}
      • - {% endfor %} -
    -
    diff --git a/templates/packages/files-list.html b/templates/packages/files-list.html new file mode 100644 index 00000000..9b4c83ad --- /dev/null +++ b/templates/packages/files-list.html @@ -0,0 +1,8 @@ +
    +

    Files:

    +
      + {% for file in files %} +
    • {{ file.path }}
      • + {% endfor %} +
    +
    diff --git a/templates/packages/files.html b/templates/packages/files.html index dc0c64b7..1d87246c 100644 --- a/templates/packages/files.html +++ b/templates/packages/files.html @@ -3,14 +3,7 @@ {% block content %}

    {{ pkg.pkgname }} {{ pkg.pkgver }}-{{ pkg.pkgrel }}

    -
    -

    Files:

    -
      - {% for file in files %} -
    • {{ file.path }}
      • - {% endfor %} -
    -
    + {% include "packages/files-list.html" %}
    {% endblock %} -- cgit v1.2.3-55-g3dc8 From 8eff04788d0c62af01848a22a91efe74b87380b2 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 12:34:36 -0600 Subject: reporead: support reading files entries again This depends on some changes I made to our script that generates the file list databases, but it allows us to treat the files databases in an almost identical manner to a regular database. The only difference is the fact that it contains 'files' entries. One catch that will be addressed in a separate patch: if the files DB lags behind the regular DB, running an update from it could cause packages in the web interface to be downgraded. A 'no-add/remove' option could be helpful for this case. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index 3ff6f596..baa81d94 100755 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -6,7 +6,7 @@ Parses a repo.db.tar.gz file and updates the Arch database with the relevant changes. Usage: ./manage.py reporead ARCH PATH - ARCH: architecture to update, and can be one of: i686, x86_64 + ARCH: architecture to update; must be available in the database PATH: full path to the repo.db.tar.gz file. Example: @@ -153,9 +153,14 @@ def populate_pkg(dbpkg, repopkg, timestamp=None): dbpkg.needupdate = False dbpkg.last_update = timestamp dbpkg.save() - # files are not in the repo.db.tar.gz - #for x in repopkg.files: - # dbpkg.packagefile_set.create(path=x) + + # only delete files if we are reading a DB that contains them + if 'files' in repopkg.__dict__: + dbpkg.packagefile_set.all().delete() + logger.debug("adding %d files for package %s" % (len(repopkg.files), dbpkg.pkgname)) + for x in repopkg.files: + dbpkg.packagefile_set.create(path=x) + dbpkg.packagedepend_set.all().delete() if 'depends' in repopkg.__dict__: for y in repopkg.depends: @@ -297,13 +302,18 @@ def parse_repo(repopath): logger.info("Reading repo tarfile %s", repopath) filename = os.path.split(repopath)[1] - rindex = filename.rindex('.db.tar.gz') - reponame = filename[:rindex] - + m = re.match(r"^(.*)\.(db|files)\.tar\.(.*)$", filename) + if m: + reponame = m.group(1) + else: + logger.error("File does not have the proper extension") + raise SomethingFishyException("File does not have the proper extension") + repodb = tarfile.open(repopath,"r:gz") ## assuming well formed tar, with dir first then files after ## repo-add enforces this logger.debug("Starting package parsing") + dbfiles = ('desc', 'depends', 'files') pkgs = [] tpkg = None while True: @@ -321,7 +331,8 @@ def parse_repo(repopath): # set new tpkg tpkg = StringIO() if tarinfo.isreg(): - if os.path.split(tarinfo.name)[1] in ('desc','depends'): + fname = os.path.split(tarinfo.name)[1] + if fname in dbfiles: tpkg.write(repodb.extractfile(tarinfo).read()) tpkg.write('\n') # just in case repodb.close() -- cgit v1.2.3-55-g3dc8 From ffa7ea1b1f276ba146be5a8533a1125ee947e433 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 12:44:18 -0600 Subject: Show message if no files available Signed-off-by: Dan McGee --- templates/packages/details.html | 6 +++--- templates/packages/files-list.html | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/templates/packages/details.html b/templates/packages/details.html index 7e5216e3..94e9f161 100644 --- a/templates/packages/details.html +++ b/templates/packages/details.html @@ -103,9 +103,9 @@

    Files:

    - +

    + View File List +

    diff --git a/templates/packages/files-list.html b/templates/packages/files-list.html index 9b4c83ad..d26a11e9 100644 --- a/templates/packages/files-list.html +++ b/templates/packages/files-list.html @@ -1,8 +1,12 @@

    Files:

    + {% if files.count %}
      {% for file in files %}
    • {{ file.path }}
      • {% endfor %}
    + {% else %} +

    No filelist available.

    + {% endif %}
    -- cgit v1.2.3-55-g3dc8 From 512f20ed034d4e5e7e81fea9c271a957cc768dfe Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 13:21:18 -0600 Subject: Add a files_last_update column This is necessary to keep all of our junk in sync since we aren't guaranteed to have an up to date files database all the time. Signed-off-by: Dan McGee --- main/migrations/0007_add_files_last_update.py | 193 ++++++++++++++++++++++++++ main/models.py | 1 + 2 files changed, 194 insertions(+) create mode 100644 main/migrations/0007_add_files_last_update.py diff --git a/main/migrations/0007_add_files_last_update.py b/main/migrations/0007_add_files_last_update.py new file mode 100644 index 00000000..36f99c26 --- /dev/null +++ b/main/migrations/0007_add_files_last_update.py @@ -0,0 +1,193 @@ +from south.db import db +from django.db import models +from main.models import * + +class Migration: + def forwards(self, orm): + # Adding field 'Package.files_last_update' + db.add_column('packages', 'files_last_update', orm['main.package:files_last_update']) + + def backwards(self, orm): + # Deleting field 'Package.files_last_update' + db.delete_column('packages', 'files_last_update') + + models = { + 'auth.group': { + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), + 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}) + }, + 'auth.permission': { + 'Meta': {'unique_together': "(('content_type', 'codename'),)"}, + 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) + }, + 'auth.user': { + 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), + 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), + 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), + 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}), + 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) + }, + 'contenttypes.contenttype': { + 'Meta': {'unique_together': "(('app_label', 'model'),)", 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + 'main.altforum': { + 'Meta': {'db_table': "'alt_forums'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'language': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.arch': { + 'Meta': {'db_table': "'arches'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.donor': { + 'Meta': {'db_table': "'donors'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.externalproject': { + 'description': ('django.db.models.fields.CharField', [], {'max_length': '128'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '64'}), + 'url': ('django.db.models.fields.URLField', [], {'max_length': '200'}) + }, + 'main.mirror': { + 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'admin_email': ('django.db.models.fields.EmailField', [], {'max_length': '255', 'blank': 'True'}), + 'country': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'isos': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'notes': ('django.db.models.fields.CharField', [], {'max_length': '255', 'blank': 'True'}), + 'public': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}) + }, + 'main.mirrorprotocol': { + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'protocol': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '10'}) + }, + 'main.mirrorrsync': { + 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'ip': ('django.db.models.fields.CharField', [], {'max_length': '24'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'rsync_ips'", 'to': "orm['main.Mirror']"}) + }, + 'main.mirrorurl': { + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'mirror': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['main.Mirror']"}), + 'protocol': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'urls'", 'to': "orm['main.MirrorProtocol']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.news': { + 'Meta': {'db_table': "'news'"}, + 'author': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'news_author'", 'to': "orm['auth.User']"}), + 'content': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'postdate': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'title': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.package': { + 'Meta': {'db_table': "'packages'"}, + 'arch': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Arch']"}), + 'build_date': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), + 'compressed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'files_last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'installed_size': ('django.db.models.fields.PositiveIntegerField', [], {'null': 'True'}), + 'last_update': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), + 'license': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'maintainer': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'maintained_packages'", 'null': 'True', 'to': "orm['auth.User']"}), + 'needupdate': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'pkgbase': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'pkgdesc': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'repo': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'packages'", 'to': "orm['main.Repo']"}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.packagedepend': { + 'Meta': {'db_table': "'package_depends'"}, + 'depname': ('django.db.models.fields.CharField', [], {'max_length': '255', 'db_index': 'True'}), + 'depvcmp': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.packagefile': { + 'Meta': {'db_table': "'package_files'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'path': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.press': { + 'Meta': {'db_table': "'press'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'url': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.repo': { + 'Meta': {'db_table': "'repos'"}, + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '255'}) + }, + 'main.signoff': { + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'packager': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}), + 'pkgrel': ('django.db.models.fields.CharField', [], {'max_length': '255'}), + 'pkgver': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolist': { + 'Meta': {'db_table': "'todolists'"}, + 'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), + 'date_added': ('django.db.models.fields.DateField', [], {'auto_now_add': 'True', 'blank': 'True'}), + 'description': ('django.db.models.fields.TextField', [], {}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '255'}) + }, + 'main.todolistpkg': { + 'Meta': {'unique_together': "(('list', 'pkg'),)", 'db_table': "'todolist_pkgs'"}, + 'complete': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'list': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Todolist']"}), + 'pkg': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['main.Package']"}) + }, + 'main.userprofile': { + 'Meta': {'db_table': "'user_profiles'"}, + 'alias': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'allowed_repos': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['main.Repo']", 'blank': 'True'}), + 'favorite_distros': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'interests': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'languages': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'location': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'notify': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), + 'occupation': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), + 'other_contact': ('django.db.models.fields.CharField', [], {'max_length': '100', 'null': 'True', 'blank': 'True'}), + 'picture': ('django.db.models.fields.files.FileField', [], {'default': "'devs/silhouette.png'", 'max_length': '100'}), + 'public_email': ('django.db.models.fields.CharField', [], {'max_length': '50'}), + 'roles': ('django.db.models.fields.CharField', [], {'max_length': '255', 'null': 'True', 'blank': 'True'}), + 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'userprofile_user'", 'unique': 'True', 'to': "orm['auth.User']"}), + 'website': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}), + 'yob': ('django.db.models.fields.IntegerField', [], {'null': 'True', 'blank': 'True'}) + } + } + + complete_apps = ['main'] diff --git a/main/models.py b/main/models.py index 373c3e00..dd37f885 100644 --- a/main/models.py +++ b/main/models.py @@ -174,6 +174,7 @@ class Package(models.Model): installed_size = models.PositiveIntegerField(null=True) build_date = models.DateTimeField(null=True) last_update = models.DateTimeField(null=True, blank=True) + files_last_update = models.DateTimeField(null=True, blank=True) license = models.CharField(max_length=255) objects = PackageManager() class Meta: -- cgit v1.2.3-55-g3dc8 From ab459d4d182cc56bbaf24b0e483e83267a9df432 Mon Sep 17 00:00:00 2001 From: Dan McGee Date: Sat, 27 Feb 2010 13:25:28 -0600 Subject: reporead: add --filesonly option This will allow files to be imported for all existing packages in the database while not worrying about the files database being a touch out of date. It utilizes the new files_last_update column to perform the insertion and updating of file lists intelligently. Signed-off-by: Dan McGee --- devel/management/commands/reporead.py | 73 ++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 26 deletions(-) diff --git a/devel/management/commands/reporead.py b/devel/management/commands/reporead.py index baa81d94..34b291eb 100755 --- a/devel/management/commands/reporead.py +++ b/devel/management/commands/reporead.py @@ -55,6 +55,8 @@ class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option('-f', '--force', action='store_true', dest='force', default=False, help='Force a re-import of data for all packages instead of only new ones. Will not touch the \'last updated\' value.'), + make_option('--filesonly', action='store_true', dest='filesonly', default=False, + help='Load filelists if they are outdated, but will not add or remove any packages. Will not touch the \'last updated\' value.'), ) help = "Runs a package repository import for the given arch and file." args = " " @@ -132,7 +134,7 @@ class Pkg(object): return None -def populate_pkg(dbpkg, repopkg, timestamp=None): +def populate_pkg(dbpkg, repopkg, force=False, timestamp=None): dbpkg.pkgbase = repopkg.base dbpkg.pkgver = repopkg.ver dbpkg.pkgrel = repopkg.rel @@ -154,12 +156,7 @@ def populate_pkg(dbpkg, repopkg, timestamp=None): dbpkg.last_update = timestamp dbpkg.save() - # only delete files if we are reading a DB that contains them - if 'files' in repopkg.__dict__: - dbpkg.packagefile_set.all().delete() - logger.debug("adding %d files for package %s" % (len(repopkg.files), dbpkg.pkgname)) - for x in repopkg.files: - dbpkg.packagefile_set.create(path=x) + populate_files(dbpkg, repopkg, force=force) dbpkg.packagedepend_set.all().delete() if 'depends' in repopkg.__dict__: @@ -173,8 +170,22 @@ def populate_pkg(dbpkg, repopkg, timestamp=None): dbpkg.packagedepend_set.create(depname=dpname, depvcmp=dpvcmp) logger.debug('Added %s as dep for pkg %s' % (dpname,repopkg.name)) +def populate_files(dbpkg, repopkg, force=False): + if not force: + if not dbpkg.files_last_update or not dbpkg.last_update: + pass + elif dbpkg.files_last_update > dbpkg.last_update: + return + # only delete files if we are reading a DB that contains them + if 'files' in repopkg.__dict__: + dbpkg.packagefile_set.all().delete() + logger.info("adding %d files for package %s" % (len(repopkg.files), dbpkg.pkgname)) + for x in repopkg.files: + dbpkg.packagefile_set.create(path=x) + dbpkg.files_last_update = datetime.now() + dbpkg.save() -def db_update(archname, pkgs, force): +def db_update(archname, pkgs, options): """ Parses a list and updates the Arch dev database accordingly. @@ -183,6 +194,8 @@ def db_update(archname, pkgs, force): """ logger.info('Updating Arch: %s' % archname) + force = options.get('force', False) + filesonly = options.get('filesonly', False) repository = Repo.objects.get(name__iexact=pkgs[0].repo) architecture = Arch.objects.get(name__iexact=archname) dbpkgs = Package.objects.filter(arch=architecture, repo=repository) @@ -222,19 +235,22 @@ def db_update(archname, pkgs, force): if dbpercent < 75.0: logger.warning(".db.tar.gz has %.1f%% the number of packages in the web database." % dbpercent) - - for p in [x for x in pkgs if x.name in in_sync_not_db]: - logger.info("Adding package %s", p.name) - pkg = Package(pkgname = p.name, arch = architecture, repo = repository) - populate_pkg(pkg, p, timestamp=datetime.now()) - - # packages in database and not in syncdb (remove from database) - logger.debug("Set theory: Packages in database not in syncdb") - in_db_not_sync = dbset - syncset - for p in in_db_not_sync: - logger.info("Removing package %s from database", p) - Package.objects.get( - pkgname=p, arch=architecture, repo=repository).delete() + + if not filesonly: + # packages in syncdb and not in database (add to database) + logger.debug("Set theory: Packages in syncdb not in database") + for p in [x for x in pkgs if x.name in in_sync_not_db]: + logger.info("Adding package %s", p.name) + pkg = Package(pkgname = p.name, arch = architecture, repo = repository) + populate_pkg(pkg, p, timestamp=datetime.now()) + + # packages in database and not in syncdb (remove from database) + logger.debug("Set theory: Packages in database not in syncdb") + in_db_not_sync = dbset - syncset + for p in in_db_not_sync: + logger.info("Removing package %s from database", p) + Package.objects.get( + pkgname=p, arch=architecture, repo=repository).delete() # packages in both database and in syncdb (update in database) logger.debug("Set theory: Packages in database and syncdb") @@ -245,15 +261,21 @@ def db_update(archname, pkgs, force): timestamp = None # for a force, we don't want to update the timestamp. # for a non-force, we don't want to do anything at all. - if ''.join((p.ver,p.rel)) == ''.join((dbp.pkgver,dbp.pkgrel)): + if filesonly: + pass + elif ''.join((p.ver,p.rel)) == ''.join((dbp.pkgver,dbp.pkgrel)): if not force: continue else: timestamp = datetime.now() - logger.info("Updating package %s in database", p.name) pkg = Package.objects.get( pkgname=p.name,arch=architecture, repo=repository) - populate_pkg(pkg, p, timestamp=timestamp) + if filesonly: + logger.info("Possibly populating files for package %s in database", p.name) + populate_files(pkg, p) + else: + logger.info("Updating package %s in database", p.name) + populate_pkg(pkg, p, force=force, timestamp=timestamp) logger.info('Finished updating Arch: %s' % archname) @@ -364,11 +386,10 @@ def read_repo(primary_arch, file, options): logger.warning("Package %s arch = %s" % ( package.name,package.arch)) #package.arch = primary_arch - f = options.get('force', False) logger.info('Starting database updates.') for (arch, pkgs) in packages_arches.items(): if len(pkgs) > 0: - db_update(arch, pkgs, f) + db_update(arch, pkgs, options) logger.info('Finished database updates.') return 0 -- cgit v1.2.3-55-g3dc8