From 063784db760a85236068a8bbacd01af2d1036d4b Mon Sep 17 00:00:00 2001 From: Angel Velasquez Date: Wed, 24 May 2017 14:15:12 -0400 Subject: Fix broken reports This should fix reports that were broken cause username was passed and it wasn't needed. Also some format changes were done. --- devel/reports.py | 156 +++++++++++++++++++++++++++++-------------------------- devel/views.py | 143 +++++++++++++++++++++++++------------------------- 2 files changed, 152 insertions(+), 147 deletions(-) diff --git a/devel/reports.py b/devel/reports.py index b754b264..3d8b0fbd 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -1,17 +1,24 @@ from datetime import timedelta -import pytz +import pytz from django.db.models import F from django.template.defaultfilters import filesizeformat from django.utils.timezone import now +from main.models import PackageFile +from packages.models import Depend, PackageRelation from .models import DeveloperKey -from main.models import PackageFile -from packages.models import PackageRelation, Depend + class DeveloperReport(object): - def __init__(self, slug, name, desc, packages_func, - names=None, attrs=None, personal=True): + def __init__(self, + slug, + name, + desc, + packages_func, + names=None, + attrs=None, + personal=True): self.slug = slug self.name = name self.description = desc @@ -23,41 +30,38 @@ class DeveloperReport(object): def old(packages): cutoff = now() - timedelta(days=365 * 2) - return packages.filter( - build_date__lt=cutoff).order_by('build_date') + return packages.filter(build_date__lt=cutoff).order_by('build_date') def outofdate(packages): cutoff = now() - timedelta(days=30) - return packages.filter( - flag_date__lt=cutoff).order_by('flag_date') + return packages.filter(flag_date__lt=cutoff).order_by('flag_date') def big(packages): cutoff = 50 * 1024 * 1024 packages = packages.filter( - compressed_size__gte=cutoff).order_by('-compressed_size') + compressed_size__gte=cutoff).order_by('-compressed_size') # Format the compressed and installed sizes with MB/GB/etc suffixes for package in packages: package.compressed_size_pretty = filesizeformat( package.compressed_size) - package.installed_size_pretty = filesizeformat( - package.installed_size) + package.installed_size_pretty = filesizeformat(package.installed_size) return packages def badcompression(packages): cutoff = 0.90 * F('installed_size') - packages = packages.filter(compressed_size__gt=25*1024, - installed_size__gt=25*1024, - compressed_size__gte=cutoff).order_by('-compressed_size') + packages = packages.filter( + compressed_size__gt=25 * 1024, + installed_size__gt=25 * 1024, + compressed_size__gte=cutoff).order_by('-compressed_size') # Format the compressed and installed sizes with MB/GB/etc suffixes for package in packages: package.compressed_size_pretty = filesizeformat( package.compressed_size) - package.installed_size_pretty = filesizeformat( - package.installed_size) + package.installed_size_pretty = filesizeformat(package.installed_size) ratio = package.compressed_size / float(package.installed_size) package.ratio = '%.3f' % ratio package.compress_type = package.filename.split('.')[-1] @@ -67,18 +71,17 @@ def badcompression(packages): def uncompressed_man(packages, username): # checking for all '.0'...'.9' + '.n' extensions - bad_files = PackageFile.objects.filter(is_directory=False, - directory__contains='/man/', - filename__regex=r'\.[0-9n]').exclude( - filename__endswith='.gz').exclude( + bad_files = PackageFile.objects.filter( + is_directory=False, + directory__contains='/man/', + filename__regex=r'\.[0-9n]').exclude(filename__endswith='.gz').exclude( filename__endswith='.xz').exclude( - filename__endswith='.bz2').exclude( - filename__endswith='.html') + filename__endswith='.bz2').exclude(filename__endswith='.html') if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list( - 'pkg_id', flat=True).order_by().distinct() + bad_files = bad_files.values_list('pkg_id', + flat=True).order_by().distinct() return packages.filter(id__in=set(bad_files)) @@ -86,12 +89,13 @@ def uncompressed_info(packages, username): # we don't worry about looking for '*.info-1', etc., given that an # uncompressed root page probably exists in the package anyway bad_files = PackageFile.objects.filter(is_directory=False, - directory__endswith='/info/', filename__endswith='.info') + directory__endswith='/info/', + filename__endswith='.info') if username: pkg_ids = set(packages.values_list('id', flat=True)) bad_files = bad_files.filter(pkg__in=pkg_ids) - bad_files = bad_files.values_list( - 'pkg_id', flat=True).order_by().distinct() + bad_files = bad_files.values_list('pkg_id', + flat=True).order_by().distinct() return packages.filter(id__in=set(bad_files)) @@ -99,16 +103,15 @@ def unneeded_orphans(packages): owned = PackageRelation.objects.all().values('pkgbase') required = Depend.objects.all().values('name') # The two separate calls to exclude is required to do the right thing - return packages.exclude(pkgbase__in=owned).exclude( - pkgname__in=required) + return packages.exclude(pkgbase__in=owned).exclude(pkgname__in=required) def mismatched_signature(packages): filtered = [] packages = packages.select_related( - 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) - known_keys = DeveloperKey.objects.select_related( - 'owner').filter(owner__isnull=False) + 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) + known_keys = DeveloperKey.objects.select_related('owner').filter( + owner__isnull=False) known_keys = {dk.key: dk for dk in known_keys} for package in packages: bad = False @@ -131,7 +134,7 @@ def signature_time(packages): cutoff = timedelta(hours=24) filtered = [] packages = packages.select_related( - 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) + 'arch', 'repo', 'packager').filter(signature_bytes__isnull=False) for package in packages: sig = package.signature sig_date = sig.creation_time.replace(tzinfo=pytz.utc) @@ -142,57 +145,60 @@ def signature_time(packages): return filtered -REPORT_OLD = DeveloperReport('old', 'Old', - 'Packages last built more than two years ago', old) +REPORT_OLD = DeveloperReport( + 'old', 'Old', 'Packages last built more than two years ago', old) -REPORT_OUTOFDATE = DeveloperReport('long-out-of-date', 'Long Out-of-date', - 'Packages marked out-of-date more than 30 days ago', outofdate) +REPORT_OUTOFDATE = DeveloperReport( + 'long-out-of-date', 'Long Out-of-date', + 'Packages marked out-of-date more than 30 days ago', outofdate) -REPORT_BIG = DeveloperReport('big', 'Big', - 'Packages with compressed size > 50 MiB', big, - ['Compressed Size', 'Installed Size'], - ['compressed_size_pretty', 'installed_size_pretty']) +REPORT_BIG = DeveloperReport( + 'big', 'Big', 'Packages with compressed size > 50 MiB', big, + ['Compressed Size', 'Installed Size'], + ['compressed_size_pretty', 'installed_size_pretty']) -REPORT_BADCOMPRESS = DeveloperReport('badcompression', 'Bad Compression', - 'Packages > 25 KiB with a compression ratio < 10%', badcompression, - ['Compressed Size', 'Installed Size', 'Ratio', 'Type'], - ['compressed_size_pretty', 'installed_size_pretty','ratio', 'compress_type']) +REPORT_BADCOMPRESS = DeveloperReport( + 'badcompression', 'Bad Compression', + 'Packages > 25 KiB with a compression ratio < 10%', badcompression, + ['Compressed Size', 'Installed Size', 'Ratio', 'Type'], + ['compressed_size_pretty', 'installed_size_pretty', 'ratio', + 'compress_type']) REPORT_MAN = DeveloperReport('uncompressed-man', 'Uncompressed Manpages', - 'Packages with uncompressed manpages', uncompressed_man) + 'Packages with uncompressed manpages', + uncompressed_man) REPORT_INFO = DeveloperReport('uncompressed-info', 'Uncompressed Info Pages', - 'Packages with uncompressed info pages', uncompressed_info) + 'Packages with uncompressed info pages', + uncompressed_info) -REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans', - 'Packages that have no maintainer and are not required by any ' - + 'other package in any repository', unneeded_orphans, - personal=False) +REPORT_ORPHANS = DeveloperReport( + 'unneeded-orphans', + 'Unneeded Orphans', + 'Packages that have no maintainer and are not required by any ' + + 'other package in any repository', + unneeded_orphans, + personal=False) -REPORT_SIGNATURE = DeveloperReport('mismatched-signature', - 'Mismatched Signatures', - 'Packages where the signing key is unknown or signer != packager', - mismatched_signature, - ['Signed By', 'Packager'], - ['sig_by', 'packager']) +REPORT_SIGNATURE = DeveloperReport( + 'mismatched-signature', 'Mismatched Signatures', + 'Packages where the signing key is unknown or signer != packager', + mismatched_signature, ['Signed By', 'Packager'], ['sig_by', 'packager']) -REPORT_SIG_TIME = DeveloperReport('signature-time', 'Signature Time', - 'Packages where the signature timestamp is more than 24 hours ' - + 'after the build timestamp', - signature_time, - ['Signature Date', 'Packager'], - ['sig_date', 'packager']) +REPORT_SIG_TIME = DeveloperReport( + 'signature-time', 'Signature Time', + 'Packages where the signature timestamp is more than 24 hours ' + + 'after the build timestamp', signature_time, + ['Signature Date', 'Packager'], ['sig_date', 'packager']) def available_reports(): - return ( - REPORT_OLD, - REPORT_OUTOFDATE, - REPORT_BIG, - REPORT_BADCOMPRESS, - REPORT_MAN, - REPORT_INFO, - REPORT_ORPHANS, - REPORT_SIGNATURE, - REPORT_SIG_TIME, - ) + return (REPORT_OLD, + REPORT_OUTOFDATE, + REPORT_BIG, + REPORT_BADCOMPRESS, + REPORT_MAN, + REPORT_INFO, + REPORT_ORPHANS, + REPORT_SIGNATURE, + REPORT_SIG_TIME, ) diff --git a/devel/views.py b/devel/views.py index 66f6a965..e86c60c1 100644 --- a/devel/views.py +++ b/devel/views.py @@ -1,33 +1,32 @@ -from datetime import timedelta import operator import time +from datetime import timedelta -from django.http import HttpResponseRedirect -from django.contrib.auth.decorators import \ - login_required, permission_required, user_passes_test from django.contrib import admin -from django.contrib.admin.models import LogEntry, ADDITION +from django.contrib.admin.models import ADDITION, LogEntry +from django.contrib.auth.decorators import (login_required, + permission_required, + user_passes_test) from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import transaction from django.db.models import Count, Max -from django.http import Http404 +from django.http import Http404, HttpResponseRedirect from django.shortcuts import get_object_or_404, render -from django.views.decorators.cache import never_cache from django.utils.encoding import force_unicode from django.utils.http import http_date from django.utils.timezone import now - -from .forms import ProfileForm, UserProfileForm, NewUserForm -from .models import UserProfile -from .reports import available_reports -from main.models import Package -from main.models import Arch, Repo +from django.views.decorators.cache import never_cache +from main.models import Arch, Package, Repo from news.models import News -from packages.models import PackageRelation, Signoff, FlagRequest +from packages.models import FlagRequest, PackageRelation, Signoff from packages.utils import get_signoff_groups from todolists.models import TodolistPackage from todolists.utils import get_annotated_todolists + +from .forms import NewUserForm, ProfileForm, UserProfileForm +from .models import UserProfile +from .reports import available_reports from .utils import get_annotated_maintainers @@ -41,25 +40,26 @@ def index(request): inner_q = inner_q.values('pkgbase') flagged = Package.objects.normal().filter( - flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') + flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname') todopkgs = TodolistPackage.objects.select_related( - 'todolist', 'pkg', 'arch', 'repo').exclude( + 'todolist', 'pkg', 'arch', 'repo').exclude( status=TodolistPackage.COMPLETE).filter(removed__isnull=True) - todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by( - 'todolist__name', 'pkgname') + todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by('todolist__name', + 'pkgname') todolists = get_annotated_todolists(incomplete_only=True) - signoffs = sorted(get_signoff_groups(user=request.user), - key=operator.attrgetter('pkgbase')) + signoffs = sorted( + get_signoff_groups(user=request.user), + key=operator.attrgetter('pkgbase')) page_dict = { - 'todos': todolists, - 'flagged': flagged, - 'todopkgs': todopkgs, - 'signoffs': signoffs, - 'reports': available_reports(), + 'todos': todolists, + 'flagged': flagged, + 'todopkgs': todopkgs, + 'signoffs': signoffs, + 'reports': available_reports(), } return render(request, 'devel/index.html', page_dict) @@ -69,35 +69,37 @@ def index(request): def stats(request): """The second half of the dev dashboard.""" arches = Arch.objects.all().annotate( - total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + total_ct=Count('packages'), + flagged_ct=Count('packages__flag_date')) repos = Repo.objects.all().annotate( - total_ct=Count('packages'), flagged_ct=Count('packages__flag_date')) + total_ct=Count('packages'), + flagged_ct=Count('packages__flag_date')) # the join is huge unless we do this separately, so merge the result here repo_maintainers = dict(Repo.objects.order_by().filter( - userprofile__user__is_active=True).values_list('id').annotate( - Count('userprofile'))) + userprofile__user__is_active=True).values_list('id').annotate(Count( + 'userprofile'))) for repo in repos: repo.maintainer_ct = repo_maintainers.get(repo.id, 0) maintainers = get_annotated_maintainers() maintained = PackageRelation.objects.filter( - type=PackageRelation.MAINTAINER).values('pkgbase') + type=PackageRelation.MAINTAINER).values('pkgbase') total_orphans = Package.objects.exclude(pkgbase__in=maintained).count() total_flagged_orphans = Package.objects.filter( - flag_date__isnull=False).exclude(pkgbase__in=maintained).count() + flag_date__isnull=False).exclude(pkgbase__in=maintained).count() total_updated = Package.objects.filter(packager__isnull=True).count() orphan = { - 'package_count': total_orphans, - 'flagged_count': total_flagged_orphans, - 'updated_count': total_updated, + 'package_count': total_orphans, + 'flagged_count': total_flagged_orphans, + 'updated_count': total_updated, } page_dict = { - 'arches': arches, - 'repos': repos, - 'maintainers': maintainers, - 'orphan': orphan, + 'arches': arches, + 'repos': repos, + 'maintainers': maintainers, + 'orphan': orphan, } return render(request, 'devel/stats.html', page_dict) @@ -106,25 +108,23 @@ def stats(request): @login_required def clock(request): devs = User.objects.filter(is_active=True).order_by( - 'first_name', 'last_name').select_related('userprofile') + 'first_name', 'last_name').select_related('userprofile') - latest_news = dict(News.objects.filter( - author__is_active=True).values_list('author').order_by( - ).annotate(last_post=Max('postdate'))) + latest_news = dict(News.objects.filter(author__is_active=True).values_list( + 'author').order_by().annotate(last_post=Max('postdate'))) latest_package = dict(Package.objects.filter( - packager__is_active=True).values_list('packager').order_by( - ).annotate(last_build=Max('build_date'))) + packager__is_active=True).values_list('packager').order_by().annotate( + last_build=Max('build_date'))) latest_signoff = dict(Signoff.objects.filter( - user__is_active=True).values_list('user').order_by( - ).annotate(last_signoff=Max('created'))) + user__is_active=True).values_list('user').order_by().annotate( + last_signoff=Max('created'))) # The extra() bit ensures we can use our 'user_id IS NOT NULL' index latest_flagreq = dict(FlagRequest.objects.filter( - user__is_active=True).extra( - where=['user_id IS NOT NULL']).values_list('user_id').order_by( - ).annotate(last_flagrequest=Max('created'))) + user__is_active=True).extra(where=['user_id IS NOT NULL']).values_list( + 'user_id').order_by().annotate(last_flagrequest=Max('created'))) latest_log = dict(LogEntry.objects.filter( - user__is_active=True).values_list('user').order_by( - ).annotate(last_log=Max('action_time'))) + user__is_active=True).values_list('user').order_by().annotate( + last_log=Max('action_time'))) for dev in devs: dates = [ @@ -142,10 +142,7 @@ def clock(request): dev.last_action = None current_time = now() - page_dict = { - 'developers': devs, - 'utc_now': current_time, - } + page_dict = {'developers': devs, 'utc_now': current_time, } response = render(request, 'devel/clock.html', page_dict) if not response.has_header('Expires'): @@ -162,8 +159,9 @@ def change_profile(request): profile, _ = UserProfile.objects.get_or_create(user=request.user) if request.POST: form = ProfileForm(request.POST) - profile_form = UserProfileForm(request.POST, request.FILES, - instance=profile) + profile_form = UserProfileForm(request.POST, + request.FILES, + instance=profile) if form.is_valid() and profile_form.is_valid(): request.user.email = form.cleaned_data['email'] if form.cleaned_data['passwd1']: @@ -176,7 +174,8 @@ def change_profile(request): form = ProfileForm(initial={'email': request.user.email}) profile_form = UserProfileForm(instance=profile) return render(request, 'devel/profile.html', - {'form': form, 'profile_form': profile_form}) + {'form': form, + 'profile_form': profile_form}) @login_required @@ -190,14 +189,18 @@ def report(request, report_name, username=None): user = None if username: user = get_object_or_404(User, username=username, is_active=True) - maintained = PackageRelation.objects.filter(user=user, - type=PackageRelation.MAINTAINER).values('pkgbase') + maintained = PackageRelation.objects.filter( + user=user, type=PackageRelation.MAINTAINER).values('pkgbase') packages = packages.filter(pkgbase__in=maintained) maints = User.objects.filter(id__in=PackageRelation.objects.filter( type=PackageRelation.MAINTAINER).values('user')) - packages = report.packages(packages, username) + if report.slug == 'uncompressed-man' or report.slug == 'uncompressed-info': + packages = report.packages(packages, username) + else: + packages = report.packages(packages) + arches = {pkg.arch for pkg in packages} repos = {pkg.repo for pkg in packages} context = { @@ -217,13 +220,12 @@ def report(request, report_name, username=None): def log_addition(request, obj): """Cribbed from ModelAdmin.log_addition.""" LogEntry.objects.log_action( - user_id = request.user.pk, - content_type_id = ContentType.objects.get_for_model(obj).pk, - object_id = obj.pk, - object_repr = force_unicode(obj), - action_flag = ADDITION, - change_message = "Added via Create New User form." - ) + user_id=request.user.pk, + content_type_id=ContentType.objects.get_for_model(obj).pk, + object_id=obj.pk, + object_repr=force_unicode(obj), + action_flag=ADDITION, + change_message="Added via Create New User form.") @permission_required('auth.add_user') @@ -257,10 +259,7 @@ def admin_log(request, username=None): user = None if username: user = get_object_or_404(User, username=username) - context = { - 'title': "Admin Action Log", - 'log_user': user, - } + context = {'title': "Admin Action Log", 'log_user': user, } context.update(admin.site.each_context()) return render(request, 'devel/admin_log.html', context) -- cgit v1.2.3-55-g3dc8