Evolinx/evorepo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix broken reports

Browse Source 
This should fix reports that were broken cause username was passed and
it wasn't needed.

Also some format changes were done.
This commit is contained in:
Angel Velasquez 2017-05-24 14:15:12 -04:00
parent ecc70364bb
commit 063784db76
2 changed files with 152 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

156
devel/reports.py
View File

@ -1,17 +1,24 @@
from datetime import timedelta
import pytz
import pytz
from django.db.models import F
from django.template.defaultfilters import filesizeformat
from django.utils.timezone import now
from main.models import PackageFile
from packages.models import Depend, PackageRelation
from .models import DeveloperKey
from main.models import PackageFile
from packages.models import PackageRelation, Depend
class DeveloperReport(object):
def __init__(self, slug, name, desc, packages_func,
names=None, attrs=None, personal=True):
def __init__(self,
slug,
name,
desc,
packages_func,
names=None,
attrs=None,
personal=True):
self.slug = slug
self.name = name
self.description = desc
@ -23,41 +30,38 @@ def __init__(self, slug, name, desc, packages_func,
def old(packages):
cutoff = now() - timedelta(days=365 * 2)
return packages.filter(
build_date__lt=cutoff).order_by('build_date')
return packages.filter(build_date__lt=cutoff).order_by('build_date')
def outofdate(packages):
cutoff = now() - timedelta(days=30)
return packages.filter(
flag_date__lt=cutoff).order_by('flag_date')
return packages.filter(flag_date__lt=cutoff).order_by('flag_date')
def big(packages):
cutoff = 50 * 1024 * 1024
packages = packages.filter(
compressed_size__gte=cutoff).order_by('-compressed_size')
compressed_size__gte=cutoff).order_by('-compressed_size')
# Format the compressed and installed sizes with MB/GB/etc suffixes
for package in packages:
package.compressed_size_pretty = filesizeformat(
package.compressed_size)
package.installed_size_pretty = filesizeformat(
package.installed_size)
package.installed_size_pretty = filesizeformat(package.installed_size)
return packages
def badcompression(packages):
cutoff = 0.90 * F('installed_size')
packages = packages.filter(compressed_size__gt=25*1024,
installed_size__gt=25*1024,
compressed_size__gte=cutoff).order_by('-compressed_size')
packages = packages.filter(
compressed_size__gt=25 * 1024,
installed_size__gt=25 * 1024,
compressed_size__gte=cutoff).order_by('-compressed_size')
# Format the compressed and installed sizes with MB/GB/etc suffixes
for package in packages:
package.compressed_size_pretty = filesizeformat(
package.compressed_size)
package.installed_size_pretty = filesizeformat(
package.installed_size)
package.installed_size_pretty = filesizeformat(package.installed_size)
ratio = package.compressed_size / float(package.installed_size)
package.ratio = '%.3f' % ratio
package.compress_type = package.filename.split('.')[-1]
@ -67,18 +71,17 @@ def badcompression(packages):
def uncompressed_man(packages, username):
# checking for all '.0'...'.9' + '.n' extensions
bad_files = PackageFile.objects.filter(is_directory=False,
directory__contains='/man/',
filename__regex=r'\.[0-9n]').exclude(
filename__endswith='.gz').exclude(
bad_files = PackageFile.objects.filter(
is_directory=False,
directory__contains='/man/',
filename__regex=r'\.[0-9n]').exclude(filename__endswith='.gz').exclude(
filename__endswith='.xz').exclude(
filename__endswith='.bz2').exclude(
filename__endswith='.html')
filename__endswith='.bz2').exclude(filename__endswith='.html')
if username:
pkg_ids = set(packages.values_list('id', flat=True))
bad_files = bad_files.filter(pkg__in=pkg_ids)
bad_files = bad_files.values_list(
'pkg_id', flat=True).order_by().distinct()
bad_files = bad_files.values_list('pkg_id',
flat=True).order_by().distinct()
return packages.filter(id__in=set(bad_files))
@ -86,12 +89,13 @@ def uncompressed_info(packages, username):
# we don't worry about looking for '*.info-1', etc., given that an
# uncompressed root page probably exists in the package anyway
bad_files = PackageFile.objects.filter(is_directory=False,
directory__endswith='/info/', filename__endswith='.info')
directory__endswith='/info/',
filename__endswith='.info')
if username:
pkg_ids = set(packages.values_list('id', flat=True))
bad_files = bad_files.filter(pkg__in=pkg_ids)
bad_files = bad_files.values_list(
'pkg_id', flat=True).order_by().distinct()
bad_files = bad_files.values_list('pkg_id',
flat=True).order_by().distinct()
return packages.filter(id__in=set(bad_files))
@ -99,16 +103,15 @@ def unneeded_orphans(packages):
owned = PackageRelation.objects.all().values('pkgbase')
required = Depend.objects.all().values('name')
# The two separate calls to exclude is required to do the right thing
return packages.exclude(pkgbase__in=owned).exclude(
pkgname__in=required)
return packages.exclude(pkgbase__in=owned).exclude(pkgname__in=required)
def mismatched_signature(packages):
filtered = []
packages = packages.select_related(
'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
known_keys = DeveloperKey.objects.select_related(
'owner').filter(owner__isnull=False)
'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
known_keys = DeveloperKey.objects.select_related('owner').filter(
owner__isnull=False)
known_keys = {dk.key: dk for dk in known_keys}
for package in packages:
bad = False
@ -131,7 +134,7 @@ def signature_time(packages):
cutoff = timedelta(hours=24)
filtered = []
packages = packages.select_related(
'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
'arch', 'repo', 'packager').filter(signature_bytes__isnull=False)
for package in packages:
sig = package.signature
sig_date = sig.creation_time.replace(tzinfo=pytz.utc)
@ -142,57 +145,60 @@ def signature_time(packages):
return filtered
REPORT_OLD = DeveloperReport('old', 'Old',
'Packages last built more than two years ago', old)
REPORT_OLD = DeveloperReport(
'old', 'Old', 'Packages last built more than two years ago', old)
REPORT_OUTOFDATE = DeveloperReport('long-out-of-date', 'Long Out-of-date',
'Packages marked out-of-date more than 30 days ago', outofdate)
REPORT_OUTOFDATE = DeveloperReport(
'long-out-of-date', 'Long Out-of-date',
'Packages marked out-of-date more than 30 days ago', outofdate)
REPORT_BIG = DeveloperReport('big', 'Big',
'Packages with compressed size > 50 MiB', big,
['Compressed Size', 'Installed Size'],
['compressed_size_pretty', 'installed_size_pretty'])
REPORT_BIG = DeveloperReport(
'big', 'Big', 'Packages with compressed size > 50 MiB', big,
['Compressed Size', 'Installed Size'],
['compressed_size_pretty', 'installed_size_pretty'])
REPORT_BADCOMPRESS = DeveloperReport('badcompression', 'Bad Compression',
'Packages > 25 KiB with a compression ratio < 10%', badcompression,
['Compressed Size', 'Installed Size', 'Ratio', 'Type'],
['compressed_size_pretty', 'installed_size_pretty','ratio', 'compress_type'])
REPORT_BADCOMPRESS = DeveloperReport(
'badcompression', 'Bad Compression',
'Packages > 25 KiB with a compression ratio < 10%', badcompression,
['Compressed Size', 'Installed Size', 'Ratio', 'Type'],
['compressed_size_pretty', 'installed_size_pretty', 'ratio',
'compress_type'])
REPORT_MAN = DeveloperReport('uncompressed-man', 'Uncompressed Manpages',
'Packages with uncompressed manpages', uncompressed_man)
'Packages with uncompressed manpages',
uncompressed_man)
REPORT_INFO = DeveloperReport('uncompressed-info', 'Uncompressed Info Pages',
'Packages with uncompressed info pages', uncompressed_info)
'Packages with uncompressed info pages',
uncompressed_info)
REPORT_ORPHANS = DeveloperReport('unneeded-orphans', 'Unneeded Orphans',
'Packages that have no maintainer and are not required by any '
+ 'other package in any repository', unneeded_orphans,
personal=False)
REPORT_ORPHANS = DeveloperReport(
'unneeded-orphans',
'Unneeded Orphans',
'Packages that have no maintainer and are not required by any ' +
'other package in any repository',
unneeded_orphans,
personal=False)
REPORT_SIGNATURE = DeveloperReport('mismatched-signature',
'Mismatched Signatures',
'Packages where the signing key is unknown or signer != packager',
mismatched_signature,
['Signed By', 'Packager'],
['sig_by', 'packager'])
REPORT_SIGNATURE = DeveloperReport(
'mismatched-signature', 'Mismatched Signatures',
'Packages where the signing key is unknown or signer != packager',
mismatched_signature, ['Signed By', 'Packager'], ['sig_by', 'packager'])
REPORT_SIG_TIME = DeveloperReport('signature-time', 'Signature Time',
'Packages where the signature timestamp is more than 24 hours '
+ 'after the build timestamp',
signature_time,
['Signature Date', 'Packager'],
['sig_date', 'packager'])
REPORT_SIG_TIME = DeveloperReport(
'signature-time', 'Signature Time',
'Packages where the signature timestamp is more than 24 hours ' +
'after the build timestamp', signature_time,
['Signature Date', 'Packager'], ['sig_date', 'packager'])
def available_reports():
return (
REPORT_OLD,
REPORT_OUTOFDATE,
REPORT_BIG,
REPORT_BADCOMPRESS,
REPORT_MAN,
REPORT_INFO,
REPORT_ORPHANS,
REPORT_SIGNATURE,
REPORT_SIG_TIME,
)
return (REPORT_OLD,
REPORT_OUTOFDATE,
REPORT_BIG,
REPORT_BADCOMPRESS,
REPORT_MAN,
REPORT_INFO,
REPORT_ORPHANS,
REPORT_SIGNATURE,
REPORT_SIG_TIME, )

143
devel/views.py
View File

@ -1,33 +1,32 @@
from datetime import timedelta
import operator
import time
from datetime import timedelta
from django.http import HttpResponseRedirect
from django.contrib.auth.decorators import \
login_required, permission_required, user_passes_test
from django.contrib import admin
from django.contrib.admin.models import LogEntry, ADDITION
from django.contrib.admin.models import ADDITION, LogEntry
from django.contrib.auth.decorators import (login_required,
permission_required,
user_passes_test)
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import transaction
from django.db.models import Count, Max
from django.http import Http404
from django.http import Http404, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.views.decorators.cache import never_cache
from django.utils.encoding import force_unicode
from django.utils.http import http_date
from django.utils.timezone import now
from .forms import ProfileForm, UserProfileForm, NewUserForm
from .models import UserProfile
from .reports import available_reports
from main.models import Package
from main.models import Arch, Repo
from django.views.decorators.cache import never_cache
from main.models import Arch, Package, Repo
from news.models import News
from packages.models import PackageRelation, Signoff, FlagRequest
from packages.models import FlagRequest, PackageRelation, Signoff
from packages.utils import get_signoff_groups
from todolists.models import TodolistPackage
from todolists.utils import get_annotated_todolists
from .forms import NewUserForm, ProfileForm, UserProfileForm
from .models import UserProfile
from .reports import available_reports
from .utils import get_annotated_maintainers
@ -41,25 +40,26 @@ def index(request):
inner_q = inner_q.values('pkgbase')
flagged = Package.objects.normal().filter(
flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname')
flag_date__isnull=False, pkgbase__in=inner_q).order_by('pkgname')
todopkgs = TodolistPackage.objects.select_related(
'todolist', 'pkg', 'arch', 'repo').exclude(
'todolist', 'pkg', 'arch', 'repo').exclude(
status=TodolistPackage.COMPLETE).filter(removed__isnull=True)
todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by(
'todolist__name', 'pkgname')
todopkgs = todopkgs.filter(pkgbase__in=inner_q).order_by('todolist__name',
'pkgname')
todolists = get_annotated_todolists(incomplete_only=True)
signoffs = sorted(get_signoff_groups(user=request.user),
key=operator.attrgetter('pkgbase'))
signoffs = sorted(
get_signoff_groups(user=request.user),
key=operator.attrgetter('pkgbase'))
page_dict = {
'todos': todolists,
'flagged': flagged,
'todopkgs': todopkgs,
'signoffs': signoffs,
'reports': available_reports(),
'todos': todolists,
'flagged': flagged,
'todopkgs': todopkgs,
'signoffs': signoffs,
'reports': available_reports(),
}
return render(request, 'devel/index.html', page_dict)
@ -69,35 +69,37 @@ def index(request):
def stats(request):
"""The second half of the dev dashboard."""
arches = Arch.objects.all().annotate(
total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
total_ct=Count('packages'),
flagged_ct=Count('packages__flag_date'))
repos = Repo.objects.all().annotate(
total_ct=Count('packages'), flagged_ct=Count('packages__flag_date'))
total_ct=Count('packages'),
flagged_ct=Count('packages__flag_date'))
# the join is huge unless we do this separately, so merge the result here
repo_maintainers = dict(Repo.objects.order_by().filter(
userprofile__user__is_active=True).values_list('id').annotate(
Count('userprofile')))
userprofile__user__is_active=True).values_list('id').annotate(Count(
'userprofile')))
for repo in repos:
repo.maintainer_ct = repo_maintainers.get(repo.id, 0)
maintainers = get_annotated_maintainers()
maintained = PackageRelation.objects.filter(
type=PackageRelation.MAINTAINER).values('pkgbase')
type=PackageRelation.MAINTAINER).values('pkgbase')
total_orphans = Package.objects.exclude(pkgbase__in=maintained).count()
total_flagged_orphans = Package.objects.filter(
flag_date__isnull=False).exclude(pkgbase__in=maintained).count()
flag_date__isnull=False).exclude(pkgbase__in=maintained).count()
total_updated = Package.objects.filter(packager__isnull=True).count()
orphan = {
'package_count': total_orphans,
'flagged_count': total_flagged_orphans,
'updated_count': total_updated,
'package_count': total_orphans,
'flagged_count': total_flagged_orphans,
'updated_count': total_updated,
}
page_dict = {
'arches': arches,
'repos': repos,
'maintainers': maintainers,
'orphan': orphan,
'arches': arches,
'repos': repos,
'maintainers': maintainers,
'orphan': orphan,
}
return render(request, 'devel/stats.html', page_dict)
@ -106,25 +108,23 @@ def stats(request):
@login_required
def clock(request):
devs = User.objects.filter(is_active=True).order_by(
'first_name', 'last_name').select_related('userprofile')
'first_name', 'last_name').select_related('userprofile')
latest_news = dict(News.objects.filter(
author__is_active=True).values_list('author').order_by(
).annotate(last_post=Max('postdate')))
latest_news = dict(News.objects.filter(author__is_active=True).values_list(
'author').order_by().annotate(last_post=Max('postdate')))
latest_package = dict(Package.objects.filter(
packager__is_active=True).values_list('packager').order_by(
).annotate(last_build=Max('build_date')))
packager__is_active=True).values_list('packager').order_by().annotate(
last_build=Max('build_date')))
latest_signoff = dict(Signoff.objects.filter(
user__is_active=True).values_list('user').order_by(
).annotate(last_signoff=Max('created')))
user__is_active=True).values_list('user').order_by().annotate(
last_signoff=Max('created')))
# The extra() bit ensures we can use our 'user_id IS NOT NULL' index
latest_flagreq = dict(FlagRequest.objects.filter(
user__is_active=True).extra(
where=['user_id IS NOT NULL']).values_list('user_id').order_by(
).annotate(last_flagrequest=Max('created')))
user__is_active=True).extra(where=['user_id IS NOT NULL']).values_list(
'user_id').order_by().annotate(last_flagrequest=Max('created')))
latest_log = dict(LogEntry.objects.filter(
user__is_active=True).values_list('user').order_by(
).annotate(last_log=Max('action_time')))
user__is_active=True).values_list('user').order_by().annotate(
last_log=Max('action_time')))
for dev in devs:
dates = [
@ -142,10 +142,7 @@ def clock(request):
dev.last_action = None
current_time = now()
page_dict = {
'developers': devs,
'utc_now': current_time,
}
page_dict = {'developers': devs, 'utc_now': current_time, }
response = render(request, 'devel/clock.html', page_dict)
if not response.has_header('Expires'):
@ -162,8 +159,9 @@ def change_profile(request):
profile, _ = UserProfile.objects.get_or_create(user=request.user)
if request.POST:
form = ProfileForm(request.POST)
profile_form = UserProfileForm(request.POST, request.FILES,
instance=profile)
profile_form = UserProfileForm(request.POST,
request.FILES,
instance=profile)
if form.is_valid() and profile_form.is_valid():
request.user.email = form.cleaned_data['email']
if form.cleaned_data['passwd1']:
@ -176,7 +174,8 @@ def change_profile(request):
form = ProfileForm(initial={'email': request.user.email})
profile_form = UserProfileForm(instance=profile)
return render(request, 'devel/profile.html',
{'form': form, 'profile_form': profile_form})
{'form': form,
'profile_form': profile_form})
@login_required
@ -190,14 +189,18 @@ def report(request, report_name, username=None):
user = None
if username:
user = get_object_or_404(User, username=username, is_active=True)
maintained = PackageRelation.objects.filter(user=user,
type=PackageRelation.MAINTAINER).values('pkgbase')
maintained = PackageRelation.objects.filter(
user=user, type=PackageRelation.MAINTAINER).values('pkgbase')
packages = packages.filter(pkgbase__in=maintained)
maints = User.objects.filter(id__in=PackageRelation.objects.filter(
type=PackageRelation.MAINTAINER).values('user'))
packages = report.packages(packages, username)
if report.slug == 'uncompressed-man' or report.slug == 'uncompressed-info':
packages = report.packages(packages, username)
else:
packages = report.packages(packages)
arches = {pkg.arch for pkg in packages}
repos = {pkg.repo for pkg in packages}
context = {
@ -217,13 +220,12 @@ def report(request, report_name, username=None):
def log_addition(request, obj):
"""Cribbed from ModelAdmin.log_addition."""
LogEntry.objects.log_action(
user_id = request.user.pk,
content_type_id = ContentType.objects.get_for_model(obj).pk,
object_id = obj.pk,
object_repr = force_unicode(obj),
action_flag = ADDITION,
change_message = "Added via Create New User form."
)
user_id=request.user.pk,
content_type_id=ContentType.objects.get_for_model(obj).pk,
object_id=obj.pk,
object_repr=force_unicode(obj),
action_flag=ADDITION,
change_message="Added via Create New User form.")
@permission_required('auth.add_user')
@ -257,10 +259,7 @@ def admin_log(request, username=None):
user = None
if username:
user = get_object_or_404(User, username=username)
context = {
'title': "Admin Action Log",
'log_user': user,
}
context = {'title': "Admin Action Log", 'log_user': user, }
context.update(admin.site.each_context())
return render(request, 'devel/admin_log.html', context)