6b16b9487a
Remove never_cache from many places now that we don't actually need it since we aren't caching by default. Adjust our cache_function decorator times be shorter values, and also randomize them a bit to make cache invalidations not all line up. Signed-off-by: Dan McGee <dan@archlinux.org>
426 lines
15 KiB
Python
426 lines
15 KiB
Python
from collections import defaultdict
|
|
from itertools import chain
|
|
from operator import itemgetter
|
|
|
|
from django.db import connection
|
|
from django.db.models import Count, Max, F
|
|
from django.contrib.auth.models import User
|
|
|
|
from main.models import Package, Arch, Repo
|
|
from main.utils import cache_function, groupby_preserve_order, PackageStandin
|
|
from .models import (PackageGroup, PackageRelation,
|
|
SignoffSpecification, Signoff, DEFAULT_SIGNOFF_SPEC)
|
|
|
|
@cache_function(127)
|
|
def get_group_info(include_arches=None):
|
|
raw_groups = PackageGroup.objects.values_list(
|
|
'name', 'pkg__arch__name').order_by('name').annotate(
|
|
cnt=Count('pkg'), last_update=Max('pkg__last_update'))
|
|
# now for post_processing. we need to seperate things out and add
|
|
# the count in for 'any' to all of the other architectures.
|
|
group_mapping = {}
|
|
for grp in raw_groups:
|
|
arch_groups = group_mapping.setdefault(grp[1], {})
|
|
arch_groups[grp[0]] = {'name': grp[0], 'arch': grp[1],
|
|
'count': grp[2], 'last_update': grp[3]}
|
|
|
|
# we want to promote the count of 'any' packages in groups to the
|
|
# other architectures, and also add any 'any'-only groups
|
|
if 'any' in group_mapping:
|
|
any_groups = group_mapping['any']
|
|
del group_mapping['any']
|
|
for arch, arch_groups in group_mapping.iteritems():
|
|
for grp in any_groups.itervalues():
|
|
if grp['name'] in arch_groups:
|
|
found = arch_groups[grp['name']]
|
|
found['count'] += grp['count']
|
|
if grp['last_update'] > found['last_update']:
|
|
found['last_update'] = grp['last_update']
|
|
else:
|
|
new_g = grp.copy()
|
|
# override the arch to not be 'any'
|
|
new_g['arch'] = arch
|
|
arch_groups[grp['name']] = new_g
|
|
|
|
# now transform it back into a sorted list, including only the specified
|
|
# architectures if we got a list
|
|
groups = []
|
|
for key, val in group_mapping.iteritems():
|
|
if not include_arches or key in include_arches:
|
|
groups.extend(val.itervalues())
|
|
return sorted(groups, key=itemgetter('name', 'arch'))
|
|
|
|
def get_split_packages_info():
|
|
'''Return info on split packages that do not have an actual package name
|
|
matching the split pkgbase.'''
|
|
pkgnames = Package.objects.values('pkgname')
|
|
split_pkgs = Package.objects.exclude(pkgname=F('pkgbase')).exclude(
|
|
pkgbase__in=pkgnames).values('pkgbase', 'repo', 'arch').annotate(
|
|
last_update=Max('last_update'))
|
|
all_arches = Arch.objects.in_bulk(set(s['arch'] for s in split_pkgs))
|
|
all_repos = Repo.objects.in_bulk(set(s['repo'] for s in split_pkgs))
|
|
for split in split_pkgs:
|
|
split['arch'] = all_arches[split['arch']]
|
|
split['repo'] = all_repos[split['repo']]
|
|
return split_pkgs
|
|
|
|
|
|
class Difference(object):
|
|
def __init__(self, pkgname, repo, pkg_a, pkg_b):
|
|
self.pkgname = pkgname
|
|
self.repo = repo
|
|
self.pkg_a = pkg_a
|
|
self.pkg_b = pkg_b
|
|
|
|
def classes(self):
|
|
'''A list of CSS classes that should be applied to this row in any
|
|
generated HTML. Useful for sorting, filtering, etc. Contains whether
|
|
this difference is in both architectures or the sole architecture it
|
|
belongs to, as well as the repo name.'''
|
|
css_classes = [self.repo.name.lower()]
|
|
if self.pkg_a and self.pkg_b:
|
|
css_classes.append('both')
|
|
elif self.pkg_a:
|
|
css_classes.append(self.pkg_a.arch.name)
|
|
elif self.pkg_b:
|
|
css_classes.append(self.pkg_b.arch.name)
|
|
return ' '.join(css_classes)
|
|
|
|
def __cmp__(self, other):
|
|
if isinstance(other, Difference):
|
|
return cmp(self.__dict__, other.__dict__)
|
|
return False
|
|
|
|
|
|
@cache_function(127)
|
|
def get_differences_info(arch_a, arch_b):
|
|
# This is a monster. Join packages against itself, looking for packages in
|
|
# our non-'any' architectures only, and not having a corresponding package
|
|
# entry in the other table (or having one with a different pkgver). We will
|
|
# then go and fetch all of these packages from the database and display
|
|
# them later using normal ORM models.
|
|
sql = """
|
|
SELECT p.id, q.id
|
|
FROM packages p
|
|
LEFT JOIN packages q
|
|
ON (
|
|
p.pkgname = q.pkgname
|
|
AND p.repo_id = q.repo_id
|
|
AND p.arch_id != q.arch_id
|
|
AND p.id != q.id
|
|
)
|
|
WHERE p.arch_id IN (%s, %s)
|
|
AND (
|
|
q.id IS NULL
|
|
OR p.pkgver != q.pkgver
|
|
OR p.pkgrel != q.pkgrel
|
|
OR p.epoch != q.epoch
|
|
)
|
|
"""
|
|
cursor = connection.cursor()
|
|
cursor.execute(sql, [arch_a.id, arch_b.id])
|
|
results = cursor.fetchall()
|
|
# column A will always have a value, column B might be NULL
|
|
to_fetch = [row[0] for row in results]
|
|
# fetch all of the necessary packages
|
|
pkgs = Package.objects.normal().in_bulk(to_fetch)
|
|
# now build a list of tuples containing differences
|
|
differences = []
|
|
for row in results:
|
|
pkg_a = pkgs.get(row[0])
|
|
pkg_b = pkgs.get(row[1])
|
|
# We want arch_a to always appear first
|
|
# pkg_a should never be None
|
|
if pkg_a.arch == arch_a:
|
|
item = Difference(pkg_a.pkgname, pkg_a.repo, pkg_a, pkg_b)
|
|
else:
|
|
# pkg_b can be None in this case, so be careful
|
|
name = pkg_a.pkgname if pkg_a else pkg_b.pkgname
|
|
repo = pkg_a.repo if pkg_a else pkg_b.repo
|
|
item = Difference(name, repo, pkg_b, pkg_a)
|
|
if item not in differences:
|
|
differences.append(item)
|
|
|
|
# now sort our list by repository, package name
|
|
differences.sort(key=lambda a: (a.repo.name, a.pkgname))
|
|
return differences
|
|
|
|
|
|
def multilib_differences():
|
|
# Query for checking multilib out of date-ness
|
|
sql = """
|
|
SELECT ml.id, reg.id
|
|
FROM packages ml
|
|
JOIN packages reg
|
|
ON (
|
|
reg.pkgname = (
|
|
CASE WHEN ml.pkgname LIKE %s
|
|
THEN SUBSTRING(ml.pkgname, 7)
|
|
WHEN ml.pkgname LIKE %s
|
|
THEN SUBSTRING(ml.pkgname FROM 1 FOR CHAR_LENGTH(ml.pkgname) - 9)
|
|
ELSE
|
|
ml.pkgname
|
|
END
|
|
)
|
|
AND reg.pkgver != ml.pkgver
|
|
)
|
|
JOIN repos r ON reg.repo_id = r.id
|
|
WHERE ml.repo_id = %s
|
|
AND r.testing = %s
|
|
AND r.staging = %s
|
|
AND reg.arch_id = %s
|
|
ORDER BY ml.last_update
|
|
"""
|
|
multilib = Repo.objects.get(name__iexact='multilib')
|
|
i686 = Arch.objects.get(name='i686')
|
|
params = ['lib32-%', '%-multilib', multilib.id, False, False, i686.id]
|
|
|
|
cursor = connection.cursor()
|
|
cursor.execute(sql, params)
|
|
results = cursor.fetchall()
|
|
|
|
# fetch all of the necessary packages
|
|
to_fetch = set(chain.from_iterable(results))
|
|
pkgs = Package.objects.normal().in_bulk(to_fetch)
|
|
|
|
return [(pkgs[ml], pkgs[reg]) for ml, reg in results]
|
|
|
|
|
|
def get_wrong_permissions():
|
|
sql = """
|
|
SELECT DISTINCT id
|
|
FROM (
|
|
SELECT pr.id, p.repo_id, pr.user_id
|
|
FROM packages p
|
|
JOIN packages_packagerelation pr ON p.pkgbase = pr.pkgbase
|
|
WHERE pr.type = %s
|
|
) pkgs
|
|
WHERE pkgs.repo_id NOT IN (
|
|
SELECT repo_id FROM user_profiles_allowed_repos ar
|
|
INNER JOIN user_profiles up ON ar.userprofile_id = up.id
|
|
WHERE up.user_id = pkgs.user_id
|
|
)
|
|
"""
|
|
cursor = connection.cursor()
|
|
cursor.execute(sql, [PackageRelation.MAINTAINER])
|
|
to_fetch = [row[0] for row in cursor.fetchall()]
|
|
relations = PackageRelation.objects.select_related('user').filter(
|
|
id__in=to_fetch)
|
|
return relations
|
|
|
|
|
|
def attach_maintainers(packages):
|
|
'''Given a queryset or something resembling it of package objects, find all
|
|
the maintainers and attach them to the packages to prevent N+1 query
|
|
cascading.'''
|
|
packages = list(packages)
|
|
pkgbases = set(p.pkgbase for p in packages)
|
|
rels = PackageRelation.objects.filter(type=PackageRelation.MAINTAINER,
|
|
pkgbase__in=pkgbases).values_list('pkgbase', 'user_id').distinct()
|
|
|
|
# get all the user objects we will need
|
|
user_ids = set(rel[1] for rel in rels)
|
|
users = User.objects.in_bulk(user_ids)
|
|
|
|
# now build a pkgbase -> [maintainers...] map
|
|
maintainers = defaultdict(list)
|
|
for rel in rels:
|
|
maintainers[rel[0]].append(users[rel[1]])
|
|
|
|
annotated = []
|
|
# and finally, attach the maintainer lists on the original packages
|
|
for package in packages:
|
|
package.maintainers = maintainers[package.pkgbase]
|
|
annotated.append(package)
|
|
|
|
return annotated
|
|
|
|
|
|
def approved_by_signoffs(signoffs, spec):
|
|
if signoffs:
|
|
good_signoffs = sum(1 for s in signoffs if not s.revoked)
|
|
return good_signoffs >= spec.required
|
|
return False
|
|
|
|
class PackageSignoffGroup(object):
|
|
'''Encompasses all packages in testing with the same pkgbase.'''
|
|
def __init__(self, packages):
|
|
if len(packages) == 0:
|
|
raise Exception
|
|
self.packages = packages
|
|
self.user = None
|
|
self.target_repo = None
|
|
self.signoffs = set()
|
|
self.specification = DEFAULT_SIGNOFF_SPEC
|
|
self.default_spec = True
|
|
|
|
first = packages[0]
|
|
self.pkgbase = first.pkgbase
|
|
self.arch = first.arch
|
|
self.repo = first.repo
|
|
self.version = ''
|
|
self.last_update = first.last_update
|
|
self.packager = first.packager
|
|
self.maintainers = first.maintainers
|
|
|
|
version = first.full_version
|
|
if all(version == pkg.full_version for pkg in packages):
|
|
self.version = version
|
|
|
|
@property
|
|
def package(self):
|
|
'''Try and return a relevant single package object representing this
|
|
group. Start by seeing if there is only one package, then look for the
|
|
matching package by name, finally falling back to a standin package
|
|
object.'''
|
|
if len(self.packages) == 1:
|
|
return self.packages[0]
|
|
|
|
same_pkgs = [p for p in self.packages if p.pkgname == p.pkgbase]
|
|
if same_pkgs:
|
|
return same_pkgs[0]
|
|
|
|
return PackageStandin(self.packages[0])
|
|
|
|
def find_signoffs(self, all_signoffs):
|
|
'''Look through a list of Signoff objects for ones matching this
|
|
particular group and store them on the object.'''
|
|
for s in all_signoffs:
|
|
if s.pkgbase != self.pkgbase:
|
|
continue
|
|
if self.version and not s.full_version == self.version:
|
|
continue
|
|
if s.arch_id == self.arch.id and s.repo_id == self.repo.id:
|
|
self.signoffs.add(s)
|
|
|
|
def find_specification(self, specifications):
|
|
for spec in specifications:
|
|
if spec.pkgbase != self.pkgbase:
|
|
continue
|
|
if self.version and not spec.full_version == self.version:
|
|
continue
|
|
if spec.arch_id == self.arch.id and spec.repo_id == self.repo.id:
|
|
self.specification = spec
|
|
self.default_spec = False
|
|
return
|
|
|
|
def approved(self):
|
|
return approved_by_signoffs(self.signoffs, self.specification)
|
|
|
|
@property
|
|
def completed(self):
|
|
return sum(1 for s in self.signoffs if not s.revoked)
|
|
|
|
@property
|
|
def required(self):
|
|
return self.specification.required
|
|
|
|
def user_signed_off(self, user=None):
|
|
'''Did a given user signoff on this package? user can be passed as an
|
|
argument, or attached to the group object itself so this can be called
|
|
from a template.'''
|
|
if user is None:
|
|
user = self.user
|
|
return user in (s.user for s in self.signoffs if not s.revoked)
|
|
|
|
def __unicode__(self):
|
|
return u'%s-%s (%s): %d' % (
|
|
self.pkgbase, self.version, self.arch, len(self.signoffs))
|
|
|
|
_SQL_SPEC_OR_SIGNOFF = """
|
|
SELECT DISTINCT s.id
|
|
FROM %s s
|
|
JOIN packages p ON (
|
|
s.pkgbase = p.pkgbase
|
|
AND s.pkgver = p.pkgver
|
|
AND s.pkgrel = p.pkgrel
|
|
AND s.epoch = p.epoch
|
|
AND s.arch_id = p.arch_id
|
|
AND s.repo_id = p.repo_id
|
|
)
|
|
AND p.repo_id IN (%s)
|
|
"""
|
|
|
|
def get_current_signoffs(repos):
|
|
'''Returns a mapping of pkgbase -> signoff objects for the given repos.'''
|
|
cursor = connection.cursor()
|
|
# query pre-process- fill in table name and placeholders for IN
|
|
sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoff',
|
|
','.join(['%s' for r in repos]))
|
|
cursor.execute(sql, [r.pk for r in repos])
|
|
|
|
results = cursor.fetchall()
|
|
# fetch all of the returned signoffs by ID
|
|
to_fetch = [row[0] for row in results]
|
|
signoffs = Signoff.objects.select_related('user').in_bulk(to_fetch)
|
|
return signoffs.values()
|
|
|
|
def get_current_specifications(repos):
|
|
'''Returns a mapping of pkgbase -> signoff specification objects for the
|
|
given repos.'''
|
|
cursor = connection.cursor()
|
|
sql = _SQL_SPEC_OR_SIGNOFF % ('packages_signoffspecification',
|
|
','.join(['%s' for r in repos]))
|
|
cursor.execute(sql, [r.pk for r in repos])
|
|
|
|
results = cursor.fetchall()
|
|
to_fetch = [row[0] for row in results]
|
|
return SignoffSpecification.objects.in_bulk(to_fetch).values()
|
|
|
|
def get_target_repo_map(repos):
|
|
sql = """
|
|
SELECT DISTINCT p1.pkgbase, r.name
|
|
FROM packages p1
|
|
JOIN repos r ON p1.repo_id = r.id
|
|
JOIN packages p2 ON p1.pkgbase = p2.pkgbase
|
|
WHERE r.staging = %s
|
|
AND r.testing = %s
|
|
AND p2.repo_id IN (
|
|
"""
|
|
sql += ','.join(['%s' for _ in repos])
|
|
sql += ")"
|
|
|
|
params = [False, False]
|
|
params.extend(r.pk for r in repos)
|
|
|
|
cursor = connection.cursor()
|
|
cursor.execute(sql, params)
|
|
return dict(cursor.fetchall())
|
|
|
|
def get_signoff_groups(repos=None, user=None):
|
|
if repos is None:
|
|
repos = Repo.objects.filter(testing=True)
|
|
repo_ids = [r.pk for r in repos]
|
|
|
|
test_pkgs = Package.objects.select_related(
|
|
'arch', 'repo', 'packager').filter(repo__in=repo_ids)
|
|
packages = test_pkgs.order_by('pkgname')
|
|
packages = attach_maintainers(packages)
|
|
|
|
# Filter by user if asked to do so
|
|
if user is not None:
|
|
packages = [p for p in packages if user == p.packager
|
|
or user in p.maintainers]
|
|
|
|
# Collect all pkgbase values in testing repos
|
|
pkgtorepo = get_target_repo_map(repos)
|
|
|
|
# Collect all possible signoffs and specifications for these packages
|
|
signoffs = get_current_signoffs(repos)
|
|
specs = get_current_specifications(repos)
|
|
|
|
same_pkgbase_key = lambda x: (x.repo.name, x.arch.name, x.pkgbase)
|
|
grouped = groupby_preserve_order(packages, same_pkgbase_key)
|
|
signoff_groups = []
|
|
for group in grouped:
|
|
signoff_group = PackageSignoffGroup(group)
|
|
signoff_group.target_repo = pkgtorepo.get(signoff_group.pkgbase,
|
|
"Unknown")
|
|
signoff_group.find_signoffs(signoffs)
|
|
signoff_group.find_specification(specs)
|
|
signoff_groups.append(signoff_group)
|
|
|
|
return signoff_groups
|
|
|
|
# vim: set ts=4 sw=4 et:
|