diff options
author | Jelle van der Waa <jelle@archlinux.org> | 2020-05-30 17:14:46 +0200 |
---|---|---|
committer | Jelle van der Waa <jelle@archlinux.org> | 2020-06-29 16:28:27 +0200 |
commit | 1a15dea6700eadc8379d0fef2e0b3f37de92c54e (patch) | |
tree | bff44cd06c66c0faf08d42fa4c01d7d9d9aea34e /devel | |
parent | d0bafe48c47876492170eb1aa77a8dc4685db919 (diff) | |
download | archweb-1a15dea6700eadc8379d0fef2e0b3f37de92c54e.tar.gz archweb-1a15dea6700eadc8379d0fef2e0b3f37de92c54e.zip |
Add rebuilderd status import and reporting functionalityrelease_2020-06-29
Import the rebuilderd status periodically with a django management
command into RebuilderdStats which holds one record per pkgname with
it's pkgver/pkgrel/epoch all recorded. Shown as a developer dashboard
and with opt in mail notifications for when a package becomes not
reproducible.
Diffstat (limited to 'devel')
-rw-r--r-- | devel/management/commands/read_reproducible_status.py | 146 | ||||
-rw-r--r-- | devel/migrations/0006_userprofile_rebuilderd_updates.py | 18 | ||||
-rw-r--r-- | devel/models.py | 2 | ||||
-rw-r--r-- | devel/reports.py | 18 |
4 files changed, 182 insertions, 2 deletions
diff --git a/devel/management/commands/read_reproducible_status.py b/devel/management/commands/read_reproducible_status.py new file mode 100644 index 00000000..f640572b --- /dev/null +++ b/devel/management/commands/read_reproducible_status.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +""" +read_reproducible_status command + +Import reproducible status of packages, rebuilderd url configured in django +settings. + +Usage: ./manage.py read_reproducible_status +""" + +import logging +import re +import sys + +from collections import defaultdict + +import requests + +from django.core.mail import send_mail +from django.conf import settings +from django.core.management.base import BaseCommand +from django.template import loader + +from devel.models import UserProfile +from main.models import Arch, Repo, Package, RebuilderdStatus + + +EPOCH_REGEX = r'^(\d+):' + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + + +class Command(BaseCommand): + help = "Import reproducible status from rebuilderd." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + url = getattr(settings, "REBUILDERD_URL", None) + if not url: + logger.error("no rebuilderd_url configured in local_settings.py") + + was_repro = import_rebuilderd_status(url) + + send_repro_emails(was_repro) + + +def send_repro_emails(was_repro): + template = loader.get_template('devel/email_reproduciblebuilds.txt') + enabled_users = [prof.user for prof in UserProfile.objects.filter(rebuilderd_updates=True).all()] + + # Group statusses by maintainer + maintainers_map = defaultdict(list) + + for status in was_repro: + for maintainer in status.pkg.maintainers: + if maintainer not in enabled_users: + continue + + maintainers_map[maintainer.userprofile].append(status.pkg.pkgname) + for maintainer, pkgs in maintainers_map.items(): + send_mail('Packages which have become not reproducible', + template.render({'pkgs': pkgs}), + 'Arch Website Notification <nobody@archlinux.org>', + [maintainer.public_email], + fail_silently=True) + + +def import_rebuilderd_status(url): + statuses = [] + was_repro = [] + + req = requests.get(url) + data = req.json() + + for pkg in data: + arch = Arch.objects.get(name=pkg['architecture']) + repository = Repo.objects.get(name__iexact=pkg['suite']) + + epoch = 0 + pkgname = pkg['name'] + version = pkg['version'] + + matches = re.search(EPOCH_REGEX, version) + if matches: + epoch = matches.group(1) + + pkgver, pkgrel = pkg['version'].rsplit('-', 1) + + dbpkg = Package.objects.filter(pkgname=pkgname, pkgver=pkgver, + pkgrel=pkgrel, epoch=epoch, + repo=repository, + arch=arch).first() + if not dbpkg: + continue + + rbstatus = RebuilderdStatus.objects.filter(pkg=dbpkg).first() + status = RebuilderdStatus.REBUILDERD_API_STATUSES.get(pkg['status'], RebuilderdStatus.UNKNOWN) + + # Existing status + if rbstatus: + # If status has become BAD, set was_repro + if rbstatus.status == RebuilderdStatus.GOOD and status == RebuilderdStatus.BAD: + was_repro.append(rbstatus) + rbstatus.was_repro = True + logger.info("package '%s' was good is now bad", pkg['name']) + else: # reset status + rbstatus.was_repro = False + + if rbstatus.pkgver != pkgver or rbstatus.pkgrel != pkgrel or rbstatus.epoch != epoch: + logger.info('updating status for package: %s', pkg['name']) + rbstatus.epoch = epoch + rbstatus.pkgver = pkgver + rbstatus.pkgrel = pkgrel + rbstatus.status = status + rbstatus.arch = arch + rbstatus.repo = repository + elif rbstatus.status != status: # Rebuilderd rebuild the same package? + logger.info('status for package: %s changed', pkg['name']) + rbstatus.status = status + + # TODO: does django know when a model was really modified? + rbstatus.save() + + else: # new package/status + logger.info('adding status for package: %s', pkg['name']) + rbstatus = RebuilderdStatus(pkg=dbpkg, status=status, arch=arch, repo=repository, + pkgname=pkgname, epoch=epoch, pkgrel=pkgrel, + pkgver=pkgver) + statuses.append(rbstatus) + + if statuses: + RebuilderdStatus.objects.bulk_create(statuses) + + return was_repro diff --git a/devel/migrations/0006_userprofile_rebuilderd_updates.py b/devel/migrations/0006_userprofile_rebuilderd_updates.py new file mode 100644 index 00000000..711ea5ea --- /dev/null +++ b/devel/migrations/0006_userprofile_rebuilderd_updates.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2020-06-28 21:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('devel', '0005_auto_20200628_1600'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='rebuilderd_updates', + field=models.BooleanField(default=False, help_text='Receive reproducible build package updates'), + ), + ] diff --git a/devel/models.py b/devel/models.py index d7b6c379..59324746 100644 --- a/devel/models.py +++ b/devel/models.py @@ -50,6 +50,8 @@ class UserProfile(models.Model): allowed_repos = models.ManyToManyField('main.Repo', blank=True) latin_name = models.CharField(max_length=255, null=True, blank=True, help_text="Latin-form name; used only for non-Latin full names") + rebuilderd_updates = models.BooleanField(default=False, + help_text='Receive reproducible build package updates') last_modified = models.DateTimeField(editable=False) class Meta: diff --git a/devel/reports.py b/devel/reports.py index b4f9d794..6113195a 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -5,7 +5,7 @@ from django.db.models import F from django.template.defaultfilters import filesizeformat from django.db import connection from django.utils.timezone import now -from main.models import Package, PackageFile +from main.models import Package, PackageFile, RebuilderdStatus from packages.models import Depend, PackageRelation from .models import DeveloperKey @@ -145,6 +145,7 @@ def signature_time(packages): return filtered + def non_existing_dependencies(packages): cursor = connection.cursor() query = """ @@ -167,6 +168,10 @@ def non_existing_dependencies(packages): return packages +def non_reproducible_packages(packages): + statuses = RebuilderdStatus.objects.exclude(status=RebuilderdStatus.GOOD).values('pkg__pkgname') + return packages.filter(pkgname__in=statuses) + REPORT_OLD = DeveloperReport( 'old', 'Old', 'Packages last built more than two years ago', old) @@ -223,6 +228,14 @@ NON_EXISTING_DEPENDENCIES = DeveloperReport( ['nonexistingdep'], personal=False) +REBUILDERD_PACKAGES = DeveloperReport( + 'non-reproducible-packages', + 'Non Reproducible package', + 'Packages that are not reproducible on our reproducible.archlinux.org test environment', + non_reproducible_packages, + ) + + def available_reports(): return (REPORT_OLD, REPORT_OUTOFDATE, @@ -233,4 +246,5 @@ def available_reports(): REPORT_ORPHANS, REPORT_SIGNATURE, REPORT_SIG_TIME, - NON_EXISTING_DEPENDENCIES, ) + NON_EXISTING_DEPENDENCIES, + REBUILDERD_PACKAGES, ) |