From 1a15dea6700eadc8379d0fef2e0b3f37de92c54e Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Sat, 30 May 2020 17:14:46 +0200 Subject: Add rebuilderd status import and reporting functionality Import the rebuilderd status periodically with a django management command into RebuilderdStats which holds one record per pkgname with it's pkgver/pkgrel/epoch all recorded. Shown as a developer dashboard and with opt in mail notifications for when a package becomes not reproducible. --- .../commands/read_reproducible_status.py | 146 +++++++++++++++++++++ .../0006_userprofile_rebuilderd_updates.py | 18 +++ devel/models.py | 2 + devel/reports.py | 18 ++- main/migrations/0003_rebuilderdstatus.py | 29 ++++ main/models.py | 35 +++++ requirements.txt | 1 + settings.py | 3 + templates/devel/email_reproduciblebuilds.txt | 4 + 9 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 devel/management/commands/read_reproducible_status.py create mode 100644 devel/migrations/0006_userprofile_rebuilderd_updates.py create mode 100644 main/migrations/0003_rebuilderdstatus.py create mode 100644 templates/devel/email_reproduciblebuilds.txt diff --git a/devel/management/commands/read_reproducible_status.py b/devel/management/commands/read_reproducible_status.py new file mode 100644 index 00000000..f640572b --- /dev/null +++ b/devel/management/commands/read_reproducible_status.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- +""" +read_reproducible_status command + +Import reproducible status of packages, rebuilderd url configured in django +settings. + +Usage: ./manage.py read_reproducible_status +""" + +import logging +import re +import sys + +from collections import defaultdict + +import requests + +from django.core.mail import send_mail +from django.conf import settings +from django.core.management.base import BaseCommand +from django.template import loader + +from devel.models import UserProfile +from main.models import Arch, Repo, Package, RebuilderdStatus + + +EPOCH_REGEX = r'^(\d+):' + +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s -> %(levelname)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + + +class Command(BaseCommand): + help = "Import reproducible status from rebuilderd." + + def handle(self, *args, **options): + v = int(options.get('verbosity', None)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + url = getattr(settings, "REBUILDERD_URL", None) + if not url: + logger.error("no rebuilderd_url configured in local_settings.py") + + was_repro = import_rebuilderd_status(url) + + send_repro_emails(was_repro) + + +def send_repro_emails(was_repro): + template = loader.get_template('devel/email_reproduciblebuilds.txt') + enabled_users = [prof.user for prof in UserProfile.objects.filter(rebuilderd_updates=True).all()] + + # Group statusses by maintainer + maintainers_map = defaultdict(list) + + for status in was_repro: + for maintainer in status.pkg.maintainers: + if maintainer not in enabled_users: + continue + + maintainers_map[maintainer.userprofile].append(status.pkg.pkgname) + for maintainer, pkgs in maintainers_map.items(): + send_mail('Packages which have become not reproducible', + template.render({'pkgs': pkgs}), + 'Arch Website Notification ', + [maintainer.public_email], + fail_silently=True) + + +def import_rebuilderd_status(url): + statuses = [] + was_repro = [] + + req = requests.get(url) + data = req.json() + + for pkg in data: + arch = Arch.objects.get(name=pkg['architecture']) + repository = Repo.objects.get(name__iexact=pkg['suite']) + + epoch = 0 + pkgname = pkg['name'] + version = pkg['version'] + + matches = re.search(EPOCH_REGEX, version) + if matches: + epoch = matches.group(1) + + pkgver, pkgrel = pkg['version'].rsplit('-', 1) + + dbpkg = Package.objects.filter(pkgname=pkgname, pkgver=pkgver, + pkgrel=pkgrel, epoch=epoch, + repo=repository, + arch=arch).first() + if not dbpkg: + continue + + rbstatus = RebuilderdStatus.objects.filter(pkg=dbpkg).first() + status = RebuilderdStatus.REBUILDERD_API_STATUSES.get(pkg['status'], RebuilderdStatus.UNKNOWN) + + # Existing status + if rbstatus: + # If status has become BAD, set was_repro + if rbstatus.status == RebuilderdStatus.GOOD and status == RebuilderdStatus.BAD: + was_repro.append(rbstatus) + rbstatus.was_repro = True + logger.info("package '%s' was good is now bad", pkg['name']) + else: # reset status + rbstatus.was_repro = False + + if rbstatus.pkgver != pkgver or rbstatus.pkgrel != pkgrel or rbstatus.epoch != epoch: + logger.info('updating status for package: %s', pkg['name']) + rbstatus.epoch = epoch + rbstatus.pkgver = pkgver + rbstatus.pkgrel = pkgrel + rbstatus.status = status + rbstatus.arch = arch + rbstatus.repo = repository + elif rbstatus.status != status: # Rebuilderd rebuild the same package? + logger.info('status for package: %s changed', pkg['name']) + rbstatus.status = status + + # TODO: does django know when a model was really modified? + rbstatus.save() + + else: # new package/status + logger.info('adding status for package: %s', pkg['name']) + rbstatus = RebuilderdStatus(pkg=dbpkg, status=status, arch=arch, repo=repository, + pkgname=pkgname, epoch=epoch, pkgrel=pkgrel, + pkgver=pkgver) + statuses.append(rbstatus) + + if statuses: + RebuilderdStatus.objects.bulk_create(statuses) + + return was_repro diff --git a/devel/migrations/0006_userprofile_rebuilderd_updates.py b/devel/migrations/0006_userprofile_rebuilderd_updates.py new file mode 100644 index 00000000..711ea5ea --- /dev/null +++ b/devel/migrations/0006_userprofile_rebuilderd_updates.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.7 on 2020-06-28 21:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('devel', '0005_auto_20200628_1600'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='rebuilderd_updates', + field=models.BooleanField(default=False, help_text='Receive reproducible build package updates'), + ), + ] diff --git a/devel/models.py b/devel/models.py index d7b6c379..59324746 100644 --- a/devel/models.py +++ b/devel/models.py @@ -50,6 +50,8 @@ class UserProfile(models.Model): allowed_repos = models.ManyToManyField('main.Repo', blank=True) latin_name = models.CharField(max_length=255, null=True, blank=True, help_text="Latin-form name; used only for non-Latin full names") + rebuilderd_updates = models.BooleanField(default=False, + help_text='Receive reproducible build package updates') last_modified = models.DateTimeField(editable=False) class Meta: diff --git a/devel/reports.py b/devel/reports.py index b4f9d794..6113195a 100644 --- a/devel/reports.py +++ b/devel/reports.py @@ -5,7 +5,7 @@ from django.db.models import F from django.template.defaultfilters import filesizeformat from django.db import connection from django.utils.timezone import now -from main.models import Package, PackageFile +from main.models import Package, PackageFile, RebuilderdStatus from packages.models import Depend, PackageRelation from .models import DeveloperKey @@ -145,6 +145,7 @@ def signature_time(packages): return filtered + def non_existing_dependencies(packages): cursor = connection.cursor() query = """ @@ -167,6 +168,10 @@ def non_existing_dependencies(packages): return packages +def non_reproducible_packages(packages): + statuses = RebuilderdStatus.objects.exclude(status=RebuilderdStatus.GOOD).values('pkg__pkgname') + return packages.filter(pkgname__in=statuses) + REPORT_OLD = DeveloperReport( 'old', 'Old', 'Packages last built more than two years ago', old) @@ -223,6 +228,14 @@ NON_EXISTING_DEPENDENCIES = DeveloperReport( ['nonexistingdep'], personal=False) +REBUILDERD_PACKAGES = DeveloperReport( + 'non-reproducible-packages', + 'Non Reproducible package', + 'Packages that are not reproducible on our reproducible.archlinux.org test environment', + non_reproducible_packages, + ) + + def available_reports(): return (REPORT_OLD, REPORT_OUTOFDATE, @@ -233,4 +246,5 @@ def available_reports(): REPORT_ORPHANS, REPORT_SIGNATURE, REPORT_SIG_TIME, - NON_EXISTING_DEPENDENCIES, ) + NON_EXISTING_DEPENDENCIES, + REBUILDERD_PACKAGES, ) diff --git a/main/migrations/0003_rebuilderdstatus.py b/main/migrations/0003_rebuilderdstatus.py new file mode 100644 index 00000000..5cc0ad59 --- /dev/null +++ b/main/migrations/0003_rebuilderdstatus.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.7 on 2020-06-28 19:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0002_repo_public_testing'), + ] + + operations = [ + migrations.CreateModel( + name='RebuilderdStatus', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pkgname', models.CharField(max_length=255)), + ('pkgver', models.CharField(max_length=255)), + ('pkgrel', models.CharField(max_length=255)), + ('epoch', models.PositiveIntegerField(default=0)), + ('status', models.SmallIntegerField(choices=[(0, 'Good'), (1, 'Bad'), (2, 'Unknown')], default=2)), + ('was_repro', models.BooleanField(default=False)), + ('arch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Arch')), + ('pkg', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Package')), + ('repo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Repo')), + ], + ), + ] diff --git a/main/models.py b/main/models.py index 094e80a9..7f60276c 100644 --- a/main/models.py +++ b/main/models.py @@ -444,6 +444,41 @@ class PackageFile(models.Model): db_table = 'package_files' +class RebuilderdStatus(models.Model): + # https://github.com/kpcyrd/rebuilderd/blob/2eeef725940ca8710c7c21579ce093445d1875c8/common/src/api.rs#L279 + GOOD = 0 + BAD = 1 + UNKNOWN = 2 + + REBUILDERD_API_STATUSES = { + 'GOOD': GOOD, + 'BAD': BAD, + 'UNKWN': UNKNOWN, + } + REBUILDERD_STATUSES = ( + (GOOD, 'Good'), + (BAD, 'Bad'), + (UNKNOWN, 'Unknown'), + ) + + pkg = models.ForeignKey(Package, on_delete=models.CASCADE) + pkgname = models.CharField(max_length=255) + pkgver = models.CharField(max_length=255) + pkgrel = models.CharField(max_length=255) + epoch = models.PositiveIntegerField(default=0) + arch = models.ForeignKey(Arch, on_delete=models.CASCADE) + repo = models.ForeignKey(Repo, on_delete=models.CASCADE) + status = models.SmallIntegerField(default=UNKNOWN, choices=REBUILDERD_STATUSES) + was_repro = models.BooleanField(default=False) + + @property + def status_str(self): + return self.REBUILDERD_STATUSES[self.status][1] + + def __str__(self): + return "pkg=%s, status=%s" % (self.pkg, self.status_str) + + from django.db.models.signals import pre_save # note: reporead sets the 'created' field on Package objects, so no signal diff --git a/requirements.txt b/requirements.txt index efbf68b7..548cba8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ django-csp==3.6 ptpython==2.0.4 feedparser==5.2.1 bleach==3.1.5 +requests==2.22.0 diff --git a/settings.py b/settings.py index ec1d9d24..b68332e3 100644 --- a/settings.py +++ b/settings.py @@ -195,6 +195,9 @@ DATABASES = { # Planet limit of items per feed to keep the feed size in check. RSS_FEED_LIMIT = 25 +# Rebuilderd API endpoint +REBUILDERD_URL = 'https://reproducible.archlinux.org/api/v0/pkgs/list' + # Import local settings try: from local_settings import * diff --git a/templates/devel/email_reproduciblebuilds.txt b/templates/devel/email_reproduciblebuilds.txt new file mode 100644 index 00000000..385bd20f --- /dev/null +++ b/templates/devel/email_reproduciblebuilds.txt @@ -0,0 +1,4 @@ +The following packages have become non reproducible: +{% for pkg in pkgs %} +* {{ pkg }} +{% endfor %} -- cgit v1.2.3-55-g3dc8