summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJelle van der Waa <jelle@archlinux.org>2020-05-30 17:14:46 +0200
committerJelle van der Waa <jelle@archlinux.org>2020-06-29 16:28:27 +0200
commit1a15dea6700eadc8379d0fef2e0b3f37de92c54e (patch)
treebff44cd06c66c0faf08d42fa4c01d7d9d9aea34e
parentd0bafe48c47876492170eb1aa77a8dc4685db919 (diff)
downloadarchweb-release_2020-06-29.tar.gz
archweb-release_2020-06-29.zip
Add rebuilderd status import and reporting functionalityrelease_2020-06-29
Import the rebuilderd status periodically with a django management command into RebuilderdStats which holds one record per pkgname with it's pkgver/pkgrel/epoch all recorded. Shown as a developer dashboard and with opt in mail notifications for when a package becomes not reproducible.
-rw-r--r--devel/management/commands/read_reproducible_status.py146
-rw-r--r--devel/migrations/0006_userprofile_rebuilderd_updates.py18
-rw-r--r--devel/models.py2
-rw-r--r--devel/reports.py18
-rw-r--r--main/migrations/0003_rebuilderdstatus.py29
-rw-r--r--main/models.py35
-rw-r--r--requirements.txt1
-rw-r--r--settings.py3
-rw-r--r--templates/devel/email_reproduciblebuilds.txt4
9 files changed, 254 insertions, 2 deletions
diff --git a/devel/management/commands/read_reproducible_status.py b/devel/management/commands/read_reproducible_status.py
new file mode 100644
index 00000000..f640572b
--- /dev/null
+++ b/devel/management/commands/read_reproducible_status.py
@@ -0,0 +1,146 @@
+# -*- coding: utf-8 -*-
+"""
+read_reproducible_status command
+
+Import reproducible status of packages, rebuilderd url configured in django
+settings.
+
+Usage: ./manage.py read_reproducible_status
+"""
+
+import logging
+import re
+import sys
+
+from collections import defaultdict
+
+import requests
+
+from django.core.mail import send_mail
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.template import loader
+
+from devel.models import UserProfile
+from main.models import Arch, Repo, Package, RebuilderdStatus
+
+
+EPOCH_REGEX = r'^(\d+):'
+
+logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s -> %(levelname)s: %(message)s',
+ datefmt='%Y-%m-%d %H:%M:%S',
+ stream=sys.stderr)
+logger = logging.getLogger()
+
+
+class Command(BaseCommand):
+ help = "Import reproducible status from rebuilderd."
+
+ def handle(self, *args, **options):
+ v = int(options.get('verbosity', None))
+ if v == 0:
+ logger.level = logging.ERROR
+ elif v == 1:
+ logger.level = logging.INFO
+ elif v >= 2:
+ logger.level = logging.DEBUG
+
+ url = getattr(settings, "REBUILDERD_URL", None)
+ if not url:
+ logger.error("no rebuilderd_url configured in local_settings.py")
+
+ was_repro = import_rebuilderd_status(url)
+
+ send_repro_emails(was_repro)
+
+
+def send_repro_emails(was_repro):
+ template = loader.get_template('devel/email_reproduciblebuilds.txt')
+ enabled_users = [prof.user for prof in UserProfile.objects.filter(rebuilderd_updates=True).all()]
+
+ # Group statusses by maintainer
+ maintainers_map = defaultdict(list)
+
+ for status in was_repro:
+ for maintainer in status.pkg.maintainers:
+ if maintainer not in enabled_users:
+ continue
+
+ maintainers_map[maintainer.userprofile].append(status.pkg.pkgname)
+ for maintainer, pkgs in maintainers_map.items():
+ send_mail('Packages which have become not reproducible',
+ template.render({'pkgs': pkgs}),
+ 'Arch Website Notification <nobody@archlinux.org>',
+ [maintainer.public_email],
+ fail_silently=True)
+
+
+def import_rebuilderd_status(url):
+ statuses = []
+ was_repro = []
+
+ req = requests.get(url)
+ data = req.json()
+
+ for pkg in data:
+ arch = Arch.objects.get(name=pkg['architecture'])
+ repository = Repo.objects.get(name__iexact=pkg['suite'])
+
+ epoch = 0
+ pkgname = pkg['name']
+ version = pkg['version']
+
+ matches = re.search(EPOCH_REGEX, version)
+ if matches:
+ epoch = matches.group(1)
+
+ pkgver, pkgrel = pkg['version'].rsplit('-', 1)
+
+ dbpkg = Package.objects.filter(pkgname=pkgname, pkgver=pkgver,
+ pkgrel=pkgrel, epoch=epoch,
+ repo=repository,
+ arch=arch).first()
+ if not dbpkg:
+ continue
+
+ rbstatus = RebuilderdStatus.objects.filter(pkg=dbpkg).first()
+ status = RebuilderdStatus.REBUILDERD_API_STATUSES.get(pkg['status'], RebuilderdStatus.UNKNOWN)
+
+ # Existing status
+ if rbstatus:
+ # If status has become BAD, set was_repro
+ if rbstatus.status == RebuilderdStatus.GOOD and status == RebuilderdStatus.BAD:
+ was_repro.append(rbstatus)
+ rbstatus.was_repro = True
+ logger.info("package '%s' was good is now bad", pkg['name'])
+ else: # reset status
+ rbstatus.was_repro = False
+
+ if rbstatus.pkgver != pkgver or rbstatus.pkgrel != pkgrel or rbstatus.epoch != epoch:
+ logger.info('updating status for package: %s', pkg['name'])
+ rbstatus.epoch = epoch
+ rbstatus.pkgver = pkgver
+ rbstatus.pkgrel = pkgrel
+ rbstatus.status = status
+ rbstatus.arch = arch
+ rbstatus.repo = repository
+ elif rbstatus.status != status: # Rebuilderd rebuild the same package?
+ logger.info('status for package: %s changed', pkg['name'])
+ rbstatus.status = status
+
+ # TODO: does django know when a model was really modified?
+ rbstatus.save()
+
+ else: # new package/status
+ logger.info('adding status for package: %s', pkg['name'])
+ rbstatus = RebuilderdStatus(pkg=dbpkg, status=status, arch=arch, repo=repository,
+ pkgname=pkgname, epoch=epoch, pkgrel=pkgrel,
+ pkgver=pkgver)
+ statuses.append(rbstatus)
+
+ if statuses:
+ RebuilderdStatus.objects.bulk_create(statuses)
+
+ return was_repro
diff --git a/devel/migrations/0006_userprofile_rebuilderd_updates.py b/devel/migrations/0006_userprofile_rebuilderd_updates.py
new file mode 100644
index 00000000..711ea5ea
--- /dev/null
+++ b/devel/migrations/0006_userprofile_rebuilderd_updates.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.0.7 on 2020-06-28 21:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('devel', '0005_auto_20200628_1600'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='userprofile',
+ name='rebuilderd_updates',
+ field=models.BooleanField(default=False, help_text='Receive reproducible build package updates'),
+ ),
+ ]
diff --git a/devel/models.py b/devel/models.py
index d7b6c379..59324746 100644
--- a/devel/models.py
+++ b/devel/models.py
@@ -50,6 +50,8 @@ class UserProfile(models.Model):
allowed_repos = models.ManyToManyField('main.Repo', blank=True)
latin_name = models.CharField(max_length=255, null=True, blank=True,
help_text="Latin-form name; used only for non-Latin full names")
+ rebuilderd_updates = models.BooleanField(default=False,
+ help_text='Receive reproducible build package updates')
last_modified = models.DateTimeField(editable=False)
class Meta:
diff --git a/devel/reports.py b/devel/reports.py
index b4f9d794..6113195a 100644
--- a/devel/reports.py
+++ b/devel/reports.py
@@ -5,7 +5,7 @@ from django.db.models import F
from django.template.defaultfilters import filesizeformat
from django.db import connection
from django.utils.timezone import now
-from main.models import Package, PackageFile
+from main.models import Package, PackageFile, RebuilderdStatus
from packages.models import Depend, PackageRelation
from .models import DeveloperKey
@@ -145,6 +145,7 @@ def signature_time(packages):
return filtered
+
def non_existing_dependencies(packages):
cursor = connection.cursor()
query = """
@@ -167,6 +168,10 @@ def non_existing_dependencies(packages):
return packages
+def non_reproducible_packages(packages):
+ statuses = RebuilderdStatus.objects.exclude(status=RebuilderdStatus.GOOD).values('pkg__pkgname')
+ return packages.filter(pkgname__in=statuses)
+
REPORT_OLD = DeveloperReport(
'old', 'Old', 'Packages last built more than two years ago', old)
@@ -223,6 +228,14 @@ NON_EXISTING_DEPENDENCIES = DeveloperReport(
['nonexistingdep'],
personal=False)
+REBUILDERD_PACKAGES = DeveloperReport(
+ 'non-reproducible-packages',
+ 'Non Reproducible package',
+ 'Packages that are not reproducible on our reproducible.archlinux.org test environment',
+ non_reproducible_packages,
+ )
+
+
def available_reports():
return (REPORT_OLD,
REPORT_OUTOFDATE,
@@ -233,4 +246,5 @@ def available_reports():
REPORT_ORPHANS,
REPORT_SIGNATURE,
REPORT_SIG_TIME,
- NON_EXISTING_DEPENDENCIES, )
+ NON_EXISTING_DEPENDENCIES,
+ REBUILDERD_PACKAGES, )
diff --git a/main/migrations/0003_rebuilderdstatus.py b/main/migrations/0003_rebuilderdstatus.py
new file mode 100644
index 00000000..5cc0ad59
--- /dev/null
+++ b/main/migrations/0003_rebuilderdstatus.py
@@ -0,0 +1,29 @@
+# Generated by Django 3.0.7 on 2020-06-28 19:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('main', '0002_repo_public_testing'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='RebuilderdStatus',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('pkgname', models.CharField(max_length=255)),
+ ('pkgver', models.CharField(max_length=255)),
+ ('pkgrel', models.CharField(max_length=255)),
+ ('epoch', models.PositiveIntegerField(default=0)),
+ ('status', models.SmallIntegerField(choices=[(0, 'Good'), (1, 'Bad'), (2, 'Unknown')], default=2)),
+ ('was_repro', models.BooleanField(default=False)),
+ ('arch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Arch')),
+ ('pkg', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Package')),
+ ('repo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='main.Repo')),
+ ],
+ ),
+ ]
diff --git a/main/models.py b/main/models.py
index 094e80a9..7f60276c 100644
--- a/main/models.py
+++ b/main/models.py
@@ -444,6 +444,41 @@ class PackageFile(models.Model):
db_table = 'package_files'
+class RebuilderdStatus(models.Model):
+ # https://github.com/kpcyrd/rebuilderd/blob/2eeef725940ca8710c7c21579ce093445d1875c8/common/src/api.rs#L279
+ GOOD = 0
+ BAD = 1
+ UNKNOWN = 2
+
+ REBUILDERD_API_STATUSES = {
+ 'GOOD': GOOD,
+ 'BAD': BAD,
+ 'UNKWN': UNKNOWN,
+ }
+ REBUILDERD_STATUSES = (
+ (GOOD, 'Good'),
+ (BAD, 'Bad'),
+ (UNKNOWN, 'Unknown'),
+ )
+
+ pkg = models.ForeignKey(Package, on_delete=models.CASCADE)
+ pkgname = models.CharField(max_length=255)
+ pkgver = models.CharField(max_length=255)
+ pkgrel = models.CharField(max_length=255)
+ epoch = models.PositiveIntegerField(default=0)
+ arch = models.ForeignKey(Arch, on_delete=models.CASCADE)
+ repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
+ status = models.SmallIntegerField(default=UNKNOWN, choices=REBUILDERD_STATUSES)
+ was_repro = models.BooleanField(default=False)
+
+ @property
+ def status_str(self):
+ return self.REBUILDERD_STATUSES[self.status][1]
+
+ def __str__(self):
+ return "pkg=%s, status=%s" % (self.pkg, self.status_str)
+
+
from django.db.models.signals import pre_save
# note: reporead sets the 'created' field on Package objects, so no signal
diff --git a/requirements.txt b/requirements.txt
index efbf68b7..548cba8b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -15,3 +15,4 @@ django-csp==3.6
ptpython==2.0.4
feedparser==5.2.1
bleach==3.1.5
+requests==2.22.0
diff --git a/settings.py b/settings.py
index ec1d9d24..b68332e3 100644
--- a/settings.py
+++ b/settings.py
@@ -195,6 +195,9 @@ DATABASES = {
# Planet limit of items per feed to keep the feed size in check.
RSS_FEED_LIMIT = 25
+# Rebuilderd API endpoint
+REBUILDERD_URL = 'https://reproducible.archlinux.org/api/v0/pkgs/list'
+
# Import local settings
try:
from local_settings import *
diff --git a/templates/devel/email_reproduciblebuilds.txt b/templates/devel/email_reproduciblebuilds.txt
new file mode 100644
index 00000000..385bd20f
--- /dev/null
+++ b/templates/devel/email_reproduciblebuilds.txt
@@ -0,0 +1,4 @@
+The following packages have become non reproducible:
+{% for pkg in pkgs %}
+* {{ pkg }}
+{% endfor %}