From 17a12f07fb04e4e3d7155be39d5e90a57d6dce61 Mon Sep 17 00:00:00 2001 From: Jelle van der Waa Date: Tue, 8 Oct 2019 15:52:38 +0200 Subject: Introduce planet functionality in archweb This change introduces a replacment for planet.archlinux.org which uses a python 2 project to generate static html from multiple RSS feed sources. For archweb a set of 'static' feeds can be created in the django admin view for the Arch forums and other static feeds, archweb users can add their own blog rss feed in their profile which will create a Feed model. When running the update_planet command, all Feed models are iterated over and the rss feed is parsed. The latest FeedItem is queried matching the current Feed model and every newer entry in the RSS feed is added as new FeedItem. Since the body is also stored in the FeedItem there is a limit to the amount of FeedItems per Feed configured in settings.py of which the default is 25. When a user is marked as inactive his Feed model and items are removed automatically to avoid keeping stale data around. Closes: #261 --- devel/migrations/0003_auto_20191009_1924.py | 18 +++ devel/migrations/0004_userprofile_website_rss.py | 18 +++ devel/models.py | 67 ++++++++++- feeds.py | 46 +++++++ planet/__init__.py | 0 planet/admin.py | 25 ++++ planet/management/__init__.py | 0 planet/management/commands/__init__.py | 0 planet/management/commands/update_planet.py | 145 +++++++++++++++++++++++ planet/migrations/0001_initial.py | 62 ++++++++++ planet/migrations/__init__.py | 0 planet/models.py | 60 ++++++++++ planet/tests/__init__.py | 0 planet/tests/test_views.py | 12 ++ planet/views.py | 14 +++ public/management/__init__.py | 0 requirements.txt | 2 + settings.py | 4 + sitestatic/archweb.css | 50 ++++++++ templates/planet/index.html | 59 +++++++++ urls.py | 5 +- 21 files changed, 584 insertions(+), 3 deletions(-) create mode 100644 devel/migrations/0003_auto_20191009_1924.py create mode 100644 devel/migrations/0004_userprofile_website_rss.py create mode 100644 planet/__init__.py create mode 100644 planet/admin.py create mode 100644 planet/management/__init__.py create mode 100644 planet/management/commands/__init__.py create mode 100644 planet/management/commands/update_planet.py create mode 100644 planet/migrations/0001_initial.py create mode 100644 planet/migrations/__init__.py create mode 100644 planet/models.py create mode 100644 planet/tests/__init__.py create mode 100644 planet/tests/test_views.py create mode 100644 planet/views.py create mode 100644 public/management/__init__.py create mode 100644 templates/planet/index.html diff --git a/devel/migrations/0003_auto_20191009_1924.py b/devel/migrations/0003_auto_20191009_1924.py new file mode 100644 index 00000000..e292a877 --- /dev/null +++ b/devel/migrations/0003_auto_20191009_1924.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.5 on 2019-10-09 19:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('devel', '0002_auto_20181216_1605'), + ] + + operations = [ + migrations.AlterField( + model_name='userprofile', + name='time_zone', + field=models.CharField(choices=[('Africa/Abidjan', 'Africa/Abidjan'), ('Africa/Accra', 'Africa/Accra'), ('Africa/Addis_Ababa', 'Africa/Addis_Ababa'), ('Africa/Algiers', 'Africa/Algiers'), ('Africa/Asmara', 'Africa/Asmara'), ('Africa/Bamako', 'Africa/Bamako'), ('Africa/Bangui', 'Africa/Bangui'), ('Africa/Banjul', 'Africa/Banjul'), ('Africa/Bissau', 'Africa/Bissau'), ('Africa/Blantyre', 'Africa/Blantyre'), ('Africa/Brazzaville', 'Africa/Brazzaville'), ('Africa/Bujumbura', 'Africa/Bujumbura'), ('Africa/Cairo', 'Africa/Cairo'), ('Africa/Casablanca', 'Africa/Casablanca'), ('Africa/Ceuta', 'Africa/Ceuta'), ('Africa/Conakry', 'Africa/Conakry'), ('Africa/Dakar', 'Africa/Dakar'), ('Africa/Dar_es_Salaam', 'Africa/Dar_es_Salaam'), ('Africa/Djibouti', 'Africa/Djibouti'), ('Africa/Douala', 'Africa/Douala'), ('Africa/El_Aaiun', 'Africa/El_Aaiun'), ('Africa/Freetown', 'Africa/Freetown'), ('Africa/Gaborone', 'Africa/Gaborone'), ('Africa/Harare', 'Africa/Harare'), ('Africa/Johannesburg', 'Africa/Johannesburg'), ('Africa/Juba', 'Africa/Juba'), ('Africa/Kampala', 'Africa/Kampala'), ('Africa/Khartoum', 'Africa/Khartoum'), ('Africa/Kigali', 'Africa/Kigali'), ('Africa/Kinshasa', 'Africa/Kinshasa'), ('Africa/Lagos', 'Africa/Lagos'), ('Africa/Libreville', 'Africa/Libreville'), ('Africa/Lome', 'Africa/Lome'), ('Africa/Luanda', 'Africa/Luanda'), ('Africa/Lubumbashi', 'Africa/Lubumbashi'), ('Africa/Lusaka', 'Africa/Lusaka'), ('Africa/Malabo', 'Africa/Malabo'), ('Africa/Maputo', 'Africa/Maputo'), ('Africa/Maseru', 'Africa/Maseru'), ('Africa/Mbabane', 'Africa/Mbabane'), ('Africa/Mogadishu', 'Africa/Mogadishu'), ('Africa/Monrovia', 'Africa/Monrovia'), ('Africa/Nairobi', 'Africa/Nairobi'), ('Africa/Ndjamena', 'Africa/Ndjamena'), ('Africa/Niamey', 'Africa/Niamey'), ('Africa/Nouakchott', 'Africa/Nouakchott'), ('Africa/Ouagadougou', 'Africa/Ouagadougou'), ('Africa/Porto-Novo', 'Africa/Porto-Novo'), ('Africa/Sao_Tome', 'Africa/Sao_Tome'), ('Africa/Tripoli', 'Africa/Tripoli'), ('Africa/Tunis', 'Africa/Tunis'), ('Africa/Windhoek', 'Africa/Windhoek'), ('America/Adak', 'America/Adak'), ('America/Anchorage', 'America/Anchorage'), ('America/Anguilla', 'America/Anguilla'), ('America/Antigua', 'America/Antigua'), ('America/Araguaina', 'America/Araguaina'), ('America/Argentina/Buenos_Aires', 'America/Argentina/Buenos_Aires'), ('America/Argentina/Catamarca', 'America/Argentina/Catamarca'), ('America/Argentina/Cordoba', 'America/Argentina/Cordoba'), ('America/Argentina/Jujuy', 'America/Argentina/Jujuy'), ('America/Argentina/La_Rioja', 'America/Argentina/La_Rioja'), ('America/Argentina/Mendoza', 'America/Argentina/Mendoza'), ('America/Argentina/Rio_Gallegos', 'America/Argentina/Rio_Gallegos'), ('America/Argentina/Salta', 'America/Argentina/Salta'), ('America/Argentina/San_Juan', 'America/Argentina/San_Juan'), ('America/Argentina/San_Luis', 'America/Argentina/San_Luis'), ('America/Argentina/Tucuman', 'America/Argentina/Tucuman'), ('America/Argentina/Ushuaia', 'America/Argentina/Ushuaia'), ('America/Aruba', 'America/Aruba'), ('America/Asuncion', 'America/Asuncion'), ('America/Atikokan', 'America/Atikokan'), ('America/Bahia', 'America/Bahia'), ('America/Bahia_Banderas', 'America/Bahia_Banderas'), ('America/Barbados', 'America/Barbados'), ('America/Belem', 'America/Belem'), ('America/Belize', 'America/Belize'), ('America/Blanc-Sablon', 'America/Blanc-Sablon'), ('America/Boa_Vista', 'America/Boa_Vista'), ('America/Bogota', 'America/Bogota'), ('America/Boise', 'America/Boise'), ('America/Cambridge_Bay', 'America/Cambridge_Bay'), ('America/Campo_Grande', 'America/Campo_Grande'), ('America/Cancun', 'America/Cancun'), ('America/Caracas', 'America/Caracas'), ('America/Cayenne', 'America/Cayenne'), ('America/Cayman', 'America/Cayman'), ('America/Chicago', 'America/Chicago'), ('America/Chihuahua', 'America/Chihuahua'), ('America/Costa_Rica', 'America/Costa_Rica'), ('America/Creston', 'America/Creston'), ('America/Cuiaba', 'America/Cuiaba'), ('America/Curacao', 'America/Curacao'), ('America/Danmarkshavn', 'America/Danmarkshavn'), ('America/Dawson', 'America/Dawson'), ('America/Dawson_Creek', 'America/Dawson_Creek'), ('America/Denver', 'America/Denver'), ('America/Detroit', 'America/Detroit'), ('America/Dominica', 'America/Dominica'), ('America/Edmonton', 'America/Edmonton'), ('America/Eirunepe', 'America/Eirunepe'), ('America/El_Salvador', 'America/El_Salvador'), ('America/Fort_Nelson', 'America/Fort_Nelson'), ('America/Fortaleza', 'America/Fortaleza'), ('America/Glace_Bay', 'America/Glace_Bay'), ('America/Godthab', 'America/Godthab'), ('America/Goose_Bay', 'America/Goose_Bay'), ('America/Grand_Turk', 'America/Grand_Turk'), ('America/Grenada', 'America/Grenada'), ('America/Guadeloupe', 'America/Guadeloupe'), ('America/Guatemala', 'America/Guatemala'), ('America/Guayaquil', 'America/Guayaquil'), ('America/Guyana', 'America/Guyana'), ('America/Halifax', 'America/Halifax'), ('America/Havana', 'America/Havana'), ('America/Hermosillo', 'America/Hermosillo'), ('America/Indiana/Indianapolis', 'America/Indiana/Indianapolis'), ('America/Indiana/Knox', 'America/Indiana/Knox'), ('America/Indiana/Marengo', 'America/Indiana/Marengo'), ('America/Indiana/Petersburg', 'America/Indiana/Petersburg'), ('America/Indiana/Tell_City', 'America/Indiana/Tell_City'), ('America/Indiana/Vevay', 'America/Indiana/Vevay'), ('America/Indiana/Vincennes', 'America/Indiana/Vincennes'), ('America/Indiana/Winamac', 'America/Indiana/Winamac'), ('America/Inuvik', 'America/Inuvik'), ('America/Iqaluit', 'America/Iqaluit'), ('America/Jamaica', 'America/Jamaica'), ('America/Juneau', 'America/Juneau'), ('America/Kentucky/Louisville', 'America/Kentucky/Louisville'), ('America/Kentucky/Monticello', 'America/Kentucky/Monticello'), ('America/Kralendijk', 'America/Kralendijk'), ('America/La_Paz', 'America/La_Paz'), ('America/Lima', 'America/Lima'), ('America/Los_Angeles', 'America/Los_Angeles'), ('America/Lower_Princes', 'America/Lower_Princes'), ('America/Maceio', 'America/Maceio'), ('America/Managua', 'America/Managua'), ('America/Manaus', 'America/Manaus'), ('America/Marigot', 'America/Marigot'), ('America/Martinique', 'America/Martinique'), ('America/Matamoros', 'America/Matamoros'), ('America/Mazatlan', 'America/Mazatlan'), ('America/Menominee', 'America/Menominee'), ('America/Merida', 'America/Merida'), ('America/Metlakatla', 'America/Metlakatla'), ('America/Mexico_City', 'America/Mexico_City'), ('America/Miquelon', 'America/Miquelon'), ('America/Moncton', 'America/Moncton'), ('America/Monterrey', 'America/Monterrey'), ('America/Montevideo', 'America/Montevideo'), ('America/Montserrat', 'America/Montserrat'), ('America/Nassau', 'America/Nassau'), ('America/New_York', 'America/New_York'), ('America/Nipigon', 'America/Nipigon'), ('America/Nome', 'America/Nome'), ('America/Noronha', 'America/Noronha'), ('America/North_Dakota/Beulah', 'America/North_Dakota/Beulah'), ('America/North_Dakota/Center', 'America/North_Dakota/Center'), ('America/North_Dakota/New_Salem', 'America/North_Dakota/New_Salem'), ('America/Ojinaga', 'America/Ojinaga'), ('America/Panama', 'America/Panama'), ('America/Pangnirtung', 'America/Pangnirtung'), ('America/Paramaribo', 'America/Paramaribo'), ('America/Phoenix', 'America/Phoenix'), ('America/Port-au-Prince', 'America/Port-au-Prince'), ('America/Port_of_Spain', 'America/Port_of_Spain'), ('America/Porto_Velho', 'America/Porto_Velho'), ('America/Puerto_Rico', 'America/Puerto_Rico'), ('America/Punta_Arenas', 'America/Punta_Arenas'), ('America/Rainy_River', 'America/Rainy_River'), ('America/Rankin_Inlet', 'America/Rankin_Inlet'), ('America/Recife', 'America/Recife'), ('America/Regina', 'America/Regina'), ('America/Resolute', 'America/Resolute'), ('America/Rio_Branco', 'America/Rio_Branco'), ('America/Santarem', 'America/Santarem'), ('America/Santiago', 'America/Santiago'), ('America/Santo_Domingo', 'America/Santo_Domingo'), ('America/Sao_Paulo', 'America/Sao_Paulo'), ('America/Scoresbysund', 'America/Scoresbysund'), ('America/Sitka', 'America/Sitka'), ('America/St_Barthelemy', 'America/St_Barthelemy'), ('America/St_Johns', 'America/St_Johns'), ('America/St_Kitts', 'America/St_Kitts'), ('America/St_Lucia', 'America/St_Lucia'), ('America/St_Thomas', 'America/St_Thomas'), ('America/St_Vincent', 'America/St_Vincent'), ('America/Swift_Current', 'America/Swift_Current'), ('America/Tegucigalpa', 'America/Tegucigalpa'), ('America/Thule', 'America/Thule'), ('America/Thunder_Bay', 'America/Thunder_Bay'), ('America/Tijuana', 'America/Tijuana'), ('America/Toronto', 'America/Toronto'), ('America/Tortola', 'America/Tortola'), ('America/Vancouver', 'America/Vancouver'), ('America/Whitehorse', 'America/Whitehorse'), ('America/Winnipeg', 'America/Winnipeg'), ('America/Yakutat', 'America/Yakutat'), ('America/Yellowknife', 'America/Yellowknife'), ('Antarctica/Casey', 'Antarctica/Casey'), ('Antarctica/Davis', 'Antarctica/Davis'), ('Antarctica/DumontDUrville', 'Antarctica/DumontDUrville'), ('Antarctica/Macquarie', 'Antarctica/Macquarie'), ('Antarctica/Mawson', 'Antarctica/Mawson'), ('Antarctica/McMurdo', 'Antarctica/McMurdo'), ('Antarctica/Palmer', 'Antarctica/Palmer'), ('Antarctica/Rothera', 'Antarctica/Rothera'), ('Antarctica/Syowa', 'Antarctica/Syowa'), ('Antarctica/Troll', 'Antarctica/Troll'), ('Antarctica/Vostok', 'Antarctica/Vostok'), ('Arctic/Longyearbyen', 'Arctic/Longyearbyen'), ('Asia/Aden', 'Asia/Aden'), ('Asia/Almaty', 'Asia/Almaty'), ('Asia/Amman', 'Asia/Amman'), ('Asia/Anadyr', 'Asia/Anadyr'), ('Asia/Aqtau', 'Asia/Aqtau'), ('Asia/Aqtobe', 'Asia/Aqtobe'), ('Asia/Ashgabat', 'Asia/Ashgabat'), ('Asia/Atyrau', 'Asia/Atyrau'), ('Asia/Baghdad', 'Asia/Baghdad'), ('Asia/Bahrain', 'Asia/Bahrain'), ('Asia/Baku', 'Asia/Baku'), ('Asia/Bangkok', 'Asia/Bangkok'), ('Asia/Barnaul', 'Asia/Barnaul'), ('Asia/Beirut', 'Asia/Beirut'), ('Asia/Bishkek', 'Asia/Bishkek'), ('Asia/Brunei', 'Asia/Brunei'), ('Asia/Chita', 'Asia/Chita'), ('Asia/Choibalsan', 'Asia/Choibalsan'), ('Asia/Colombo', 'Asia/Colombo'), ('Asia/Damascus', 'Asia/Damascus'), ('Asia/Dhaka', 'Asia/Dhaka'), ('Asia/Dili', 'Asia/Dili'), ('Asia/Dubai', 'Asia/Dubai'), ('Asia/Dushanbe', 'Asia/Dushanbe'), ('Asia/Famagusta', 'Asia/Famagusta'), ('Asia/Gaza', 'Asia/Gaza'), ('Asia/Hebron', 'Asia/Hebron'), ('Asia/Ho_Chi_Minh', 'Asia/Ho_Chi_Minh'), ('Asia/Hong_Kong', 'Asia/Hong_Kong'), ('Asia/Hovd', 'Asia/Hovd'), ('Asia/Irkutsk', 'Asia/Irkutsk'), ('Asia/Jakarta', 'Asia/Jakarta'), ('Asia/Jayapura', 'Asia/Jayapura'), ('Asia/Jerusalem', 'Asia/Jerusalem'), ('Asia/Kabul', 'Asia/Kabul'), ('Asia/Kamchatka', 'Asia/Kamchatka'), ('Asia/Karachi', 'Asia/Karachi'), ('Asia/Kathmandu', 'Asia/Kathmandu'), ('Asia/Khandyga', 'Asia/Khandyga'), ('Asia/Kolkata', 'Asia/Kolkata'), ('Asia/Krasnoyarsk', 'Asia/Krasnoyarsk'), ('Asia/Kuala_Lumpur', 'Asia/Kuala_Lumpur'), ('Asia/Kuching', 'Asia/Kuching'), ('Asia/Kuwait', 'Asia/Kuwait'), ('Asia/Macau', 'Asia/Macau'), ('Asia/Magadan', 'Asia/Magadan'), ('Asia/Makassar', 'Asia/Makassar'), ('Asia/Manila', 'Asia/Manila'), ('Asia/Muscat', 'Asia/Muscat'), ('Asia/Nicosia', 'Asia/Nicosia'), ('Asia/Novokuznetsk', 'Asia/Novokuznetsk'), ('Asia/Novosibirsk', 'Asia/Novosibirsk'), ('Asia/Omsk', 'Asia/Omsk'), ('Asia/Oral', 'Asia/Oral'), ('Asia/Phnom_Penh', 'Asia/Phnom_Penh'), ('Asia/Pontianak', 'Asia/Pontianak'), ('Asia/Pyongyang', 'Asia/Pyongyang'), ('Asia/Qatar', 'Asia/Qatar'), ('Asia/Qostanay', 'Asia/Qostanay'), ('Asia/Qyzylorda', 'Asia/Qyzylorda'), ('Asia/Riyadh', 'Asia/Riyadh'), ('Asia/Sakhalin', 'Asia/Sakhalin'), ('Asia/Samarkand', 'Asia/Samarkand'), ('Asia/Seoul', 'Asia/Seoul'), ('Asia/Shanghai', 'Asia/Shanghai'), ('Asia/Singapore', 'Asia/Singapore'), ('Asia/Srednekolymsk', 'Asia/Srednekolymsk'), ('Asia/Taipei', 'Asia/Taipei'), ('Asia/Tashkent', 'Asia/Tashkent'), ('Asia/Tbilisi', 'Asia/Tbilisi'), ('Asia/Tehran', 'Asia/Tehran'), ('Asia/Thimphu', 'Asia/Thimphu'), ('Asia/Tokyo', 'Asia/Tokyo'), ('Asia/Tomsk', 'Asia/Tomsk'), ('Asia/Ulaanbaatar', 'Asia/Ulaanbaatar'), ('Asia/Urumqi', 'Asia/Urumqi'), ('Asia/Ust-Nera', 'Asia/Ust-Nera'), ('Asia/Vientiane', 'Asia/Vientiane'), ('Asia/Vladivostok', 'Asia/Vladivostok'), ('Asia/Yakutsk', 'Asia/Yakutsk'), ('Asia/Yangon', 'Asia/Yangon'), ('Asia/Yekaterinburg', 'Asia/Yekaterinburg'), ('Asia/Yerevan', 'Asia/Yerevan'), ('Atlantic/Azores', 'Atlantic/Azores'), ('Atlantic/Bermuda', 'Atlantic/Bermuda'), ('Atlantic/Canary', 'Atlantic/Canary'), ('Atlantic/Cape_Verde', 'Atlantic/Cape_Verde'), ('Atlantic/Faroe', 'Atlantic/Faroe'), ('Atlantic/Madeira', 'Atlantic/Madeira'), ('Atlantic/Reykjavik', 'Atlantic/Reykjavik'), ('Atlantic/South_Georgia', 'Atlantic/South_Georgia'), ('Atlantic/St_Helena', 'Atlantic/St_Helena'), ('Atlantic/Stanley', 'Atlantic/Stanley'), ('Australia/Adelaide', 'Australia/Adelaide'), ('Australia/Brisbane', 'Australia/Brisbane'), ('Australia/Broken_Hill', 'Australia/Broken_Hill'), ('Australia/Currie', 'Australia/Currie'), ('Australia/Darwin', 'Australia/Darwin'), ('Australia/Eucla', 'Australia/Eucla'), ('Australia/Hobart', 'Australia/Hobart'), ('Australia/Lindeman', 'Australia/Lindeman'), ('Australia/Lord_Howe', 'Australia/Lord_Howe'), ('Australia/Melbourne', 'Australia/Melbourne'), ('Australia/Perth', 'Australia/Perth'), ('Australia/Sydney', 'Australia/Sydney'), ('Canada/Atlantic', 'Canada/Atlantic'), ('Canada/Central', 'Canada/Central'), ('Canada/Eastern', 'Canada/Eastern'), ('Canada/Mountain', 'Canada/Mountain'), ('Canada/Newfoundland', 'Canada/Newfoundland'), ('Canada/Pacific', 'Canada/Pacific'), ('Europe/Amsterdam', 'Europe/Amsterdam'), ('Europe/Andorra', 'Europe/Andorra'), ('Europe/Astrakhan', 'Europe/Astrakhan'), ('Europe/Athens', 'Europe/Athens'), ('Europe/Belgrade', 'Europe/Belgrade'), ('Europe/Berlin', 'Europe/Berlin'), ('Europe/Bratislava', 'Europe/Bratislava'), ('Europe/Brussels', 'Europe/Brussels'), ('Europe/Bucharest', 'Europe/Bucharest'), ('Europe/Budapest', 'Europe/Budapest'), ('Europe/Busingen', 'Europe/Busingen'), ('Europe/Chisinau', 'Europe/Chisinau'), ('Europe/Copenhagen', 'Europe/Copenhagen'), ('Europe/Dublin', 'Europe/Dublin'), ('Europe/Gibraltar', 'Europe/Gibraltar'), ('Europe/Guernsey', 'Europe/Guernsey'), ('Europe/Helsinki', 'Europe/Helsinki'), ('Europe/Isle_of_Man', 'Europe/Isle_of_Man'), ('Europe/Istanbul', 'Europe/Istanbul'), ('Europe/Jersey', 'Europe/Jersey'), ('Europe/Kaliningrad', 'Europe/Kaliningrad'), ('Europe/Kiev', 'Europe/Kiev'), ('Europe/Kirov', 'Europe/Kirov'), ('Europe/Lisbon', 'Europe/Lisbon'), ('Europe/Ljubljana', 'Europe/Ljubljana'), ('Europe/London', 'Europe/London'), ('Europe/Luxembourg', 'Europe/Luxembourg'), ('Europe/Madrid', 'Europe/Madrid'), ('Europe/Malta', 'Europe/Malta'), ('Europe/Mariehamn', 'Europe/Mariehamn'), ('Europe/Minsk', 'Europe/Minsk'), ('Europe/Monaco', 'Europe/Monaco'), ('Europe/Moscow', 'Europe/Moscow'), ('Europe/Oslo', 'Europe/Oslo'), ('Europe/Paris', 'Europe/Paris'), ('Europe/Podgorica', 'Europe/Podgorica'), ('Europe/Prague', 'Europe/Prague'), ('Europe/Riga', 'Europe/Riga'), ('Europe/Rome', 'Europe/Rome'), ('Europe/Samara', 'Europe/Samara'), ('Europe/San_Marino', 'Europe/San_Marino'), ('Europe/Sarajevo', 'Europe/Sarajevo'), ('Europe/Saratov', 'Europe/Saratov'), ('Europe/Simferopol', 'Europe/Simferopol'), ('Europe/Skopje', 'Europe/Skopje'), ('Europe/Sofia', 'Europe/Sofia'), ('Europe/Stockholm', 'Europe/Stockholm'), ('Europe/Tallinn', 'Europe/Tallinn'), ('Europe/Tirane', 'Europe/Tirane'), ('Europe/Ulyanovsk', 'Europe/Ulyanovsk'), ('Europe/Uzhgorod', 'Europe/Uzhgorod'), ('Europe/Vaduz', 'Europe/Vaduz'), ('Europe/Vatican', 'Europe/Vatican'), ('Europe/Vienna', 'Europe/Vienna'), ('Europe/Vilnius', 'Europe/Vilnius'), ('Europe/Volgograd', 'Europe/Volgograd'), ('Europe/Warsaw', 'Europe/Warsaw'), ('Europe/Zagreb', 'Europe/Zagreb'), ('Europe/Zaporozhye', 'Europe/Zaporozhye'), ('Europe/Zurich', 'Europe/Zurich'), ('GMT', 'GMT'), ('Indian/Antananarivo', 'Indian/Antananarivo'), ('Indian/Chagos', 'Indian/Chagos'), ('Indian/Christmas', 'Indian/Christmas'), ('Indian/Cocos', 'Indian/Cocos'), ('Indian/Comoro', 'Indian/Comoro'), ('Indian/Kerguelen', 'Indian/Kerguelen'), ('Indian/Mahe', 'Indian/Mahe'), ('Indian/Maldives', 'Indian/Maldives'), ('Indian/Mauritius', 'Indian/Mauritius'), ('Indian/Mayotte', 'Indian/Mayotte'), ('Indian/Reunion', 'Indian/Reunion'), ('Pacific/Apia', 'Pacific/Apia'), ('Pacific/Auckland', 'Pacific/Auckland'), ('Pacific/Bougainville', 'Pacific/Bougainville'), ('Pacific/Chatham', 'Pacific/Chatham'), ('Pacific/Chuuk', 'Pacific/Chuuk'), ('Pacific/Easter', 'Pacific/Easter'), ('Pacific/Efate', 'Pacific/Efate'), ('Pacific/Enderbury', 'Pacific/Enderbury'), ('Pacific/Fakaofo', 'Pacific/Fakaofo'), ('Pacific/Fiji', 'Pacific/Fiji'), ('Pacific/Funafuti', 'Pacific/Funafuti'), ('Pacific/Galapagos', 'Pacific/Galapagos'), ('Pacific/Gambier', 'Pacific/Gambier'), ('Pacific/Guadalcanal', 'Pacific/Guadalcanal'), ('Pacific/Guam', 'Pacific/Guam'), ('Pacific/Honolulu', 'Pacific/Honolulu'), ('Pacific/Kiritimati', 'Pacific/Kiritimati'), ('Pacific/Kosrae', 'Pacific/Kosrae'), ('Pacific/Kwajalein', 'Pacific/Kwajalein'), ('Pacific/Majuro', 'Pacific/Majuro'), ('Pacific/Marquesas', 'Pacific/Marquesas'), ('Pacific/Midway', 'Pacific/Midway'), ('Pacific/Nauru', 'Pacific/Nauru'), ('Pacific/Niue', 'Pacific/Niue'), ('Pacific/Norfolk', 'Pacific/Norfolk'), ('Pacific/Noumea', 'Pacific/Noumea'), ('Pacific/Pago_Pago', 'Pacific/Pago_Pago'), ('Pacific/Palau', 'Pacific/Palau'), ('Pacific/Pitcairn', 'Pacific/Pitcairn'), ('Pacific/Pohnpei', 'Pacific/Pohnpei'), ('Pacific/Port_Moresby', 'Pacific/Port_Moresby'), ('Pacific/Rarotonga', 'Pacific/Rarotonga'), ('Pacific/Saipan', 'Pacific/Saipan'), ('Pacific/Tahiti', 'Pacific/Tahiti'), ('Pacific/Tarawa', 'Pacific/Tarawa'), ('Pacific/Tongatapu', 'Pacific/Tongatapu'), ('Pacific/Wake', 'Pacific/Wake'), ('Pacific/Wallis', 'Pacific/Wallis'), ('US/Alaska', 'US/Alaska'), ('US/Arizona', 'US/Arizona'), ('US/Central', 'US/Central'), ('US/Eastern', 'US/Eastern'), ('US/Hawaii', 'US/Hawaii'), ('US/Mountain', 'US/Mountain'), ('US/Pacific', 'US/Pacific'), ('UTC', 'UTC')], default='UTC', help_text='Used for developer clock page', max_length=100), + ), + ] diff --git a/devel/migrations/0004_userprofile_website_rss.py b/devel/migrations/0004_userprofile_website_rss.py new file mode 100644 index 00000000..8014f4b1 --- /dev/null +++ b/devel/migrations/0004_userprofile_website_rss.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2019-11-24 16:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('devel', '0003_auto_20191009_1924'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='website_rss', + field=models.CharField(blank=True, max_length=200, null=True), + ), + ] diff --git a/devel/models.py b/devel/models.py index 92118a00..d7b6c379 100644 --- a/devel/models.py +++ b/devel/models.py @@ -3,13 +3,15 @@ import pytz from django.urls import reverse from django.db import models -from django.db.models.signals import pre_save +from django.db.models.signals import pre_save, post_save from django.contrib.auth.models import User, Group from django_countries.fields import CountryField from .fields import PGPKeyField from main.utils import make_choice, set_created_field +from planet.models import Feed + class UserProfile(models.Model): notify = models.BooleanField( @@ -32,6 +34,8 @@ class UserProfile(models.Model): verbose_name="PGP key fingerprint", help_text="consists of 40 hex digits; use `gpg --fingerprint`") website = models.CharField(max_length=200, null=True, blank=True) + website_rss = models.CharField(max_length=200, null=True, blank=True, + help_text='RSS Feed of your website for planet.archlinux.org') yob = models.IntegerField("Year of birth", null=True, blank=True) country = CountryField(blank=True) location = models.CharField(max_length=50, null=True, blank=True) @@ -131,7 +135,66 @@ class PGPSignature(models.Model): return '%s → %s' % (self.signer, self.signee) -pre_save.connect(set_created_field, sender=UserProfile, +def create_feed_model(sender, **kwargs): + set_created_field(sender, **kwargs) + + obj = kwargs['instance'] + + if not obj.id: + return + + dbmodel = UserProfile.objects.get(id=obj.id) + + if not obj.website_rss and dbmodel.website_rss: + Feed.objects.filter(website_rss=dbmodel.website_rss).all().delete() + return + + if not obj.website_rss: + return + + if obj.website: + website = obj.website + else: + from urllib.parse import urlparse + parsed = urlparse(obj.website_rss) + website = obj.website_rss.replace(parsed.path, '') + + # Nothing changed + if obj.website_rss == dbmodel.website_rss: + return + + title = obj.alias + if obj.user.first_name and obj.user.last_name: + title = obj.user.first_name + ' ' + obj.user.last_name + + # Remove old feeds + Feed.objects.filter(website_rss=dbmodel.website_rss).all().delete() + Feed.objects.create(title=title, website=website, + website_rss=obj.website_rss) + + +def delete_feed_model(sender, **kwargs): + '''When a user is set to inactive remove his feed model''' + + obj = kwargs['instance'] + + if not obj.id: + return + + if obj.is_active: + return + + userprofile = UserProfile.objects.filter(user=obj).first() + if not userprofile: + return + + Feed.objects.filter(website_rss=userprofile.website_rss).delete() + + +pre_save.connect(create_feed_model, sender=UserProfile, dispatch_uid="devel.models") +post_save.connect(delete_feed_model, sender=User, + dispatch_uid='main.models') + # vim: set ts=4 sw=4 et: diff --git a/feeds.py b/feeds.py index 5b49ca82..605afa2f 100644 --- a/feeds.py +++ b/feeds.py @@ -14,6 +14,7 @@ from main.models import Arch, Repo, Package from news.models import News from packages.models import Update from releng.models import Release +from planet.models import FeedItem class BatchWritesWrapper(object): @@ -387,4 +388,49 @@ class ReleaseFeed(Feed): item_enclosure_mime_type = 'application/x-bittorrent' + +class PlanetFeed(Feed): + feed_type = FasterRssFeed + + title = 'Planet Arch Linux' + link = '/planet/' + description = 'Planet Arch Linux is a window into the world, work and lives of Arch Linux hackers and developers' + subtitle = description + + def __call__(self, request, *args, **kwargs): + wrapper = condition(last_modified_func=planet_last_modified) + return wrapper(super(PlanetFeed, self).__call__)(request, *args, **kwargs) + + __name__ = 'planet_feed' + + def items(self): + return FeedItem.objects.filter().order_by('-publishdate') + + def item_title(self, item): + return item.title + + def item_description(self, item): + return item.summary + + def item_pubdate(self, item): + return datetime.combine(item.publishdate, time()).replace(tzinfo=utc) + + def item_updateddate(self, item): + return item.publishdate + + item_guid_is_permalink = False + + def item_guid(self, item): + # http://diveintomark.org/archives/2004/05/28/howto-atom-id + date = item.publishdate + return 'tag:%s,%s:%s' % (Site.objects.get_current().domain, + date.strftime('%Y-%m-%d'), item.url) + + +def planet_last_modified(request, *args, **kwargs): + try: + return FeedItem.objects.latest().publishdate + except FeedItem.DoesNotExist: + pass + # vim: set ts=4 sw=4 et: diff --git a/planet/__init__.py b/planet/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/planet/admin.py b/planet/admin.py new file mode 100644 index 00000000..59f02a0e --- /dev/null +++ b/planet/admin.py @@ -0,0 +1,25 @@ +from django.contrib import admin + +from planet.models import Feed, FeedItem, Planet + + +class FeedItemAdmin(admin.ModelAdmin): + list_display = ('title', 'publishdate',) + list_filter = ('publishdate',) + search_fields = ('title',) + +class FeedAdmin(admin.ModelAdmin): + list_display = ('title', 'website',) + list_filter = ('title',) + search_fields = ('title',) + +class PlanetAdmin(admin.ModelAdmin): + list_display = ('name', 'website',) + list_filter = ('name',) + search_fields = ('name',) + +admin.site.register(Feed, FeedAdmin) +admin.site.register(FeedItem, FeedItemAdmin) +admin.site.register(Planet, PlanetAdmin) + +# vim: set ts=4 sw=4 et: diff --git a/planet/management/__init__.py b/planet/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/planet/management/commands/__init__.py b/planet/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/planet/management/commands/update_planet.py b/planet/management/commands/update_planet.py new file mode 100644 index 00000000..fd74c291 --- /dev/null +++ b/planet/management/commands/update_planet.py @@ -0,0 +1,145 @@ +""" +update_planet + +Imports all feeds for users who have filled in a valid website and website_rss, +the amount of items imported is limited to the defined FEED_LIMT + +Usage: ./manage.py update_planet +""" + + +import logging +import sys +import time + +from datetime import datetime +from pytz import utc + +import bleach +import feedparser + +from django.core.cache import cache +from django.core.management.base import BaseCommand +from django.template.defaultfilters import truncatewords_html +from django.conf import settings + +from planet.models import Feed, FeedItem, FEEDITEM_SUMMARY_LIMIT + + +logging.basicConfig( + level=logging.WARNING, + format=u'%(asctime)s -> %(levelname)s: %(message)s', + datefmt=u'%Y-%m-%d %H:%M:%S', + stream=sys.stderr) +logger = logging.getLogger() + + +class ItemOlderThenLatest(Exception): + pass + + +class Command(BaseCommand): + + def parse_feed(self, feed_instance): + latest = None + items = [] + url = feed_instance.website_rss + + logger.debug("Import new feed items for '%s'", url) + + try: + latest = FeedItem.objects.filter(feed=feed_instance).latest() + except FeedItem.DoesNotExist: + pass + + etag = cache.get(f'planet:etag:{url}') + feed = feedparser.parse(url, etag=etag) + + if feed['status'] == 304: + logger.info("The feed '%s' has not changed since we last checked it", url) + if 'etag' in feed: + cache.set(f'planet:etag:{url}', feed.etag, 86400) + return + + if feed['status'] != 200: + logger.info("error parsing feed: '%s', status: '%s'", url, feed['status']) + return + + if not feed.entries: + logger.info("error parsing feed: '%s', feed has no more new entries", url) + return + + for entry in feed.entries[:settings.RSS_FEED_LIMIT]: + try: + item = self.parse_entry(entry, feed_instance, latest) + except ItemOlderThenLatest: + break + + if not item: + continue + + items.append(item) + + logger.debug("inserting %d feed entries", len(items)) + res = FeedItem.objects.bulk_create(items) + + if res and 'etag' in feed: + # Cache etag for one day + logger.debug("cache etag for '%s'", url) + cache.set(f'planet:etag:{url}', feed.etag, 86400) + + def parse_entry(self, entry, feed_instance, latest): + url = feed_instance.website_rss + published_parsed = entry.get('published_parsed') + # Maybe it's an atom feed? + if not published_parsed: + published_parsed = entry.get('updated_parsed') + + if not published_parsed: + logger.error("feed: '%s' has no published or updated date", url) + return + + published = datetime.fromtimestamp(time.mktime(published_parsed)).replace(tzinfo=utc) + + if latest and latest.publishdate >= published: + logger.debug("feed: '%s' has no more new entries", url) + raise ItemOlderThenLatest() + + if not entry.link: + logger.error("feed '%s' entry has no link, skipping", url) + return + + logger.debug("import feed entry '%s'", entry.title) + + item = FeedItem(title=entry.title, publishdate=published, url=entry.link) + item.feed = feed_instance + + if entry.get('description'): + summary = bleach.clean(entry.description, strip=True) + if len(summary) > FEEDITEM_SUMMARY_LIMIT: + summary = truncatewords_html(summary, 100) + item.summary = summary + + if entry.get('author'): + item.author = entry.get('author') + + return item + + def handle(self, *args, **options): + v = int(options.get('verbosity', 0)) + if v == 0: + logger.level = logging.ERROR + elif v == 1: + logger.level = logging.INFO + elif v >= 2: + logger.level = logging.DEBUG + + for feed in Feed.objects.all(): + self.parse_feed(feed) + + # Only keep RSS_FEED_LIMIT amount of feed items. + feeds_to_keep = FeedItem.objects.filter(feed=feed).order_by('-publishdate')[:settings.RSS_FEED_LIMIT] + items = FeedItem.objects.filter(feed=feed).exclude(pk__in=feeds_to_keep).delete() + logger.debug("removed %d feed entries", items[0]) + +# vim: set ts=4 sw=4 et: diff --git a/planet/migrations/0001_initial.py b/planet/migrations/0001_initial.py new file mode 100644 index 00000000..948b2532 --- /dev/null +++ b/planet/migrations/0001_initial.py @@ -0,0 +1,62 @@ +# Generated by Django 2.2.8 on 2019-12-18 19:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Feed', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('website', models.CharField(blank=True, max_length=200, null=True)), + ('website_rss', models.CharField(blank=True, max_length=200, null=True)), + ], + options={ + 'verbose_name_plural': 'feeds', + 'db_table': 'feeds', + 'ordering': ('-title',), + 'get_latest_by': 'title', + }, + ), + migrations.CreateModel( + name='Planet', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('website', models.CharField(blank=True, max_length=200, null=True)), + ], + options={ + 'verbose_name_plural': 'Worldwide Planets', + 'db_table': 'planets', + 'ordering': ('-name',), + 'get_latest_by': 'name', + }, + ), + migrations.CreateModel( + name='FeedItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255)), + ('summary', models.CharField(max_length=2048)), + ('author', models.CharField(max_length=255)), + ('publishdate', models.DateTimeField(db_index=True, verbose_name='publish date')), + ('url', models.CharField(max_length=255, verbose_name='URL')), + ('feed', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='feed', to='planet.Feed')), + ], + options={ + 'verbose_name_plural': 'Feed Items', + 'db_table': 'feeditems', + 'ordering': ('-publishdate',), + 'get_latest_by': 'publishdate', + }, + ), + ] diff --git a/planet/migrations/__init__.py b/planet/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/planet/models.py b/planet/models.py new file mode 100644 index 00000000..2aaeb7f5 --- /dev/null +++ b/planet/models.py @@ -0,0 +1,60 @@ +from django.db import models + + +# FeedItem summary field length +FEEDITEM_SUMMARY_LIMIT = 2048 + + +class Feed(models.Model): + title = models.CharField(max_length=255) + website = models.CharField(max_length=200, null=True, blank=True) + website_rss = models.CharField(max_length=200, null=True, blank=True) + + def __str__(self): + return self.title + + class Meta: + db_table = 'feeds' + verbose_name_plural = 'feeds' + get_latest_by = 'title' + ordering = ('-title',) + +class FeedItem(models.Model): + title = models.CharField(max_length=255) + summary = models.CharField(max_length=FEEDITEM_SUMMARY_LIMIT) + feed = models.ForeignKey(Feed, related_name='items', + on_delete=models.CASCADE, null=True) + author = models.CharField(max_length=255) + publishdate = models.DateTimeField("publish date", db_index=True) + url = models.CharField('URL', max_length=255) + + def get_absolute_url(self): + return self.url + + def __str__(self): + return self.title + + class Meta: + db_table = 'feeditems' + verbose_name_plural = 'Feed Items' + get_latest_by = 'publishdate' + ordering = ('-publishdate',) + +class Planet(models.Model): + ''' + The planet model contains related Arch Linux planet instances. + ''' + + name = models.CharField(max_length=255) + website = models.CharField(max_length=200, null=True, blank=True) + + def __str__(self): + return self.name + + class Meta: + db_table = 'planets' + verbose_name_plural = 'Worldwide Planets' + get_latest_by = 'name' + ordering = ('-name',) + +# vim: set ts=4 sw=4 et: diff --git a/planet/tests/__init__.py b/planet/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/planet/tests/test_views.py b/planet/tests/test_views.py new file mode 100644 index 00000000..b09c9e6b --- /dev/null +++ b/planet/tests/test_views.py @@ -0,0 +1,12 @@ +from django.test import TestCase + + +class PlanetTest(TestCase): + + def test_feed(self): + response = self.client.get('/feeds/planet/') + self.assertEqual(response.status_code, 200) + + def test_planet(self): + response = self.client.get('/planet/') + self.assertEqual(response.status_code, 200) diff --git a/planet/views.py b/planet/views.py new file mode 100644 index 00000000..d16255e0 --- /dev/null +++ b/planet/views.py @@ -0,0 +1,14 @@ +from django.shortcuts import render +from django.views.decorators.cache import cache_control + +from planet.models import Feed, FeedItem, Planet + + +@cache_control(max_age=307) +def index(request): + context = { + 'official_feeds': Feed.objects.all(), + 'planets': Planet.objects.all(), + 'feed_items': FeedItem.objects.order_by('-publishdate')[:25], + } + return render(request, 'planet/index.html', context) diff --git a/public/management/__init__.py b/public/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/requirements.txt b/requirements.txt index ce6accc5..66d1ede4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ django-jinja==2.4.1 sqlparse==0.3.0 django-csp==3.5 ptpython==2.0.4 +feedparser==5.2.1 +bleach==3.1.0 diff --git a/settings.py b/settings.py index 3201f218..5a23116a 100644 --- a/settings.py +++ b/settings.py @@ -121,6 +121,7 @@ INSTALLED_APPS = ( 'mirrors', 'news', 'packages', + 'planet', 'todolists', 'devel', 'public', @@ -188,6 +189,9 @@ DATABASES = { }, } +# Planet limit of items per feed to keep the feed size in check. +RSS_FEED_LIMIT = 25 + # Import local settings try: from local_settings import * diff --git a/sitestatic/archweb.css b/sitestatic/archweb.css index f95e3843..77379d96 100644 --- a/sitestatic/archweb.css +++ b/sitestatic/archweb.css @@ -494,6 +494,56 @@ h3 span.arrow { line-height: 0px; } +/* planet: posts */ +#planet { + margin-top: 1.5em; +} + + #planet h3 { + float: left; + padding-bottom: .5em + } + + #planet div { + margin-bottom: 1em; + } + + #planet div p { + margin-bottom: 0.5em; + } + + #planet .more { + font-weight: normal; + } + + #planet .rss-icon { + float: right; + margin-top: 1em; + } + + #planet h4 { + clear: both; + font-size: 1em; + margin-top: 1.5em; + border-bottom: 1px dotted #bbb; + } + + #planet .timestamp { + float: right; + font-size: 0.85em; + margin: -1.8em 0.5em 0 0; + } + +#planet .article-author { + font-style: italic; + text-align: right; +} + +#nav-sidebar .planet-list { + list-style: square; + padding-left: 1em; +} + /* home: pkgsearch box */ #pkgsearch { padding: 1em 0.75em; diff --git a/templates/planet/index.html b/templates/planet/index.html new file mode 100644 index 00000000..2203fc13 --- /dev/null +++ b/templates/planet/index.html @@ -0,0 +1,59 @@ +{% extends "base.html" %} +{% load cache %} +{% load static from staticfiles %} + +{% block head %} + +{% endblock %} + +{% block content_left %} +{% cache 62 planet-page-left %} +
+

Arch Planet

+ +

Planet Arch Linux is a window into the world, work and + lives of Arch Linux developers, trusted users and support staff.

+
+ +
+ RSS Feed + + {% for entry in feed_items %} +

+ {{ entry.title }} +

+

{{ entry.publishdate|date }}

+
+ {{ entry.summary |safe }} +
+

{% if entry.author %}{{ entry.author }}@{% endif %}{{ entry.feed.title }}

+ {% endfor %} +
+{% endcache %} +{% endblock %} + +{% block content_right %} +{% cache 115 planet-page-right %} + +{% endcache %} +{% endblock %} diff --git a/urls.py b/urls.py index 44a1b0e4..c4e3992c 100644 --- a/urls.py +++ b/urls.py @@ -8,7 +8,7 @@ from django.conf import settings from django.views.decorators.cache import cache_page from django.views.generic import TemplateView -from feeds import PackageFeed, NewsFeed, ReleaseFeed, PackageUpdatesFeed +from feeds import PackageFeed, NewsFeed, ReleaseFeed, PackageUpdatesFeed, PlanetFeed import sitemaps import devel.urls @@ -17,6 +17,7 @@ import mirrors.urls_mirrorlist import news.urls import packages.urls import packages.urls_groups +import planet.views import public.views import releng.urls import todolists.urls @@ -48,6 +49,7 @@ urlpatterns.extend([ url(r'^master-keys/$', public.views.keys, name='page-keys'), url(r'^master-keys/json/$', public.views.keys_json, name='pgp-keys-json'), url(r'^people/(?P[-\w]+)/$', public.views.people, name='people'), + url(r'^planet/$', planet.views.index, name='planet'), ]) # Feeds patterns, used below @@ -63,6 +65,7 @@ feeds_patterns = [ url(r'^packages/all/(?P[A-z0-9\-]+)/$', cache_page(313)(PackageFeed())), url(r'^packages/(?P[A-z0-9]+)/(?P[A-z0-9\-]+)/$', cache_page(313)(PackageFeed())), url(r'^releases/$', cache_page(317)(ReleaseFeed())), + url(r'^planet/$', cache_page(317)(PlanetFeed()), name='planet-feed'), ] # Includes and other remaining stuff -- cgit v1.2.3-55-g3dc8