better handling of exceptions, again

tl;dr, only working for github and gitlab upstreams, checks if the latest commit is more than 1 year old.
It also removes deprecated-software if required.

.github/workflows/deprecated.yml

@@ -0,0 +1,30 @@
+name: Find deprecated softwares
+
+on:
+  schedule:
+    - cron: '0 20 * * 1'
+
+jobs:
+  black:
+    name: Find deprecated softwares
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install toml python lib
+        run: |
+          pip3 install toml tomlkit gitpython
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Flag deprecated apps in the catalog"
+          commit-message: ":coffin: Flag deprecated apps in the catalog"
+          body: |
+            This was done with tools/find_deprecated.py
+          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
+          branch: actions/deprecated

README.md

@@ -1,6 +1,6 @@
 # YunoHost application catalog
 
-<img src="https://avatars.githubusercontent.com/u/1519495?s=200&v=4" width=80><img src="https://yunohost.org/user/images/yunohost_package.png" width=80>
+<img alt="YunoHost logo" src="https://avatars.githubusercontent.com/u/1519495?s=200&v=4" width=80><img alt="Package logo" src="https://yunohost.org/user/images/yunohost_package.png" width=80>
 
 This repository contains the default YunoHost app catalog, as well as related
 tools that can be run manually or automatically.
@@ -16,7 +16,7 @@ them such as their category or maintenance state. This file is regularly read by
 
 - You can browse [the contributor documentation](https://yunohost.org/contributordoc)
 - If you are not familiar with Git/GitHub, you can have a look at our [homemade guide](https://yunohost.org/packaging_apps_git)
-- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms) ... we can even schedule an audio meeting to help you get started!
+- Don't hesitate to reach for help on the dedicated [application packaging chatroom](https://yunohost.org/chat_rooms)... we can even schedule an audio meeting to help you get started!
 
 ## How to add your app to the application catalog
 
@@ -46,9 +46,9 @@ App example addition:
 
 ```toml
 [your_app]
-antifeatures = [ "deprecated-software" ]   # Remove if no relevant antifeature applies
+antifeatures = [ "deprecated-software" ]   # Replace with the appropriate category id found in antifeatures.toml, remove if no relevant antifeature applies
 potential_alternative_to = [ "YouTube" ]   # Indicate if your app can be thought of as an alternative to popular proprietary services (or remove if none applies)
-category = "foobar"                        # Replace with the appropriate category id found in categories.toml
+category = "foobar"                        # Replace with the appropriate category id found in categories.toml, don't invent a category
 state = "working"
 url = "https://github.com/YunoHost-Apps/your_app_ynh"
 ```

apps.toml

@@ -37,6 +37,7 @@ level = 6
 state = "working"
 subtags = [ "microblogging", "pictures" ]
 url = "https://github.com/YunoHost-Apps/acropolis_ynh"
+antifeatures = ["deprecated-software"]
 
 [actual]
 category = "productivity_and_management"
@@ -59,6 +60,7 @@ level = 8
 state = "working"
 subtags = [ "db" ]
 url = "https://github.com/YunoHost-Apps/adminer_ynh"
+antifeatures = ["deprecated-software"]
 
 [aeneria]
 category = "iot"
@@ -73,6 +75,7 @@ potential_alternative_to = [ "Google Agenda", "Microsoft Outlook" ]
 state = "working"
 subtags = [ "calendar" ]
 url = "https://github.com/YunoHost-Apps/agendav_ynh"
+antifeatures = ["deprecated-software"]
 
 [agora]
 category = "wat"
@@ -185,6 +188,7 @@ category = "iot"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/beehive_ynh"
+antifeatures = ["deprecated-software"]
 
 [biboumi]
 category = "communication"
@@ -199,6 +203,7 @@ level = 7
 state = "working"
 subtags = [ "books" ]
 url = "https://github.com/YunoHost-Apps/bicbucstriim_ynh"
+antifeatures = ["deprecated-software"]
 
 [blogotext]
 category = "publishing"
@@ -207,6 +212,7 @@ potential_alternative_to = [ "Blogger", "Coldfusion", "Wix" ]
 state = "working"
 subtags = [ "blog" ]
 url = "https://github.com/YunoHost-Apps/blogotext_ynh"
+antifeatures = ["deprecated-software"]
 
 [bludit]
 category = "publishing"
@@ -331,6 +337,7 @@ category = "small_utilities"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/cheky_ynh"
+antifeatures = ["deprecated-software"]
 
 [chitchatter]
 category = "communication"
@@ -434,6 +441,7 @@ potential_alternative_to = [ "Disqus" ]
 state = "working"
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/commento_ynh"
+antifeatures = ["deprecated-software"]
 
 [compteur_du_gase]
 category = "office"
@@ -487,7 +495,7 @@ subtags = [ "db" ]
 url = "https://github.com/YunoHost-Apps/couchdb_ynh"
 
 [couchpotato]
-antifeatures = [ "package-not-maintained" ]
+antifeatures = [ "package-not-maintained", "deprecated-software" ]
 category = "multimedia"
 level = 0
 potential_alternative_to = [ "Netflix" ]
@@ -657,6 +665,7 @@ level = 6
 state = "working"
 subtags = [ "pastebin" ]
 url = "https://github.com/YunoHost-Apps/distbin_ynh"
+antifeatures = ["deprecated-software"]
 
 [django-fmd]
 category = "iot"
@@ -836,6 +845,7 @@ category = "small_utilities"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/encryptor-decryptor_ynh"
+antifeatures = ["deprecated-software"]
 
 [endi]
 category = "productivity_and_management"
@@ -865,6 +875,7 @@ potential_alternative_to = [ "Google Sheets", "G Suite", "Microsoft Excel", "Off
 state = "working"
 subtags = [ "spreadsheet" ]
 url = "https://github.com/YunoHost-Apps/ethercalc_ynh"
+antifeatures = ["deprecated-software"]
 
 [etherpad]
 category = "office"
@@ -902,6 +913,7 @@ level = 8
 state = "working"
 subtags = [ "monitoring" ]
 url = "https://github.com/YunoHost-Apps/facette_ynh"
+antifeatures = ["deprecated-software"]
 
 [facilmap]
 category = "productivity_and_management"
@@ -943,6 +955,7 @@ potential_alternative_to = [ "WeTransfer" ]
 state = "working"
 subtags = [ "files" ]
 url = "https://github.com/YunoHost-Apps/filepizza_ynh"
+antifeatures = ["deprecated-software"]
 
 [firefish]
 category = "social_media"
@@ -1002,6 +1015,7 @@ potential_alternative_to = [ "Invision Community", "Vanilla", "vBulletin", "XenF
 state = "working"
 subtags = [ "forum" ]
 url = "https://github.com/YunoHost-Apps/fluxbb_ynh"
+antifeatures = ["deprecated-software"]
 
 [focalboard]
 category = "productivity_and_management"
@@ -1123,6 +1137,7 @@ level = 8
 state = "working"
 subtags = [ "website", "blog" ]
 url = "https://github.com/YunoHost-Apps/getsimple_ynh"
+antifeatures = ["deprecated-software"]
 
 [ghost]
 antifeatures = [ "arbitrary-limitations" ]
@@ -1354,6 +1369,7 @@ category = "system_tools"
 level = 7
 state = "working"
 url = "https://github.com/Yunohost-Apps/httpsh_ynh"
+antifeatures = ["deprecated-software"]
 
 [hubzilla]
 category = "social_media"
@@ -1442,7 +1458,6 @@ subtags = [ "videos" ]
 url = "https://github.com/YunoHost-Apps/invidious_ynh"
 
 [invoiceninja]
-antifeatures = [ "deprecated-software" ]
 category = "productivity_and_management"
 level = 7
 state = "working"
@@ -1475,7 +1490,7 @@ state = "working"
 url = "https://github.com/YunoHost-Apps/jackett_ynh"
 
 [jappix]
-antifeatures = [ "package-not-maintained" ]
+antifeatures = [ "package-not-maintained", "deprecated-software" ]
 category = "communication"
 level = 7
 state = "working"
@@ -1646,7 +1661,7 @@ subtags = [ "text" ]
 url = "https://github.com/YunoHost-Apps/languagetool_ynh"
 
 [laverna]
-antifeatures = [ "package-not-maintained" ]
+antifeatures = [ "package-not-maintained", "deprecated-software" ]
 category = "office"
 level = 7
 state = "working"
@@ -1727,6 +1742,7 @@ level = 7
 state = "working"
 subtags = [ "wiki" ]
 url = "https://github.com/YunoHost-Apps/libreto_ynh"
+antifeatures = ["deprecated-software"]
 
 [libretranslate]
 category = "small_utilities"
@@ -1774,6 +1790,7 @@ level = 6
 state = "working"
 subtags = [ "monitoring" ]
 url = "https://github.com/YunoHost-Apps/linuxdash_ynh"
+antifeatures = ["deprecated-software"]
 
 [lionwiki-t2t]
 category = "publishing"
@@ -1812,6 +1829,7 @@ potential_alternative_to = [ "Google Docs", "Microsoft Excel" ]
 state = "working"
 subtags = [ "spreadsheet" ]
 url = "https://github.com/YunoHost-Apps/luckysheet_ynh"
+antifeatures = ["deprecated-software"]
 
 [lufi]
 category = "synchronization"
@@ -1986,6 +2004,7 @@ level = 8
 state = "working"
 subtags = [ "chat" ]
 url = "https://github.com/YunoHost-Apps/minchat_ynh"
+antifeatures = ["deprecated-software"]
 
 [mindmaps]
 category = "office"
@@ -1993,6 +2012,7 @@ level = 8
 state = "working"
 subtags = [ "mindmap" ]
 url = "https://github.com/YunoHost-Apps/mindmaps_ynh"
+antifeatures = ["deprecated-software"]
 
 [minetest]
 category = "games"
@@ -2006,6 +2026,7 @@ category = "publishing"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/mineweb_ynh"
+antifeatures = ["deprecated-software"]
 
 [minidlna]
 category = "multimedia"
@@ -2118,6 +2139,7 @@ level = 8
 state = "working"
 subtags = [ "meeting" ]
 url = "https://github.com/YunoHost-Apps/mumble-web_ynh"
+antifeatures = ["deprecated-software"]
 
 [mumbleserver]
 category = "communication"
@@ -2353,6 +2375,7 @@ category = "wat"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/osjs_ynh"
+antifeatures = ["deprecated-software"]
 
 [osticket]
 category = "productivity_and_management"
@@ -2590,6 +2613,7 @@ level = 8
 state = "working"
 subtags = [ "website", "blog" ]
 url = "https://github.com/YunoHost-Apps/pico_ynh"
+antifeatures = ["deprecated-software"]
 
 [pihole]
 category = "system_tools"
@@ -3035,6 +3059,7 @@ category = "system_tools"
 level = 6
 state = "working"
 url = "https://github.com/YunoHost-Apps/shellinabox_ynh"
+antifeatures = ["deprecated-software"]
 
 [shiori]
 category = "small_utilities"
@@ -3057,6 +3082,7 @@ potential_alternative_to = [ "bitly" ]
 state = "working"
 subtags = [ "url_shortener" ]
 url = "https://github.com/YunoHost-Apps/shuri_ynh"
+antifeatures = ["deprecated-software"]
 
 [signaturepdf]
 category = "small_utilities"
@@ -3069,12 +3095,14 @@ category = "small_utilities"
 level = 6
 state = "working"
 url = "https://github.com/YunoHost-Apps/simple-file-manager_ynh"
+antifeatures = ["deprecated-software"]
 
 [simple-hash-generator]
 category = "small_utilities"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/simple-hash-generator_ynh"
+antifeatures = ["deprecated-software"]
 
 [simple-torrent]
 antifeatures = [ "deprecated-software" ]
@@ -3092,7 +3120,7 @@ subtags = [ "chat" ]
 url = "https://github.com/YunoHost-Apps/simplex_ynh"
 
 [simplytranslate]
-antifeatures = [ "non-free-network" ]
+antifeatures = [ "non-free-network", "deprecated-software" ]
 category = "small_utilities"
 level = 7
 state = "working"
@@ -3104,6 +3132,7 @@ level = 8
 state = "working"
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/sitemagiccms_ynh"
+antifeatures = ["deprecated-software"]
 
 [slingcode]
 category = "dev"
@@ -3111,6 +3140,7 @@ level = 8
 state = "working"
 subtags = [ "programming" ]
 url = "https://github.com/YunoHost-Apps/slingcode_ynh"
+antifeatures = ["deprecated-software"]
 
 [snappymail]
 category = "communication"
@@ -3199,6 +3229,7 @@ category = "system_tools"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/ssh_chroot_dir_ynh"
+antifeatures = ["deprecated-software"]
 
 [statpingng]
 category = "system_tools"
@@ -3228,6 +3259,7 @@ potential_alternative_to = [ "Google Slides", "Office 365", "Prezi", "SoZi" ]
 state = "working"
 subtags = [ "impress" ]
 url = "https://github.com/YunoHost-Apps/strut_ynh"
+antifeatures = ["deprecated-software"]
 
 [superset]
 category = "wat"
@@ -3284,7 +3316,7 @@ state = "working"
 url = "https://github.com/YunoHost-Apps/tandoor_ynh"
 
 [taskboard]
-antifeatures = [ "package-not-maintained" ]
+antifeatures = [ "package-not-maintained", "deprecated-software" ]
 category = "productivity_and_management"
 level = 7
 potential_alternative_to = [ "Trello" ]
@@ -3470,6 +3502,7 @@ level = 8
 state = "working"
 subtags = [ "task" ]
 url = "https://github.com/YunoHost-Apps/tyto_ynh"
+antifeatures = ["deprecated-software"]
 
 [ulogger]
 category = "small_utilities"
@@ -3755,6 +3788,7 @@ level = 6
 potential_alternative_to = [ "Mullvad" ]
 state = "working"
 url = "https://github.com/YunoHost-Apps/zeronet_ynh"
+antifeatures = ["deprecated-software"]
 
 [zerotier]
 antifeatures = [ "not-totally-free-upstream", "non-free-network" ]

tools/autoupdate_app_sources/rest_api.py

@@ -40,6 +40,10 @@ class GithubAPI:
         """Get a list of releases for project."""
         return self.internal_api(f"repos/{self.upstream_repo}/releases")
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"repos/{self.upstream_repo}")["archived"]
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         """Get a URL for a ref."""
         if ref_type == RefType.tags:
@@ -53,6 +57,7 @@ class GithubAPI:
 class GitlabAPI:
     def __init__(self, upstream: str):
         # Find gitlab api root...
+        upstream = upstream.rstrip("/")
         self.forge_root = self.get_forge_root(upstream).rstrip("/")
         self.project_path = upstream.replace(self.forge_root, "").lstrip("/")
         self.project_id = self.find_project_id(self.project_path)
@@ -128,6 +133,10 @@ class GitlabAPI:
 
         return retval
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"projects/{self.project_id}").get("archived", False)
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         name = self.project_path.split("/")[-1]
         clean_ref = ref.replace("/", "-")
@@ -166,6 +175,10 @@ class GiteaForgejoAPI:
         """Get a list of releases for project."""
         return self.internal_api(f"repos/{self.project_path}/releases")
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"repos/{self.project_path}")["archived"]
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         """Get a URL for a ref."""
         return f"{self.forge_root}/{self.project_path}/archive/{ref}.tar.gz"

tools/find_deprecated.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+import traceback
+import argparse
+import tomlkit
+import multiprocessing
+import datetime
+import json
+import sys
+from functools import cache
+import logging
+from pathlib import Path
+from typing import Optional
+
+import toml
+import tqdm
+import github
+
+# add apps/tools to sys.path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from appslib.utils import REPO_APPS_ROOT, get_catalog  # noqa: E402 pylint: disable=import-error,wrong-import-position
+from app_caches import app_cache_folder  # noqa: E402 pylint: disable=import-error,wrong-import-position
+from autoupdate_app_sources.rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType  # noqa: E402,E501 pylint: disable=import-error,wrong-import-position
+
+
+@cache
+def get_github() -> tuple[Optional[tuple[str, str]], Optional[github.Github], Optional[github.InputGitAuthor]]:
+    try:
+        github_login = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip()
+        github_token = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip()
+        github_email = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip()
+
+        auth = (github_login, github_token)
+        github_api = github.Github(github_token)
+        author = github.InputGitAuthor(github_login, github_email)
+        return auth, github_api, author
+    except Exception as e:
+        logging.warning(f"Could not get github: {e}")
+        return None, None, None
+
+
+def upstream_last_update_ago(app: str) -> tuple[str, int | None]:
+    manifest_toml = app_cache_folder(app) / "manifest.toml"
+    manifest_json = app_cache_folder(app) / "manifest.json"
+
+    if manifest_toml.exists():
+        manifest = toml.load(manifest_toml.open("r", encoding="utf-8"))
+        upstream = manifest.get("upstream", {}).get("code")
+
+    elif manifest_json.exists():
+        manifest = json.load(manifest_json.open("r", encoding="utf-8"))
+        upstream = manifest.get("upstream", {}).get("code")
+    else:
+        raise RuntimeError(f"App {app} doesn't have a manifest!")
+
+    if upstream is None:
+        raise RuntimeError(f"App {app} doesn't have an upstream code link!")
+
+    api = None
+    try:
+        if upstream.startswith("https://github.com/"):
+            try:
+                api = GithubAPI(upstream, auth=get_github()[0])
+            except AssertionError as e:
+                logging.error(f"Exception while handling {app}: {e}")
+                return app, None
+
+        if upstream.startswith("https://gitlab.") or upstream.startswith("https://framagit.org"):
+            api = GitlabAPI(upstream)
+
+        if upstream.startswith("https://codeberg.org"):
+            api = GiteaForgejoAPI(upstream)
+
+        if not api:
+            autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate")
+            if autoupdate:
+                strat = autoupdate["strategy"]
+                if "gitea" in strat or "forgejo" in strat:
+                    api = GiteaForgejoAPI(upstream)
+
+        if api:
+            if api.archived():
+                # A stupid value that we know to be higher than the trigger value
+                return app, 1000
+
+            last_commit = api.commits()[0]
+            date = last_commit["commit"]["author"]["date"]
+            date = datetime.datetime.fromisoformat(date)
+            ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None)
+            return app, ago.days
+    except Exception:
+        logging.error(f"Exception while handling {app}", traceback.format_exc())
+        raise
+
+    raise RuntimeError(f"App {app} not handled (not github, gitlab or gitea with autoupdate). Upstream is {upstream}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("apps", nargs="*", type=str,
+                        help="If not passed, the script will run on the catalog. Github keys required.")
+    parser.add_argument("-j", "--processes", type=int, default=multiprocessing.cpu_count())
+    args = parser.parse_args()
+
+    apps_dict = get_catalog()
+    if args.apps:
+        apps_dict = {app: info for app, info in apps_dict.items() if app in args.apps}
+
+    deprecated: list[str] = []
+    not_deprecated: list[str] = []
+    # for app, info in apps_dict.items():
+    with multiprocessing.Pool(processes=args.processes) as pool:
+        tasks = pool.imap_unordered(upstream_last_update_ago, apps_dict.keys())
+
+        for _ in tqdm.tqdm(range(len(apps_dict)), ascii=" ·#"):
+            try:
+                app, result = next(tasks)
+            except Exception as e:
+                print(f"Exception found: {e}")
+                continue
+
+            if result is None:
+                continue
+
+            if result > 365:
+                deprecated.append(app)
+            else:
+                not_deprecated.append(app)
+
+    catalog = tomlkit.load(open("apps.toml"))
+    for app, info in catalog.items():
+        antifeatures = info.get("antifeatures", [])
+        if app in deprecated:
+            if "deprecated-software" not in antifeatures:
+                antifeatures.append("deprecated-software")
+        elif app in not_deprecated:
+            if "deprecated-software" in antifeatures:
+                antifeatures.remove("deprecated-software")
+        else:
+            continue
+        # unique the keys
+        if antifeatures:
+            info["antifeatures"] = antifeatures
+        else:
+            if "antifeatures" in info.keys():
+                info.pop("antifeatures")
+    tomlkit.dump(catalog, open("apps.toml", "w"))
+
+
+if __name__ == "__main__":
+    main()

tools/readme_generator/make_readme.py

@@ -6,7 +6,7 @@ import os
 from pathlib import Path
 from copy import deepcopy
 
-from typing import Dict, Optional, List
+from typing import Dict, Optional, List, Tuple
 
 import toml
 from jinja2 import Environment, FileSystemLoader
@@ -51,7 +51,22 @@ def generate_READMEs(app_path: Path):
 
     env = Environment(loader=FileSystemLoader(Path(__file__).parent / "templates"))
 
-    for lang, lang_suffix in [("en", ""), ("fr", "_fr")]:
+    # parse available README template and generate a list in the form of:
+    # > [("en", ""), ("fr", "_fr"), ...]
+    available_langs: List[Tuple[str, str]] = [("en", "")]
+    for README_template in (Path(__file__).parent / "templates").iterdir():
+        # we only want README_{lang}.md.j2 files
+        if README_template.name == "README.md.j2":
+            continue
+
+        if not README_template.name.endswith(".j2") or not README_template.name.startswith("README_"):
+            continue
+
+        language_code = README_template.name.split("_")[1].split(".")[0]
+
+        available_langs.append((language_code, "_" + language_code))
+
+    for lang, lang_suffix in available_langs:
         template = env.get_template(f"README{lang_suffix}.md.j2")
 
         if (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").exists():

wishlist.toml

@@ -1711,6 +1711,12 @@ upstream = ""
 website = "https://umap.openstreetmap.fr/"
 draft = "https://github.com/YunoHost-Apps/umap_ynh"
 
+[univer]
+name = "Univer"
+description = "Replacement of Luckysheet (deprecated, in yunohost packages)"
+upstream = "https://github.com/dream-num/univer"
+website = "https://univer.ai/"
+
 [upmpdcli]
 name = "upmpdcli"
 description = ""