better handling of exceptions, again

tl;dr, only working for github and gitlab upstreams, checks if the latest commit is more than 1 year old.
It also removes deprecated-software if required.

.github/workflows/deprecated.yml

@@ -0,0 +1,30 @@
+name: Find deprecated softwares
+
+on:
+  schedule:
+    - cron: '0 20 * * 1'
+
+jobs:
+  black:
+    name: Find deprecated softwares
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Install toml python lib
+        run: |
+          pip3 install toml tomlkit gitpython
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          title: "Flag deprecated apps in the catalog"
+          commit-message: ":coffin: Flag deprecated apps in the catalog"
+          body: |
+            This was done with tools/find_deprecated.py
+          base: ${{ github.head_ref }} # Creates pull request onto pull request or commit branch
+          branch: actions/deprecated

apps.toml

@@ -2713,7 +2713,6 @@ potential_alternative_to = [ "Wix" ]
 state = "working"
 subtags = [ "website" ]
 url = "https://github.com/YunoHost-Apps/prettynoemiecms_ynh"
-antifeatures = ["deprecated-software"]
 
 [privatebin]
 category = "small_utilities"
@@ -2995,7 +2994,6 @@ category = "small_utilities"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/scrumblr_ynh"
-antifeatures = ["deprecated-software"]
 
 [scrutiny]
 category = "system_tools"
@@ -3622,7 +3620,6 @@ category = "wat"
 level = 8
 state = "working"
 url = "https://github.com/YunoHost-Apps/wemawema_ynh"
-antifeatures = ["deprecated-software"]
 
 [wetty]
 category = "system_tools"

tools/autoupdate_app_sources/rest_api.py

@@ -40,6 +40,10 @@ class GithubAPI:
         """Get a list of releases for project."""
         return self.internal_api(f"repos/{self.upstream_repo}/releases")
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"repos/{self.upstream_repo}")["archived"]
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         """Get a URL for a ref."""
         if ref_type == RefType.tags:
@@ -53,6 +57,7 @@ class GithubAPI:
 class GitlabAPI:
     def __init__(self, upstream: str):
         # Find gitlab api root...
+        upstream = upstream.rstrip("/")
         self.forge_root = self.get_forge_root(upstream).rstrip("/")
         self.project_path = upstream.replace(self.forge_root, "").lstrip("/")
         self.project_id = self.find_project_id(self.project_path)
@@ -128,6 +133,10 @@ class GitlabAPI:
 
         return retval
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"projects/{self.project_id}").get("archived", False)
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         name = self.project_path.split("/")[-1]
         clean_ref = ref.replace("/", "-")
@@ -166,6 +175,10 @@ class GiteaForgejoAPI:
         """Get a list of releases for project."""
         return self.internal_api(f"repos/{self.project_path}/releases")
 
+    def archived(self) -> bool:
+        """Return the archival status for the repository"""
+        return self.internal_api(f"repos/{self.project_path}")["archived"]
+
     def url_for_ref(self, ref: str, ref_type: RefType) -> str:
         """Get a URL for a ref."""
         return f"{self.forge_root}/{self.project_path}/archive/{ref}.tar.gz"

tools/find_deprecated.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+
+import traceback
+import argparse
+import tomlkit
+import multiprocessing
+import datetime
+import json
+import sys
+from functools import cache
+import logging
+from pathlib import Path
+from typing import Optional
+
+import toml
+import tqdm
+import github
+
+# add apps/tools to sys.path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+from appslib.utils import REPO_APPS_ROOT, get_catalog  # noqa: E402 pylint: disable=import-error,wrong-import-position
+from app_caches import app_cache_folder  # noqa: E402 pylint: disable=import-error,wrong-import-position
+from autoupdate_app_sources.rest_api import GithubAPI, GitlabAPI, GiteaForgejoAPI, RefType  # noqa: E402,E501 pylint: disable=import-error,wrong-import-position
+
+
+@cache
+def get_github() -> tuple[Optional[tuple[str, str]], Optional[github.Github], Optional[github.InputGitAuthor]]:
+    try:
+        github_login = (REPO_APPS_ROOT / ".github_login").open("r", encoding="utf-8").read().strip()
+        github_token = (REPO_APPS_ROOT / ".github_token").open("r", encoding="utf-8").read().strip()
+        github_email = (REPO_APPS_ROOT / ".github_email").open("r", encoding="utf-8").read().strip()
+
+        auth = (github_login, github_token)
+        github_api = github.Github(github_token)
+        author = github.InputGitAuthor(github_login, github_email)
+        return auth, github_api, author
+    except Exception as e:
+        logging.warning(f"Could not get github: {e}")
+        return None, None, None
+
+
+def upstream_last_update_ago(app: str) -> tuple[str, int | None]:
+    manifest_toml = app_cache_folder(app) / "manifest.toml"
+    manifest_json = app_cache_folder(app) / "manifest.json"
+
+    if manifest_toml.exists():
+        manifest = toml.load(manifest_toml.open("r", encoding="utf-8"))
+        upstream = manifest.get("upstream", {}).get("code")
+
+    elif manifest_json.exists():
+        manifest = json.load(manifest_json.open("r", encoding="utf-8"))
+        upstream = manifest.get("upstream", {}).get("code")
+    else:
+        raise RuntimeError(f"App {app} doesn't have a manifest!")
+
+    if upstream is None:
+        raise RuntimeError(f"App {app} doesn't have an upstream code link!")
+
+    api = None
+    try:
+        if upstream.startswith("https://github.com/"):
+            try:
+                api = GithubAPI(upstream, auth=get_github()[0])
+            except AssertionError as e:
+                logging.error(f"Exception while handling {app}: {e}")
+                return app, None
+
+        if upstream.startswith("https://gitlab.") or upstream.startswith("https://framagit.org"):
+            api = GitlabAPI(upstream)
+
+        if upstream.startswith("https://codeberg.org"):
+            api = GiteaForgejoAPI(upstream)
+
+        if not api:
+            autoupdate = manifest.get("resources", {}).get("sources", {}).get("main", {}).get("autoupdate")
+            if autoupdate:
+                strat = autoupdate["strategy"]
+                if "gitea" in strat or "forgejo" in strat:
+                    api = GiteaForgejoAPI(upstream)
+
+        if api:
+            if api.archived():
+                # A stupid value that we know to be higher than the trigger value
+                return app, 1000
+
+            last_commit = api.commits()[0]
+            date = last_commit["commit"]["author"]["date"]
+            date = datetime.datetime.fromisoformat(date)
+            ago: datetime.timedelta = datetime.datetime.now() - date.replace(tzinfo=None)
+            return app, ago.days
+    except Exception:
+        logging.error(f"Exception while handling {app}", traceback.format_exc())
+        raise
+
+    raise RuntimeError(f"App {app} not handled (not github, gitlab or gitea with autoupdate). Upstream is {upstream}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("apps", nargs="*", type=str,
+                        help="If not passed, the script will run on the catalog. Github keys required.")
+    parser.add_argument("-j", "--processes", type=int, default=multiprocessing.cpu_count())
+    args = parser.parse_args()
+
+    apps_dict = get_catalog()
+    if args.apps:
+        apps_dict = {app: info for app, info in apps_dict.items() if app in args.apps}
+
+    deprecated: list[str] = []
+    not_deprecated: list[str] = []
+    # for app, info in apps_dict.items():
+    with multiprocessing.Pool(processes=args.processes) as pool:
+        tasks = pool.imap_unordered(upstream_last_update_ago, apps_dict.keys())
+
+        for _ in tqdm.tqdm(range(len(apps_dict)), ascii=" ·#"):
+            try:
+                app, result = next(tasks)
+            except Exception as e:
+                print(f"Exception found: {e}")
+                continue
+
+            if result is None:
+                continue
+
+            if result > 365:
+                deprecated.append(app)
+            else:
+                not_deprecated.append(app)
+
+    catalog = tomlkit.load(open("apps.toml"))
+    for app, info in catalog.items():
+        antifeatures = info.get("antifeatures", [])
+        if app in deprecated:
+            if "deprecated-software" not in antifeatures:
+                antifeatures.append("deprecated-software")
+        elif app in not_deprecated:
+            if "deprecated-software" in antifeatures:
+                antifeatures.remove("deprecated-software")
+        else:
+            continue
+        # unique the keys
+        if antifeatures:
+            info["antifeatures"] = antifeatures
+        else:
+            if "antifeatures" in info.keys():
+                info.pop("antifeatures")
+    tomlkit.dump(catalog, open("apps.toml", "w"))
+
+
+if __name__ == "__main__":
+    main()