Rename README-generator -> readme_generator because python tools hate caps…

tools/readme_generator/README.md

@@ -0,0 +1,38 @@
+# Auto-README generation
+
+### Initial install
+
+```
+python3 -m venv venv
+source venv/bin/activate
+pip install -r requirements.txt
+```
+
+### Use on a single app
+
+```
+source venv/bin/activate
+./make_readme.py /path/to/app
+```
+
+Then the README.md in the app folder will be updated
+
+### Launch webhook service for auto update
+
+Configure the webhook on github
+
+Also need to allow the bot to push on all repos
+
+Configure nginx to reverse proxy on port 8123 (or whichever port you set in the systemd config)
+
+```bash
+echo "github_webhook_secret" > github_webhook_secret
+echo "the_bot_login" > login
+echo "the_bot_token" > token
+```
+
+Add the webhook.service to systemd config, then start it:
+
+```bash
+systemctl start the_webhook_service 
+```

tools/readme_generator/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

tools/readme_generator/make_readme.py

@@ -0,0 +1,106 @@
+#! /usr/bin/env python3
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+import toml
+from jinja2 import Environment, FileSystemLoader
+
+
+def value_for_lang(values, lang):
+     if not isinstance(values, dict):
+         return values
+     if lang in values:
+         return values[lang]
+     elif "en" in values:
+         return values["en"]
+     else:
+         return list(values.values())[0]
+
+def generate_READMEs(app_path: str):
+
+    app_path = Path(app_path)
+
+    if not app_path.exists():
+        raise Exception("App path provided doesn't exists ?!")
+
+    if os.path.exists(app_path / "manifest.json"):
+        manifest = json.load(open(app_path / "manifest.json"))
+    else:
+        manifest = toml.load(open(app_path / "manifest.toml"))
+
+    upstream = manifest.get("upstream", {})
+
+    catalog = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "apps.toml"))
+    from_catalog = catalog.get(manifest['id'], {})
+
+    antifeatures_list = toml.load(open(Path(os.path.abspath(__file__)).parent.parent.parent / "antifeatures.toml"))
+
+    if not upstream and not (app_path / "doc" / "DISCLAIMER.md").exists():
+        print(
+            "There's no 'upstream' key in the manifest, and doc/DISCLAIMER.md doesn't exists - therefore assuming that we shall not auto-update the README.md for this app yet."
+        )
+        return
+
+    env = Environment(loader=FileSystemLoader(Path(__file__).parent / "templates"))
+
+    for lang, lang_suffix in [("en", ""), ("fr", "_fr")]:
+
+        template = env.get_template(f"README{lang_suffix}.md.j2")
+
+        if (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").exists():
+            description = (app_path / "doc" / f"DESCRIPTION{lang_suffix}.md").read_text()
+        # Fallback to english if maintainer too lazy to translate the description
+        elif (app_path / "doc" / "DESCRIPTION.md").exists():
+            description = (app_path / "doc" / "DESCRIPTION.md").read_text()
+        else:
+            description = None
+
+        if (app_path / "doc" / "screenshots").exists():
+            screenshots = os.listdir(os.path.join(app_path, "doc", "screenshots"))
+            if ".gitkeep" in screenshots:
+                screenshots.remove(".gitkeep")
+        else:
+            screenshots = []
+
+        if (app_path / "doc" / f"DISCLAIMER{lang_suffix}.md").exists():
+            disclaimer = (app_path / "doc" / f"DISCLAIMER{lang_suffix}.md").read_text()
+        # Fallback to english if maintainer too lazy to translate the disclaimer idk
+        elif (app_path / "doc" / "DISCLAIMER.md").exists():
+            disclaimer = (app_path / "doc" / "DISCLAIMER.md").read_text()
+        else:
+            disclaimer = None
+
+        # TODO: Add url to the documentation... and actually create that documentation :D
+        antifeatures = { a: antifeatures_list[a] for a in from_catalog.get('antifeatures', [])}
+        for k, v in antifeatures.items():
+            antifeatures[k]['title'] = value_for_lang(v['title'], lang)
+            if manifest.get("antifeatures", {}).get(k, None):
+                antifeatures[k]['description'] = value_for_lang(manifest.get("antifeatures", {}).get(k, None), lang)
+            else:
+                antifeatures[k]['description'] = value_for_lang(antifeatures[k]['description'], lang)
+
+        out = template.render(
+            lang=lang,
+            upstream=upstream,
+            description=description,
+            screenshots=screenshots,
+            disclaimer=disclaimer,
+            antifeatures=antifeatures,
+            manifest=manifest,
+        )
+        (app_path / f"README{lang_suffix}.md").write_text(out)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Automatically (re)generate README for apps"
+    )
+    parser.add_argument(
+        "app_path", help="Path to the app to generate/update READMEs for"
+    )
+
+    args = parser.parse_args()
+    generate_READMEs(args.app_path)

tools/readme_generator/nginx.conf

@@ -0,0 +1,17 @@
+location /github {
+
+  # Force usage of https
+  if ($scheme = http) {
+    rewrite ^ https://$server_name$request_uri? permanent;
+  }
+
+  client_max_body_size 100M;
+
+  proxy_pass http://127.0.0.1:8123;
+  proxy_http_version 1.1;
+  proxy_set_header Upgrade $http_upgrade;
+  proxy_set_header Connection "Upgrade";
+
+  # preserve client IP
+  proxy_set_header X-Forwarded-For $remote_addr;
+}

tools/readme_generator/requirements.txt

@@ -0,0 +1,5 @@
+jinja2
+sanic==21.12.2
+pyyaml
+toml
+websockets==10.0

tools/readme_generator/templates/README.md.j2

@@ -0,0 +1,90 @@
+{% if manifest.id == "example" -%}
+# Packaging an app, starting from this example
+
+* Copy this app before working on it, using the ['Use this template'](https://github.com/YunoHost/example_ynh/generate) button on the Github repo.
+* Edit the `manifest.json` with app specific info.
+* Edit the `install`, `upgrade`, `remove`, `backup`, and `restore` scripts, and any relevant conf files in `conf/`.
+  * Using the [script helpers documentation.](https://yunohost.org/packaging_apps_helpers)
+* Add a `LICENSE` file for the package.
+* Edit `doc/DISCLAIMER*.md`
+* The `README.md` files are to be automatically generated by https://github.com/YunoHost/apps/tree/master/tools/readme_generator
+
+---
+{% endif -%}
+
+<!--
+N.B.: This README was automatically generated by https://github.com/YunoHost/apps/tree/master/tools/readme_generator
+It shall NOT be edited by hand.
+-->
+
+# {{manifest.name}} for YunoHost
+
+[![Integration level](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Working status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Maintenance status](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)
+
+[![Install {{manifest.name}} with YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}})
+
+*[Lire ce readme en français.](./README_fr.md)*
+
+> *This package allows you to install {{manifest.name}} quickly and simply on a YunoHost server.
+If you don't have YunoHost, please consult [the guide](https://yunohost.org/#/install) to learn how to install it.*
+
+## Overview
+
+{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %}
+
+**Shipped version:** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}
+{% endif -%}
+
+{% if upstream.demo %}
+**Demo:** {{upstream.demo}}
+{% endif -%}
+
+{% if screenshots %}
+## Screenshots
+
+{% for screenshot in screenshots -%}
+    ![Screenshot of {{manifest.name}}](./doc/screenshots/{{screenshot}})
+{% endfor %}
+{% endif -%}
+
+{% if disclaimer -%}
+## Disclaimers / important information
+
+{{ disclaimer }}
+{% endif -%}
+
+{% if antifeatures -%}
+## :red_circle: Antifeatures
+
+{% for antifeature in antifeatures.values() -%}
+  - **{{ antifeature.title }}**: {{ antifeature.description }}
+
+{% endfor -%}
+{% endif -%}
+
+## Documentation and resources
+
+{% if upstream.website -%}* Official app website: <{{ upstream.website }}>
+{% endif -%}
+{% if upstream.userdoc -%}* Official user documentation: <{{ upstream.userdoc }}>
+{% endif -%}
+{% if upstream.admindoc -%}* Official admin documentation: <{{ upstream.admindoc }}>
+{% endif -%}
+{% if upstream.code -%}* Upstream app code repository: <{{ upstream.code }}>
+{% endif -%}
+* YunoHost Store: <https://apps.yunohost.org/app/{{manifest.id}}>
+* Report a bug: <https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues>
+
+## Developer info
+
+Please send your pull request to the [testing branch](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing).
+
+To try the testing branch, please proceed like that.
+
+``` bash
+sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
+or
+sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
+```
+
+**More info regarding app packaging:** <https://yunohost.org/packaging_apps>

tools/readme_generator/templates/README_fr.md.j2

@@ -0,0 +1,77 @@
+<!--
+N.B.: This README was automatically generated by https://github.com/YunoHost/apps/tree/master/tools/readme_generator
+It shall NOT be edited by hand.
+-->
+
+# {{manifest.name}} pour YunoHost
+
+[![Niveau d’intégration](https://dash.yunohost.org/integration/{{manifest.id}}.svg)](https://dash.yunohost.org/appci/app/{{manifest.id}}) ![Statut du fonctionnement](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.status.svg) ![Statut de maintenance](https://ci-apps.yunohost.org/ci/badges/{{manifest.id}}.maintain.svg)
+
+[![Installer {{manifest.name}} avec YunoHost](https://install-app.yunohost.org/install-with-yunohost.svg)](https://install-app.yunohost.org/?app={{manifest.id}})
+
+*[Read this readme in english.](./README.md)*
+
+> *Ce package vous permet d’installer {{manifest.name}} rapidement et simplement sur un serveur YunoHost.
+Si vous n’avez pas YunoHost, regardez [ici](https://yunohost.org/#/install) pour savoir comment l’installer et en profiter.*
+
+## Vue d’ensemble
+
+{% if description %}{{description}}{% else %}{{manifest.description[lang]}}{% endif %}
+
+**Version incluse :** {% if upstream.version %}{{upstream.version}}{% else %}{{manifest.version}}
+{% endif -%}
+
+{% if upstream.demo %}
+**Démo :** {{upstream.demo}}
+{% endif -%}
+
+{% if screenshots %}
+## Captures d’écran
+
+{% for screenshot in screenshots -%}
+    ![Capture d’écran de {{manifest.name}}](./doc/screenshots/{{screenshot}})
+{% endfor %}
+{% endif -%}
+
+{% if disclaimer -%}
+## Avertissements / informations importantes
+
+{{ disclaimer }}
+{% endif -%}
+
+{% if antifeatures -%}
+## :red_circle: Fonctions indésirables
+
+{% for antifeature in antifeatures.values() -%}
+  - **{{ antifeature.title }}**: {{ antifeature.description }}
+
+{% endfor -%}
+{% endif -%}
+
+
+## Documentations et ressources
+
+{% if upstream.website -%}* Site officiel de l’app : <{{ upstream.website }}>
+{% endif -%}
+{% if upstream.userdoc -%}* Documentation officielle utilisateur : <{{ upstream.userdoc }}>
+{% endif -%}
+{% if upstream.admindoc -%}* Documentation officielle de l’admin : <{{ upstream.admindoc }}>
+{% endif -%}
+{% if upstream.code -%}* Dépôt de code officiel de l’app : <{{ upstream.code }}>
+{% endif -%}
+* YunoHost Store: <https://apps.yunohost.org/app/{{manifest.id}}>
+* Signaler un bug : <https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/issues>
+
+## Informations pour les développeurs
+
+Merci de faire vos pull request sur la [branche testing](https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing).
+
+Pour essayer la branche testing, procédez comme suit.
+
+``` bash
+sudo yunohost app install https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
+ou
+sudo yunohost app upgrade {{manifest.id}} -u https://github.com/YunoHost-Apps/{{manifest.id}}_ynh/tree/testing --debug
+```
+
+**Plus d’infos sur le packaging d’applications :** <https://yunohost.org/packaging_apps>

tools/readme_generator/webhook.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+import asyncio
+import hashlib
+import hmac
+import os
+import shlex
+import tempfile
+
+from make_readme import generate_READMEs
+from sanic import Sanic, response
+from sanic.response import text
+
+app = Sanic(__name__)
+
+github_webhook_secret = open("github_webhook_secret", "r").read().strip()
+
+login = open("login").read().strip()
+token = open("token").read().strip()
+
+my_env = os.environ.copy()
+my_env["GIT_TERMINAL_PROMPT"] = "0"
+my_env["GIT_AUTHOR_NAME"] = "yunohost-bot"
+my_env["GIT_AUTHOR_EMAIL"] = "yunohost@yunohost.org"
+my_env["GIT_COMMITTER_NAME"] = "yunohost-bot"
+my_env["GIT_COMMITTER_EMAIL"] = "yunohost@yunohost.org"
+
+
+async def git(cmd, in_folder=None):
+
+    if not isinstance(cmd, list):
+        cmd = cmd.split()
+    if in_folder:
+        cmd = ["-C", in_folder] + cmd
+    cmd = ["git"] + cmd
+    cmd = " ".join(map(shlex.quote, cmd))
+    print(cmd)
+    command = await asyncio.create_subprocess_shell(cmd, env=my_env, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+    data = await command.stdout.read()
+    return data.decode().strip()
+
+
+@app.route("/github", methods=["GET"])
+def main_route(request):
+    return text("You aren't supposed to go on this page using a browser, it's for webhooks push instead.")
+
+
+@app.route("/github", methods=["POST"])
+async def on_push(request):
+    header_signature = request.headers.get("X-Hub-Signature")
+    if header_signature is None:
+        print("no header X-Hub-Signature")
+        return response.json({"error": "No X-Hub-Signature"}, 403)
+
+    sha_name, signature = header_signature.split("=")
+    if sha_name != "sha1":
+        print("signing algo isn't sha1, it's '%s'" % sha_name)
+        return response.json({"error": "Signing algorightm is not sha1 ?!"}, 501)
+
+    # HMAC requires the key to be bytes, but data is string
+    mac = hmac.new(github_webhook_secret.encode(), msg=request.body, digestmod=hashlib.sha1)
+
+    if not hmac.compare_digest(str(mac.hexdigest()), str(signature)):
+        return response.json({"error": "Bad signature ?!"}, 403)
+
+    data = request.json
+
+    repository = data["repository"]["full_name"]
+    branch = data["ref"].split("/", 2)[2]
+
+    print(f"{repository} -> branch '{branch}'")
+
+    with tempfile.TemporaryDirectory() as folder:
+        await git(["clone", f"https://{login}:{token}@github.com/{repository}", "--single-branch", "--branch", branch, folder])
+        generate_READMEs(folder)
+
+        await git(["add", "README*.md"], in_folder=folder)
+
+        diff_not_empty = await asyncio.create_subprocess_shell(" ".join(["git", "diff", "HEAD", "--compact-summary"]), cwd=folder, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT)
+        diff_not_empty = await diff_not_empty.stdout.read()
+        diff_not_empty = diff_not_empty.decode().strip()
+        if not diff_not_empty:
+            print("nothing to do")
+            return text("nothing to do")
+
+        await git(["commit", "-a", "-m", "Auto-update README", "--author='yunohost-bot <yunohost@yunohost.org>'"], in_folder=folder)
+        await git(["push", "origin", branch, "--quiet"], in_folder=folder)
+
+    return text("ok")
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=8123)

tools/readme_generator/webhook.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Auto-README webhook gunicorn daemon
+After=network.target
+
+[Service]
+PIDFile=/run/gunicorn/autoreadme_webhook-pid
+User=autoreadme_webhook
+Group=autoreadme_webhook
+WorkingDirectory=__PATH_TO_README_GENERATOR__
+ExecStart=__PATH_TO_README_GENERATOR__/venv/bin/gunicorn -w 4 -b 127.0.0.1:8123 webhook:app
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target