maubot: add plugins & plugins update script

{ lib

, fetchPypi

, fetchpatch

, callPackage

, runCommand

, python3

, encryptionSupport ? true

      rm $out/example-config.yaml

    '';

    passthru = {

      inherit python;

    # Setuptools is trying to do python -m maubot test

    dontUseSetuptoolsCheck = true;

    pythonImportsCheck = [

      "maubot"

    ];

    passthru = let

      wrapper = callPackage ./wrapper.nix {

        unwrapped = maubot;

        python3 = python;

      };

    in

    {

      tests = {

        simple = runCommand "${pname}-tests" { } ''

          ${maubot}/bin/mbc --help > $out

        '';

      };

      inherit python;

      plugins = callPackage ./plugins {

        maubot = maubot;

        python3 = python;

      };

    # Setuptools is trying to do python -m maubot test

    dontUseSetuptoolsCheck = true;

      withPythonPackages = pythonPackages: wrapper { inherit pythonPackages; };

    pythonImportsCheck = [

      "maubot"

    ];

      # This adds the plugins to lib/maubot-plugins

      withPlugins = plugins: wrapper { inherit plugins; };

      # This changes example-config.yaml in module directory

      withBaseConfig = baseConfig: wrapper { inherit baseConfig; };

    };

    meta = with lib; {

      description = "A plugin-based Matrix bot system written in Python";

{ lib

, fetchgit

, fetchFromGitHub

, fetchFromGitLab

, fetchFromGitea

, stdenvNoCC

, callPackage

, maubot

, python3

, poetry

, formats

}:

let

  # pname: plugin id (example: xyz.maubot.echo)

  # version: plugin version

  # other attributes are passed directly to stdenv.mkDerivation (you at least need src)

  buildMaubotPlugin = attrs@{ version, pname, base_config ? null, ... }:

    stdenvNoCC.mkDerivation (builtins.removeAttrs attrs [ "base_config" ] // {

      pluginName = "${pname}-v${version}.mbp";

      nativeBuildInputs = (attrs.nativeBuildInputs or [ ]) ++ [ maubot ];

      buildPhase = ''

        runHook preBuild

        mbc build

        runHook postBuild

      '';

      postPatch = lib.optionalString (base_config != null) ''

        [ -e base-config.yaml ] || (echo "base-config.yaml doesn't exist, can't override it" && exit 1)

        cp "${if builtins.isPath base_config || lib.isDerivation base_config then base_config

          else if builtins.isString base_config then builtins.toFile "base-config.yaml" base_config

          else (formats.yaml { }).generate "base-config.yaml" base_config}" base-config.yaml

      '' + attrs.postPatch or "";

      installPhase = ''

        runHook preInstall

        mkdir -p $out/lib/maubot-plugins

        install -m 444 $pluginName $out/lib/maubot-plugins

        runHook postInstall

      '';

    });

  generated = import ./generated.nix {

    inherit lib fetchgit fetchFromGitHub fetchFromGitLab

      fetchFromGitea python3 poetry buildMaubotPlugin;

  };

in

generated // {

  inherit buildMaubotPlugin;

  allOfficialPlugins =

    builtins.filter

      (x: x.isOfficial && !x.meta.broken)

      (builtins.attrValues generated);

  allPlugins =

    builtins.filter

      (x: !x.meta.broken)

      (builtins.attrValues generated);

}

{ lib

, fetchgit

, fetchFromGitHub

, fetchFromGitLab

, fetchFromGitea

, python3

, poetry

, buildMaubotPlugin

}:

let

  json = builtins.fromJSON (builtins.readFile ./generated.json);

in

lib.flip builtins.mapAttrs json (name: entry:

let

  inherit (entry) manifest;

  resolveDeps = deps: map

    (name:

      let

        packageName = builtins.head (builtins.match "([^~=<>]*).*" name);

        lower = lib.toLower packageName;

        dash = builtins.replaceStrings ["_"] ["-"] packageName;

        lowerDash = builtins.replaceStrings ["_"] ["-"] lower;

      in

        python3.pkgs.${packageName}

        or python3.pkgs.${lower}

        or python3.pkgs.${dash}

        or python3.pkgs.${lowerDash}

        or null)

    (builtins.filter (x: x != "maubot" && x != null) deps);

  reqDeps = resolveDeps (lib.toList (manifest.dependencies or null));

  optDeps = resolveDeps (lib.toList (manifest.soft_dependencies or null));

in

lib.makeOverridable buildMaubotPlugin (entry.attrs // {

  pname = manifest.id;

  inherit (manifest) version;

  src =

    if entry?github then fetchFromGitHub entry.github

    else if entry?git then fetchgit entry.git

    else if entry?gitlab then fetchFromGitLab entry.gitlab

    else if entry?gitea then fetchFromGitea entry.gitea

    else throw "Invalid generated entry for ${manifest.id}: missing source";

  propagatedBuildInputs = builtins.filter (x: x != null) (reqDeps ++ optDeps);

  passthru.isOfficial = entry.isOfficial or false;

  meta = entry.attrs.meta // {

    license =

      let

        spdx = entry.attrs.meta.license or manifest.license or "unfree";

        spdxLicenses = builtins.listToAttrs

          (map (x: lib.nameValuePair x.spdxId x) (builtins.filter (x: x?spdxId) (builtins.attrValues lib.licenses)));

      in

      spdxLicenses.${spdx};

    broken = builtins.any (x: x == null) reqDeps;

  };

} // lib.optionalAttrs (entry.isPoetry or false) {

  nativeBuildInputs = [

    poetry

    (python3.withPackages (p: with p; [ toml ruamel-yaml isort ]))

  ];

  preBuild = lib.optionalString (entry?attrs.preBuild) (entry.attrs.preBuild + "\n") + ''

    export HOME=$(mktemp -d)

    [[ ! -d scripts ]] || patchShebangs --build scripts

    make maubot.yaml

  '';

}))

#!/usr/bin/env nix-shell

#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"

import git

import json

import os

import subprocess

import ruamel.yaml

import sys

import toml

import zipfile

from typing import Dict, List

HOSTNAMES = {

    'git.skeg1.se': 'gitlab',

    'edugit.org': 'gitlab',

    'codeberg.org': 'gitea',

}

PLUGINS: Dict[str, dict] = {}

yaml = ruamel.yaml.YAML(typ='safe')

TMP = os.environ.get('TEMPDIR', '/tmp')

def process_repo(path: str, official: bool):

    global PLUGINS

    with open(path, 'rt') as f:

        data = yaml.load(f)

    name, repourl, license, desc = data['name'], data['repo'], data['license'], data['description']

    origurl = repourl

    if '/' in name or ' ' in name:

        name = os.path.split(path)[-1].removesuffix('.yaml')

    name = name.replace('_', '-')

    if name in PLUGINS.keys():

        raise ValueError(f'Duplicate plugin {name}, refusing to continue')

    repodir = os.path.join(TMP, 'maubot-plugins', name)

    plugindir = repodir

    if '/tree/' in repourl:

        repourl, rev_path = repourl.split('/tree/')

        rev, subdir = rev_path.strip('/').split('/')

        plugindir = os.path.join(plugindir, subdir)

    else:

        rev = None

        subdir = None

    if repourl.startswith('http:'):

        repourl = 'https' + repourl[4:]

    repourl = repourl.rstrip('/')

    if not os.path.exists(repodir):

        print('Fetching', name)

        repo = git.Repo.clone_from(repourl + '.git', repodir)

    else:

        repo = git.Repo(repodir)

    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)

    tags = list(filter(lambda x: 'rc' not in str(x), tags))

    if tags:

        repo.git.checkout(tags[-1])

        rev = str(tags[-1])

    else:

        rev = str(repo.commit('HEAD'))

    ret: dict = {'attrs':{}}

    if subdir:

        ret['attrs']['postPatch'] = f'cd {subdir}'

    domain, query = repourl.removeprefix('https://').split('/', 1)

    hash = subprocess.run([

        'nurl',

        '--hash',

        f'file://{repodir}',

        rev

    ], capture_output=True, check=True).stdout.decode('utf-8')

    ret['attrs']['meta'] = {

        'description': desc,

        'homepage': origurl,

    }

    if domain.endswith('github.com'):

        owner, repo = query.split('/')

        ret['github'] = {

            'owner': owner,

            'repo': repo,

            'rev': rev,

            'hash': hash,

        }

        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'

        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'

        repobase = f'{repourl}/blob/{rev}'

    elif HOSTNAMES.get(domain, 'gitea' if 'gitea.' in domain or 'forgejo.' in domain else None) == 'gitea':

        owner, repo = query.split('/')

        ret['gitea'] = {

            'domain': domain,

            'owner': owner,

            'repo': repo,

            'rev': rev,

            'hash': hash,

        }

        repobase = f'{repourl}/src/commit/{rev}'

        ret['attrs']['meta']['downloadPage'] = f'{repourl}/releases'

        ret['attrs']['meta']['changelog'] = f'{repourl}/releases'

    elif HOSTNAMES.get(domain, 'gitlab' if 'gitlab.' in domain else None) == 'gitlab':

        owner, repo = query.split('/')

        ret['gitlab'] = {

            'owner': owner,

            'repo': repo,

            'rev': rev,

            'hash': hash,

        }

        if domain != 'gitlab.com':

            ret['gitlab']['domain'] = domain

        repobase = f'{repourl}/-/blob/{rev}'

    else:

        raise ValueError(f'Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!')

    if os.path.exists(os.path.join(plugindir, 'CHANGELOG.md')):

        ret['attrs']['meta']['changelog'] = f'{repobase}/CHANGELOG.md'

    if os.path.exists(os.path.join(plugindir, 'maubot.yaml')):

        with open(os.path.join(plugindir, 'maubot.yaml'), 'rt') as f:

            ret['manifest'] = yaml.load(f)

    elif os.path.exists(os.path.join(plugindir, 'pyproject.toml')):

        ret['isPoetry'] = True

        with open(os.path.join(plugindir, 'pyproject.toml'), 'rt') as f:

            data = toml.load(f)

        deps = []

        for key, val in data['tool']['poetry'].get('dependencies', {}).items():

            if key in ['maubot', 'mautrix', 'python']:

                continue

            reqs = []

            for req in val.split(','):

                reqs.extend(poetry_to_pep(req))

            deps.append(key + ', '.join(reqs))

        ret['manifest'] = data['tool']['maubot']

        ret['manifest']['id'] = data['tool']['poetry']['name']

        ret['manifest']['version'] = data['tool']['poetry']['version']

        ret['manifest']['license'] = data['tool']['poetry']['license']

        if deps:

            ret['manifest']['dependencies'] = deps

    else:

        raise ValueError(f'No maubot.yaml or pyproject.toml found in {repodir}')

    # normalize non-spdx-conformant licenses this way

    # (and fill out missing license info)

    if 'license' not in ret['manifest'] or ret['manifest']['license'] in ['GPLv3', 'AGPL 3.0']:

        ret['attrs']['meta']['license'] = license

    elif ret['manifest']['license'] != license:

        print(f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}")

    if official:

        ret['isOfficial'] = official

    PLUGINS[name] = ret

def next_incomp(ver_s: str) -> str:

    ver = ver_s.split('.')

    zero = False

    for i in range(len(ver)):

        try:

            seg = int(ver[i])

        except ValueError:

            if zero:

                ver = ver[:i]

                break

            continue

        if zero:

            ver[i] = '0'

        elif seg:

            ver[i] = str(seg + 1)

            zero = True

    return '.'.join(ver)

def poetry_to_pep(ver_req: str) -> List[str]:

    if '*' in ver_req:

        raise NotImplementedError('Wildcard poetry versions not implemented!')

    if ver_req.startswith('^'):

        return ['>=' + ver_req[1:], '<' + next_incomp(ver_req[1:])]

    if ver_req.startswith('~'):

        return ['~=' + ver_req[1:]]

    return [ver_req]

def main():

    cache_path = os.path.join(TMP, 'maubot-plugins')

    if not os.path.exists(cache_path):

        os.makedirs(cache_path)

        git.Repo.clone_from('https://github.com/maubot/plugins.maubot.xyz', os.path.join(cache_path, '_repo'))

    else:

        pass

    repodir = os.path.join(cache_path, '_repo')

    for suffix, official in (('official', True), ('thirdparty', False)):

        directory = os.path.join(repodir, 'data', 'plugins', suffix)

        for plugin_name in os.listdir(directory):

            process_repo(os.path.join(directory, plugin_name), official)

    if os.path.isdir('pkgs/tools/networking/maubot/plugins'):

        generated = 'pkgs/tools/networking/maubot/plugins/generated.json'

    else:

        script_dir = os.path.dirname(os.path.realpath(__file__))

        generated = os.path.join(script_dir, 'generated.json')

    with open(generated, 'wt') as file:

        json.dump(PLUGINS, file, indent='  ', separators=(',', ': '), sort_keys=True)

        file.write('\n')

if __name__ == '__main__':

    main()