Unverified Commit 30a78195 authored by Pascal Bach's avatar Pascal Bach Committed by GitHub
Browse files

llama-cpp.service: add support for --models-dir flag (#477593)

parents 2c94b36c 653f976d

nixos/modules/services/misc/llama-cpp.nix

+28 −3
Original line number Diff line number Diff line
@@ -19,9 +19,17 @@ in 
      package = lib.mkPackageOption pkgs "llama-cpp" { };



      model = lib.mkOption {

        type = lib.types.path;

        type = lib.types.nullOr lib.types.path;

        example = "/models/mistral-instruct-7b/ggml-model-q4_0.gguf";

        description = "Model path.";

        default = null;

      };



      modelsDir = lib.mkOption {

        type = lib.types.nullOr lib.types.path;

        example = "/models/";

        description = "Models directory.";

        default = null;

      };



      extraFlags = lib.mkOption {
@@ -61,7 +69,6 @@ in 
  };



  config = lib.mkIf cfg.enable {



    systemd.services.llama-cpp = {

      description = "LLaMA C++ server";

      after = [ "network.target" ];
@@ -70,7 +77,25 @@ in 
      serviceConfig = {

        Type = "idle";

        KillSignal = "SIGINT";

        ExecStart = "${cfg.package}/bin/llama-server --log-disable --host ${cfg.host} --port ${toString cfg.port} -m ${cfg.model} ${utils.escapeSystemdExecArgs cfg.extraFlags}";

        ExecStart =

          let

            args = [

              "--host"

              cfg.host

              "--port"

              (toString cfg.port)

            ]

            ++ lib.optionals (cfg.model != null) [

              "-m"

              cfg.model

            ]

            ++ lib.optionals (cfg.modelsDir != null) [

              "--models-dir"

              cfg.modelsDir

            ]

            ++ cfg.extraFlags;

          in

          "${cfg.package}/bin/llama-server ${utils.escapeSystemdExecArgs args}";

        Restart = "on-failure";

        RestartSec = 300;