Commit 7683f254 authored by Bert Proesmans's avatar Bert Proesmans
Browse files

Add missing dependencies and restore working runtimes

parent 5317e562
Loading
Loading
Loading
Loading
+37 −0
Original line number Diff line number Diff line
diff --git a/windmill-worker/nsjail/download.py.config.proto b/windmill-worker/nsjail/download.py.config.proto
index 5c2989417..7896bde7a 100644
--- a/windmill-worker/nsjail/download.py.config.proto
+++ b/windmill-worker/nsjail/download.py.config.proto
@@ -22,17 +22,24 @@ keep_caps: true
 keep_env: true
 mount_proc: true
 
+mount {
+    src: "/nix/store"
+    dst: "/nix/store"
+	is_bind: true
+}
 
 mount {
     src: "/bin"
     dst: "/bin"
 	is_bind: true
+    mandatory: false
 }
 
 mount {
     src: "/lib"
     dst: "/lib"
 	is_bind: true
+    mandatory: false
 }
 
 mount {
@@ -46,6 +53,7 @@ mount {
     src: "/usr"
     dst: "/usr"
 	is_bind: true
+    mandatory: false
 }
 
 mount {
+84 −21
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@
  rustfmt,
  stdenv,
  swagger-cli,
  perl,
  _experimental-update-script-combinators,
  nix-update-script,
  writeScript,
@@ -30,6 +31,19 @@
      inherit (callPackage ./fetchers.nix { }) fetchLibrustyV8;
    }
  ),
  libxml2,
  xmlsec,
  libxslt,
  flock,
  powershell,
  uv,
  bun,
  dotnet-sdk_9,
  php,
  procps,
  cargo,
  coreutils,
  withEnterpriseFeatures ? false,
}:

let
@@ -42,8 +56,6 @@ let
    rev = "v${version}";
    hash = "sha256-JhgqBXiX0ClEQZkWl7YBsBlQHk2Jp4jIdHy5CDvdoAM=";
  };

  pythonEnv = python3.withPackages (ps: [ ps.pip-tools ]);
in
rustPlatform.buildRustPackage (finalAttrs: {
  inherit pname version src;
@@ -65,22 +77,62 @@ rustPlatform.buildRustPackage (finalAttrs: {
    };
  };

  buildFeatures =
    [
      "embedding"
      "parquet"
      "prometheus"
      "openidconnect"
      "cloud"
      "jemalloc"
      "deno_core"
      "license"
      "http_trigger"
      "zip"
      "oauth2"
      "kafka"
      "otel"
      "dind"
      "php"
      "mysql"
      "mssql"
      "bigquery"
      "websocket"
      "python"
      "smtp"
      "csharp"
      "static_frontend"
      # "rust" # compiler environment is incomplete
    ]
    ++ (lib.optionals withEnterpriseFeatures [
      "enterprise"
      "enterprise_saml"
      "tantivy"
      "stripe"
    ]);

  patches = [
    ./swagger-cli.patch
    ./download.py.config.proto.patch
    ./python_executor.patch
    ./run.ansible.config.proto.patch
    ./run.bash.config.proto.patch
    ./run.bun.config.proto.patch
    ./run.csharp.config.proto.patch
    ./run.go.config.proto.patch
    ./run.php.config.proto.patch
    ./run.powershell.config.proto.patch
    ./run.python3.config.proto.patch
    ./run.bash.config.proto.patch
    ./run.rust.config.proto.patch
    ./rust_executor.patch
    ./swagger-cli.patch
  ];

  postPatch = ''
    substituteInPlace windmill-worker/src/bash_executor.rs \
      --replace '"/bin/bash"' '"${bash}/bin/bash"'

    substituteInPlace windmill-api/src/lib.rs \
      --replace 'unknown-version' 'v${version}'
    substituteInPlace windmill-common/src/utils.rs \
      --replace-fail 'unknown-version' 'v${version}'

    substituteInPlace src/main.rs \
      --replace 'unknown-version' 'v${version}'
    substituteInPlace windmill-worker/src/python_executor.rs \
      --replace-fail 'unknown_system_python_version' '${python3.version}'
  '';

  buildInputs = [
@@ -88,35 +140,46 @@ rustPlatform.buildRustPackage (finalAttrs: {
    rustfmt
    lld
    (lib.getLib stdenv.cc.cc)
    libxml2
    xmlsec
    libxslt
  ];

  nativeBuildInputs = [
    pkg-config
    makeWrapper
    cmake # for libz-ng-sys crate
    perl
  ];

  # needs a postgres database running
  doCheck = false;

  # TODO; Check if the rpath is still required
  # patchelf --set-rpath ${lib.makeLibraryPath [ openssl ]} $out/bin/windmill
  postFixup = ''
    patchelf --set-rpath ${lib.makeLibraryPath [ openssl ]} $out/bin/windmill

    wrapProgram "$out/bin/windmill" \
      --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ stdenv.cc.cc ]} \
      --prefix PATH : ${
        lib.makeBinPath [
          go
          pythonEnv
          deno
          nsjail
          bash
          python3 # uv searches PATH for system python
          procps # bash_executor
          coreutils # bash_executor
        ]
      } \
      --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ stdenv.cc.cc ]} \
      --set PYTHON_PATH "${pythonEnv}/bin/python3" \
      --set PYTHON_PATH "${python3}/bin/python3" \
      --set GO_PATH "${go}/bin/go" \
      --set DENO_PATH "${deno}/bin/deno" \
      --set NSJAIL_PATH "${nsjail}/bin/nsjail"
      --set NSJAIL_PATH "${nsjail}/bin/nsjail" \
      --set FLOCK_PATH "${flock}/bin/flock" \
      --set BASH_PATH "${bash}/bin/bash" \
      --set POWERSHELL_PATH "${powershell}/bin/pwsh" \
      --set BUN_PATH "${bun}/bin/bun" \
      --set UV_PATH "${uv}/bin/uv" \
      --set DOTNET_PATH "${dotnet-sdk_9}/bin/dotnet" \
      --set DOTNET_ROOT "${dotnet-sdk_9}/share/dotnet" \
      --set PHP_PATH "${php}/bin/php" \
      --set CARGO_PATH "${cargo}/bin/cargo"
  '';

  passthru.web-ui = buildNpmPackage {
+100 −0
Original line number Diff line number Diff line
diff --git a/windmill-worker/src/python_executor.rs b/windmill-worker/src/python_executor.rs
index 1180652d3..7b4ccf6cf 100644
--- a/windmill-worker/src/python_executor.rs
+++ b/windmill-worker/src/python_executor.rs
@@ -290,70 +290,8 @@ impl PyVersion {
         occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
     ) -> error::Result<()> {
         let v = self.to_string_with_dot();
-        append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await;
-        // Create dirs for newly installed python
-        // If we dont do this, NSJAIL will not be able to mount cache
-        // For the default version directory created during startup (main.rs)
-        DirBuilder::new()
-            .recursive(true)
-            .create(self.to_cache_dir())
-            .await
-            .expect("could not create initial worker dir");
-
-        let logs = String::new();
-
-        #[cfg(windows)]
-        let uv_cmd = "uv";
-
-        #[cfg(unix)]
-        let uv_cmd = UV_PATH.as_str();
-
-        let mut child_cmd = Command::new(uv_cmd);
-        child_cmd
-            .env_clear()
-            .env("HOME", HOME_ENV.to_string())
-            .env("PATH", PATH_ENV.to_string())
-            .envs(PROXY_ENVS.clone())
-            .args(["python", "install", v, "--python-preference=only-managed"])
-            // TODO: Do we need these?
-            .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
-            .stdout(Stdio::piped())
-            .stderr(Stdio::piped());
-
-        #[cfg(windows)]
-        {
-            child_cmd
-                .env("SystemRoot", SYSTEM_ROOT.as_str())
-                .env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
-                .env(
-                    "TMP",
-                    std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
-                )
-                .env(
-                    "LOCALAPPDATA",
-                    std::env::var("LOCALAPPDATA")
-                        .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
-                );
-        }
-
-        let child_process = start_child_process(child_cmd, "uv").await?;
-
-        append_logs(&job_id, &w_id, logs, db).await;
-        handle_child(
-            job_id,
-            db,
-            mem_peak,
-            &mut None,
-            child_process,
-            false,
-            worker_name,
-            &w_id,
-            "uv",
-            None,
-            false,
-            occupancy_metrics,
-        )
-        .await
+        append_logs(job_id, w_id, format!("\nREQUESTED PYTHON INSTALL IGNORED ({})", v), db).await;
+        Err(error::Error::BadConfig(format!("Python is managed through the NixOS system configuration. Change the Windmill instance setting to version 'unknown_system_python_version'")))
     }
     async fn find_python(self) -> error::Result<Option<String>> {
         #[cfg(windows)]
@@ -391,11 +329,11 @@ impl PyVersion {
                 "find",
                 self.to_string_with_dot(),
                 "--system",
-                "--python-preference=only-managed",
+                "--python-preference=only-system",
             ])
             .envs([
                 ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR),
-                ("UV_PYTHON_PREFERENCE", "only-managed"),
+                ("UV_PYTHON_PREFERENCE", "only-system"),
             ])
             // .stdout(Stdio::piped())
             .stderr(Stdio::piped())
@@ -561,7 +499,7 @@ pub async fn uv_pip_compile(
             "-p",
             &py_version.to_string_with_dot(),
             "--python-preference",
-            "only-managed",
+            "only-system",
         ]);
 
         if no_cache {
+44 −0
Original line number Diff line number Diff line
diff --git a/windmill-worker/nsjail/run.ansible.config.proto b/windmill-worker/nsjail/run.ansible.config.proto
index 65a8ea700..d4c8c2afc 100644
--- a/windmill-worker/nsjail/run.ansible.config.proto
+++ b/windmill-worker/nsjail/run.ansible.config.proto
@@ -18,16 +18,24 @@ keep_caps: false
 keep_env: true
 mount_proc: true
 
+mount {
+    src: "/nix/store"
+    dst: "/nix/store"
+    is_bind: true
+}
+
 mount {
     src: "/bin"
     dst: "/bin"
     is_bind: true
+    mandatory: false
 }
 
 mount {
     src: "/lib"
     dst: "/lib"
     is_bind: true
+    mandatory: false
 }
 
 
@@ -42,12 +50,14 @@ mount {
     src: "/root/.local/share/uv/tools/ansible"
     dst: "/root/.local/share/uv/tools/ansible"
     is_bind: true
+    mandatory: false
 }
 
 mount {
     src: "/usr"
     dst: "/usr"
     is_bind: true
+    mandatory: false
 }
 
 mount {
+22 −17
Original line number Diff line number Diff line
diff --git a/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto
index e93e6b45..bbedb165 100644
diff --git a/windmill-worker/nsjail/run.bash.config.proto b/windmill-worker/nsjail/run.bash.config.proto
index 63018f765..d8bb0c0a7 100644
--- a/windmill-worker/nsjail/run.bash.config.proto
+++ b/windmill-worker/nsjail/run.bash.config.proto
@@ -18,6 +18,12 @@ clone_newuser: {CLONE_NEWUSER}
 keep_caps: false
@@ -15,10 +15,17 @@ keep_caps: false
 keep_env: true
 mount_proc: true
 
+mount {
+    src: "/nix/store"
@@ -15,27 +15,32 @@ index e93e6b45..bbedb165 100644
 mount {
     src: "/bin"
     dst: "/bin"
@@ -25,6 +31,7 @@ mount {
 	is_bind: true
+    mandatory: false
 }
 
 mount {
@@ -32,6 +39,7 @@ mount {
     src: "/bin"
     dst: "/bin"
 	is_bind: true
+    mandatory: false
 }
 
 mount {
@@ -45,6 +53,7 @@ mount {
     src: "/lib"
     dst: "/lib"
 	is_bind: true
@@ -32,6 +39,7 @@ mount {
 
 
 mount {
+    mandatory: false
     src: "/lib64"
     dst: "/lib64"
 	is_bind: true
@@ -39,6 +47,7 @@ mount {
 }
 
 
 mount {
+    mandatory: false
@@ -60,6 +69,7 @@ mount {
     src: "/usr"
     dst: "/usr"
 	is_bind: true
+    mandatory: false
 }
 
 mount {
Loading