Commit 552043a3 authored by Alexander Tomokhov's avatar Alexander Tomokhov Committed by Weijia Wang
Browse files

nixos/castopod: fix startup, displaying images, uploads up to 500 MiB

- new maxUploadSize option
- new dataDir option (with ReadWritePaths systemd support)
- admin page reports correct free disk space (instead of /nix/store)
- fix example configuration in documentation
- now podcast creation and file upload are tested during NixOS test
- move castopod from audio to web-apps folder
- verbose logging from the browser test
parent 2f0ef551
Loading
Loading
Loading
Loading
+1 −1
Original line number Diff line number Diff line
@@ -342,7 +342,6 @@
  ./services/amqp/rabbitmq.nix
  ./services/audio/alsa.nix
  ./services/audio/botamusique.nix
  ./services/audio/castopod.nix
  ./services/audio/gmediarender.nix
  ./services/audio/gonic.nix
  ./services/audio/goxlr-utility.nix
@@ -1302,6 +1301,7 @@
  ./services/web-apps/bookstack.nix
  ./services/web-apps/c2fmzq-server.nix
  ./services/web-apps/calibre-web.nix
  ./services/web-apps/castopod.nix
  ./services/web-apps/coder.nix
  ./services/web-apps/changedetection-io.nix
  ./services/web-apps/chatgpt-retrieval-plugin.nix
+1 −0
Original line number Diff line number Diff line
@@ -4,6 +4,7 @@ Castopod is an open-source hosting platform made for podcasters who want to enga

## Quickstart {#module-services-castopod-quickstart}

Configure ACME (https://nixos.org/manual/nixos/unstable/#module-security-acme).
Use the following configuration to start a public instance of Castopod on `castopod.example.com` domain:

```nix
+41 −19
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ let
  fpm = config.services.phpfpm.pools.castopod;

  user = "castopod";
  stateDirectory = "/var/lib/castopod";

  # https://docs.castopod.org/getting-started/install.html#requirements
  phpPackage = pkgs.php.withExtensions ({ enabled, all }: with all; [
@@ -29,6 +28,15 @@ in
        defaultText = lib.literalMD "pkgs.castopod";
        description = lib.mdDoc "Which Castopod package to use.";
      };
      dataDir = lib.mkOption {
        type = lib.types.path;
        default = "/var/lib/castopod";
        description = lib.mdDoc ''
          The path where castopod stores all data. This path must be in sync
          with the castopod package (where it is hardcoded during the build in
          accordance with its own `dataDir` argument).
        '';
      };
      database = {
        createLocally = lib.mkOption {
          type = lib.types.bool;
@@ -111,6 +119,18 @@ in
          Options for Castopod's PHP pool. See the documentation on `php-fpm.conf` for details on configuration directives.
        '';
      };
      maxUploadSize = lib.mkOption {
        type = lib.types.str;
        default = "512M";
        description = lib.mdDoc ''
          Maximum supported size for a file upload in. Maximum HTTP body
          size is set to this value for nginx and PHP (because castopod doesn't
          support chunked uploads yet:
          https://code.castopod.org/adaures/castopod/-/issues/330). Note, that
          practical upload size limit is smaller. For example, with 512 MiB
          setting - around 500 MiB is possible.
        '';
      };
    };
  };

@@ -120,13 +140,13 @@ in
        sslEnabled = with config.services.nginx.virtualHosts.${cfg.localDomain}; addSSL || forceSSL || onlySSL || enableACME || useACMEHost != null;
        baseURL = "http${lib.optionalString sslEnabled "s"}://${cfg.localDomain}";
      in
      lib.mapAttrs (name: lib.mkDefault) {
      lib.mapAttrs (_: lib.mkDefault) {
        "app.forceGlobalSecureRequests" = sslEnabled;
        "app.baseURL" = baseURL;

        "media.baseURL" = "/";
        "media.baseURL" = baseURL;
        "media.root" = "media";
        "media.storage" = stateDirectory;
        "media.storage" = cfg.dataDir;

        "admin.gateway" = "admin";
        "auth.gateway" = "auth";
@@ -142,13 +162,13 @@ in
    services.phpfpm.pools.castopod = {
      inherit user;
      group = config.services.nginx.group;
      phpPackage = phpPackage;
      inherit phpPackage;
      phpOptions = ''
        # https://code.castopod.org/adaures/castopod/-/blob/main/docker/production/app/uploads.ini
        # https://code.castopod.org/adaures/castopod/-/blob/develop/docker/production/common/uploads.template.ini
        file_uploads = On
        memory_limit = 512M
        upload_max_filesize = 500M
        post_max_size = 512M
        upload_max_filesize = ${cfg.maxUploadSize}
        post_max_size = ${cfg.maxUploadSize}
        max_execution_time = 300
        max_input_time = 300
      '';
@@ -165,25 +185,25 @@ in
      path = [ pkgs.openssl phpPackage ];
      script =
        let
          envFile = "${stateDirectory}/.env";
          envFile = "${cfg.dataDir}/.env";
          media = "${cfg.settings."media.storage"}/${cfg.settings."media.root"}";
        in
        ''
          mkdir -p ${stateDirectory}/writable/{cache,logs,session,temp,uploads}
          mkdir -p ${cfg.dataDir}/writable/{cache,logs,session,temp,uploads}

          if [ ! -d ${lib.escapeShellArg media} ]; then
            cp --no-preserve=mode,ownership -r ${cfg.package}/share/castopod/public/media ${lib.escapeShellArg media}
          fi

          if [ ! -f ${stateDirectory}/salt ]; then
            openssl rand -base64 33 > ${stateDirectory}/salt
          if [ ! -f ${cfg.dataDir}/salt ]; then
            openssl rand -base64 33 > ${cfg.dataDir}/salt
          fi

          cat <<'EOF' > ${envFile}
          ${lib.generators.toKeyValue { } cfg.settings}
          EOF

          echo "analytics.salt=$(cat ${stateDirectory}/salt)" >> ${envFile}
          echo "analytics.salt=$(cat ${cfg.dataDir}/salt)" >> ${envFile}

          ${if (cfg.database.passwordFile != null) then ''
            echo "database.default.password=$(cat ${lib.escapeShellArg cfg.database.passwordFile})" >> ${envFile}
@@ -192,10 +212,10 @@ in
          ''}

          ${lib.optionalString (cfg.environmentFile != null) ''
            cat ${lib.escapeShellArg cfg.environmentFile}) >> ${envFile}
            cat ${lib.escapeShellArg cfg.environmentFile} >> ${envFile}
          ''}

          php spark castopod:database-update
          php ${cfg.package}/share/castopod/spark castopod:database-update
        '';
      serviceConfig = {
        StateDirectory = "castopod";
@@ -204,6 +224,7 @@ in
        RemainAfterExit = true;
        User = user;
        Group = config.services.nginx.group;
        ReadWritePaths = cfg.dataDir;
      };
    };

@@ -212,9 +233,7 @@ in
      wantedBy = [ "multi-user.target" ];
      path = [ phpPackage ];
      script = ''
        php public/index.php scheduled-activities
        php public/index.php scheduled-websub-publish
        php public/index.php scheduled-video-clips
        php ${cfg.package}/share/castopod/spark tasks:run
      '';
      serviceConfig = {
        StateDirectory = "castopod";
@@ -222,6 +241,8 @@ in
        Type = "oneshot";
        User = user;
        Group = config.services.nginx.group;
        ReadWritePaths = cfg.dataDir;
        LogLevelMax = "notice"; # otherwise periodic tasks flood the journal
      };
    };

@@ -251,6 +272,7 @@ in
        extraConfig = ''
          try_files $uri $uri/ /index.php?$args;
          index index.php index.html;
          client_max_body_size ${cfg.maxUploadSize};
        '';

        locations."^~ /${cfg.settings."media.root"}/" = {
@@ -278,7 +300,7 @@ in
      };
    };

    users.users.${user} = lib.mapAttrs (name: lib.mkDefault) {
    users.users.${user} = lib.mapAttrs (_: lib.mkDefault) {
      description = "Castopod user";
      isSystemUser = true;
      group = config.services.nginx.group;
+209 −63
Original line number Diff line number Diff line
@@ -4,40 +4,104 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
  meta = with lib.maintainers; {
    maintainers = [ alexoundos misuzu ];
  };

  nodes.castopod = { nodes, ... }: {
    # otherwise 500 MiB file upload fails!
    virtualisation.diskSize = 512 + 3 * 512;

    networking.firewall.allowedTCPPorts = [ 80 ];
    networking.extraHosts = ''
      127.0.0.1 castopod.example.com
    '';
    networking.extraHosts =
      lib.strings.concatStringsSep "\n"
        (lib.attrsets.mapAttrsToList
          (name: _: "127.0.0.1 ${name}")
          nodes.castopod.services.nginx.virtualHosts);

    services.castopod = {
      enable = true;
      database.createLocally = true;
      localDomain = "castopod.example.com";
      maxUploadSize = "512M";
    };
  };

  nodes.client = { nodes, pkgs, lib, ... }:
    let
      domain = nodes.castopod.services.castopod.localDomain;

      getIP = node:
        (builtins.head node.networking.interfaces.eth1.ipv4.addresses).address;

      targetPodcastSize = 500 * 1024 * 1024;
      lameMp3Bitrate = 348300;
      lameMp3FileAdjust = -800;
      targetPodcastDuration = toString
        ((targetPodcastSize + lameMp3FileAdjust) / (lameMp3Bitrate / 8));
      bannerWidth = 3000;
      banner = pkgs.runCommand "gen-castopod-cover.jpg" { } ''
        ${pkgs.imagemagick}/bin/magick `
        `-background green -bordercolor white -gravity northwest xc:black `
        `-duplicate 99 `
        `-seed 1 -resize "%[fx:rand()*72+24]" `
        `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 16x36 `
        `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "150x50!" `
        `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append `
        `-resize ${toString bannerWidth} -quality 1 $out
      '';

      coverWidth = toString 3000;
      cover = pkgs.runCommand "gen-castopod-banner.jpg" { } ''
        ${pkgs.imagemagick}/bin/magick `
        `-background white -bordercolor white -gravity northwest xc:black `
        `-duplicate 99 `
        `-seed 1 -resize "%[fx:rand()*72+24]" `
        `-seed 0 -rotate "%[fx:rand()*360]" -border 6x6 -splice 36x36 `
        `-seed 0 -rotate "%[fx:floor(rand()*4)*90]" -resize "144x144!" `
        `+append -crop 10x1@ +repage -roll "+%[fx:(t%2)*72]+0" -append `
        `-resize ${coverWidth} -quality 1 $out
      '';
    in
    {
      networking.extraHosts =
        lib.strings.concatStringsSep "\n"
          (lib.attrsets.mapAttrsToList
            (name: _: "${getIP nodes.castopod} ${name}")
            nodes.castopod.services.nginx.virtualHosts);

      environment.systemPackages =
        let
          username = "admin";
        email = "admin@castood.example.com";
        password = "v82HmEp5";
        testRunner = pkgs.writers.writePython3Bin "test-runner"
          email = "admin@${domain}";
          password = "Abcd1234";
          podcastTitle = "Some Title";
          episodeTitle = "Episode Title";
          browser-test = pkgs.writers.writePython3Bin "browser-test"
            {
              libraries = [ pkgs.python3Packages.selenium ];
            flakeIgnore = [
              "E501"
            ];
              flakeIgnore = [ "E124" "E501" ];
            } ''
            from selenium.webdriver.common.by import By
            from selenium.webdriver import Firefox
            from selenium.webdriver.firefox.options import Options
            from selenium.webdriver.firefox.service import Service
            from selenium.webdriver.support.ui import WebDriverWait
            from selenium.webdriver.support import expected_conditions as EC
            from subprocess import STDOUT
            import logging

            selenium_logger = logging.getLogger("selenium")
            selenium_logger.setLevel(logging.DEBUG)
            selenium_logger.addHandler(logging.StreamHandler())

            options = Options()
            options.add_argument('--headless')
            service = Service(log_output=STDOUT)
            driver = Firefox(options=options, service=service)
            driver = Firefox(options=options)
          try:
              driver.implicitly_wait(20)
              driver.get('http://castopod.example.com/cp-install')
            driver.implicitly_wait(30)

            # install ##########################################################

            driver.get('http://${domain}/cp-install')

            wait = WebDriverWait(driver, 10)

@@ -52,7 +116,9 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
            driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
                '${password}'
            )
              driver.find_element(By.XPATH, "//button[contains(., 'Finish install')]").click()
            driver.find_element(By.XPATH,
                                "//button[contains(., 'Finish install')]"
            ).click()

            wait.until(EC.title_contains("Auth"))

@@ -62,16 +128,94 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
            driver.find_element(By.CSS_SELECTOR, '#password').send_keys(
                '${password}'
            )
              driver.find_element(By.XPATH, "//button[contains(., 'Login')]").click()
            driver.find_element(By.XPATH,
                                "//button[contains(., 'Login')]"
            ).click()

            wait.until(EC.title_contains("Admin dashboard"))
          finally:

            # create podcast ###################################################

            driver.get('http://${domain}/admin/podcasts/new')

            wait.until(EC.title_contains("Create podcast"))

            driver.find_element(By.CSS_SELECTOR, '#cover').send_keys(
                '${cover}'
            )
            driver.find_element(By.CSS_SELECTOR, '#banner').send_keys(
                '${banner}'
            )
            driver.find_element(By.CSS_SELECTOR, '#title').send_keys(
                '${podcastTitle}'
            )
            driver.find_element(By.CSS_SELECTOR, '#handle').send_keys(
                'some_handle'
            )
            driver.find_element(By.CSS_SELECTOR, '#description').send_keys(
                'Some description'
            )
            driver.find_element(By.CSS_SELECTOR, '#owner_name').send_keys(
                'Owner Name'
            )
            driver.find_element(By.CSS_SELECTOR, '#owner_email').send_keys(
                'owner@email.xyz'
            )
            driver.find_element(By.XPATH,
                                "//button[contains(., 'Create podcast')]"
            ).click()

            wait.until(EC.title_contains("${podcastTitle}"))

            driver.find_element(By.XPATH,
                                "//span[contains(., 'Add an episode')]"
            ).click()

            wait.until(EC.title_contains("Add an episode"))

            # upload podcast ###################################################

            driver.find_element(By.CSS_SELECTOR, '#audio_file').send_keys(
                '/tmp/podcast.mp3'
            )
            driver.find_element(By.CSS_SELECTOR, '#cover').send_keys(
                '${cover}'
            )
            driver.find_element(By.CSS_SELECTOR, '#description').send_keys(
                'Episode description'
            )
            driver.find_element(By.CSS_SELECTOR, '#title').send_keys(
                '${episodeTitle}'
            )
            driver.find_element(By.XPATH,
                                "//button[contains(., 'Create episode')]"
            ).click()

            wait.until(EC.title_contains("${episodeTitle}"))

            driver.close()
            driver.quit()
          '';
        in
      [ pkgs.firefox-unwrapped pkgs.geckodriver testRunner ];
        [
          pkgs.firefox-unwrapped
          pkgs.geckodriver
          browser-test
          (pkgs.writeShellApplication {
            name = "build-mp3";
            runtimeInputs = with pkgs; [ sox lame ];
            text = ''
              out=/tmp/podcast.mp3
              sox -n -r 48000 -t wav - synth ${targetPodcastDuration} sine 440 `
              `| lame --noreplaygain -cbr -q 9 -b 320 - $out
              FILESIZE="$(stat -c%s $out)"
              [ "$FILESIZE" -gt 0 ]
              [ "$FILESIZE" -le "${toString targetPodcastSize}" ]
            '';
          })
        ];
    };

  testScript = ''
    start_all()
    castopod.wait_for_unit("castopod-setup.service")
@@ -79,9 +223,11 @@ import ./make-test-python.nix ({ pkgs, lib, ... }:
    castopod.wait_for_unit("nginx.service")
    castopod.wait_for_open_port(80)
    castopod.wait_until_succeeds("curl -sS -f http://castopod.example.com")
    castopod.succeed("curl -s http://localhost/cp-install | grep 'Create your Super Admin account' > /dev/null")

    with subtest("Create superadmin and log in"):
        castopod.succeed("PYTHONUNBUFFERED=1 systemd-cat -t test-runner test-runner")
    client.succeed("build-mp3")

    with subtest("Create superadmin, log in, create and upload a podcast"):
        client.succeed(\
          "PYTHONUNBUFFERED=1 systemd-cat -t browser-test browser-test")
  '';
})
+8 −5
Original line number Diff line number Diff line
@@ -3,7 +3,7 @@
, ffmpeg-headless
, lib
, nixosTests
, stateDirectory ? "/var/lib/castopod"
, dataDir ? "/var/lib/castopod"
}:
stdenv.mkDerivation {
  pname = "castopod";
@@ -20,13 +20,16 @@ stdenv.mkDerivation {
  postPatch = ''
    # not configurable at runtime unfortunately:
    substituteInPlace app/Config/Paths.php \
      --replace "__DIR__ . '/../../writable'" "'${stateDirectory}/writable'"
      --replace "__DIR__ . '/../../writable'" "'${dataDir}/writable'"

    # configuration file must be writable, place it to ${stateDirectory}
    substituteInPlace modules/Admin/Controllers/DashboardController.php \
      --replace "disk_total_space('./')" "disk_total_space('${dataDir}')"

    # configuration file must be writable, place it to ${dataDir}
    substituteInPlace modules/Install/Controllers/InstallController.php \
      --replace "ROOTPATH" "'${stateDirectory}/'"
      --replace "ROOTPATH" "'${dataDir}/'"
    substituteInPlace public/index.php spark \
      --replace "DotEnv(ROOTPATH)" "DotEnv('${stateDirectory}')"
      --replace "DotEnv(ROOTPATH)" "DotEnv('${dataDir}')"

    # ffmpeg is required for Video Clips feature
    substituteInPlace modules/MediaClipper/VideoClipper.php \