WIP: WAITING FOR NIXPKS UPDATE

The point was initially to quickly test the release candidate and wait
for the official v2.

Unfortunately I've encountered some issues trying to get the package
working for the official version. So let's just roll with it for now...

hosts/nixos/porthos/secrets/pyload/credentials.age

@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 cKojmg nJbOfp0/wmFOZLzcWjoGB7wEB8e56aO1NntSmn5KomU
+/Vio4Z/t7IPJrdzdwUPidVH3wrouSkwRzNHP0T4z3x0
+-> ssh-ed25519 jPowng QXg/xqs7/VfkYQg3X77w4i53q64bL9oYeTxqb9NVhiQ
+sMHIXlmrIxtIr+s0X4lBqev/PPd3AKD5P7AP5K4NeJg
+--- gzTn+6+aa4Ptic1lsvSt+r3IEBysHrvMMIyONogMDF0
+<EFBFBD>ÏÂ<EFBFBD>Ë®UE_í</¯çQ·Ü+U“AГMÄÿ/kï×dAL/”úÕįÍoæ\XïEDÇÑfã\ièÄ‘½àpF„`#¬n4è–x1î<31>ûÞèDëàÂË5CéЦ&fòB»q${Gg…Aqˆ³@üVu!Cc…R\ªÖ¨

hosts/nixos/porthos/secrets/secrets.nix

@@ -62,6 +62,8 @@ in
 
   "podgrab/password.age".publicKeys = all;
 
+  "pyload/credentials.age".publicKeys = all;
+
   "sso/auth-key.age".publicKeys = all;
   "sso/ambroisie/password-hash.age".publicKeys = all;
   "sso/ambroisie/totp-secret.age".publicKeys = all;

hosts/nixos/porthos/services.nix

@@ -134,6 +134,10 @@ in
     };
     # Regular backups
     postgresql-backup.enable = true;
+    pyload = {
+      enable = true;
+      credentialsFile = secrets."pyload/credentials".path;
+    };
     # RSS provider for websites that do not provide any feeds
     rss-bridge.enable = true;
     # Usenet client

modules/nixos/services/default.nix

@@ -26,6 +26,7 @@
     ./podgrab
     ./postgresql
     ./postgresql-backup
+    ./pyload
     ./quassel
     ./rss-bridge
     ./sabnzbd

modules/nixos/services/pyload/default.nix

@@ -0,0 +1,66 @@
+{ config, lib, ... }:
+let
+  cfg = config.my.services.pyload;
+in
+{
+  imports = [
+    ./nixos.nix
+  ];
+
+  options.my.services.pyload = with lib; {
+    enable = mkEnableOption "pyload download manager";
+
+    credentialsFile = mkOption {
+      type = types.path;
+      example = "/run/secrets/pyload-credentials.env";
+      description = "pyload credentials";
+    };
+
+    downloadDirectory = mkOption {
+      type = types.str;
+      default = "/data/downloads";
+      example = "/var/lib/pyload/download";
+      description = "Download directory";
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 9093;
+      example = 8080;
+      description = "Internal port for webui";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    services.pyload = {
+      enable = true;
+
+      # Listening on `localhost` leads to 502 with the reverse proxy...
+      listenAddress = "127.0.0.1";
+
+      inherit (cfg)
+        credentialsFile
+        downloadDirectory
+        port
+        ;
+    };
+
+    # User media group when downloading files
+    systemd. services. pyload = {
+      serviceConfig = {
+        Group = lib.mkForce "media";
+      };
+    };
+
+    # Set-up media group
+    users.groups.media = { };
+
+    my.services.nginx.virtualHosts = [
+      {
+        subdomain = "pyload";
+        # FIXME: use actual port
+        inherit (cfg) port;
+      }
+    ];
+  };
+}

modules/nixos/services/pyload/nixos.nix

@@ -0,0 +1,144 @@
+{ config, lib, pkgs, ... }:
+let
+  cfg = config.services.pyload;
+
+  stateDir = "/var/lib/pyload";
+  userDir = "${stateDir}/config";
+in
+{
+  options = with lib; {
+    services.pyload = {
+      enable = mkEnableOption "pyload download manager";
+
+      package = mkPackageOption pkgs "pyload-ng" { };
+
+      listenAddress = mkOption {
+        type = types.str;
+        default = "localhost";
+        example = "0.0.0.0";
+        description = "Address to listen on for the web UI.";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 8000;
+        example = 9876;
+        description = "Port to listen on for the web UI.";
+      };
+
+      downloadDirectory = mkOption {
+        type = types.path;
+        default = "${stateDir}/downloads";
+        example = "/mnt/downloads";
+        description = "Directory to store downloads";
+      };
+
+      credentialsFile = mkOption {
+        type = with types; nullOr path;
+        default = null;
+        example = "/run/secrets/pyload-credentials.env";
+        description = ''
+          File containing PYLOAD_DEFAULT_USERNAME and PYLOAD_DEFAULT_PASSWORD
+          in the format of an EnvironmentFile=, as described by systemd.exec(5).
+
+          If not given, they default to the username/password combo of
+          pyload/pyload.
+        '';
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.tmpfiles.rules = [
+      "d ${cfg.downloadDirectory}"
+    ];
+
+    systemd.services.pyload = {
+      description = "pyload service";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+
+      # NOTE: unlike what the documentation says, it looks like `HOME` is not
+      # defined with this service definition...
+      # Since pyload tries to do the equivalent of `cd ~`, it needs to be able
+      # to resolve $HOME, which fails when `RootDirectory` is set.
+      # FIXME: check if `SetLoginEnvironment` fixes this issue in version 255
+      environment = {
+        HOME = stateDir;
+        PYLOAD__WEBUI__HOST = cfg.listenAddress;
+        PYLOAD__WEBUI__PORT = builtins.toString cfg.port;
+      };
+
+      serviceConfig = {
+        # FIXME: use getExe
+        ExecStart = "${lib.getExe' cfg.package "pyload"} ${lib.escapeShellArgs [
+          "--userdir" userDir
+          "--storagedir"  cfg.downloadDirectory
+        ]}";
+
+        User = "pyload";
+        Group = "pyload";
+        DynamicUser = true;
+
+        EnvironmentFile = lib.optional (cfg.credentialsFile != null) cfg.credentialsFile;
+
+        StateDirectory = "pyload";
+        WorkingDirectory = stateDir;
+        RuntimeDirectory = "pyload";
+        RuntimeDirectoryMode = "0700";
+        RootDirectory = "/run/pyload";
+        BindReadOnlyPaths = [
+          builtins.storeDir # Needed to run the python interpreter
+        ];
+        BindPaths = [
+          cfg.downloadDirectory
+        ];
+
+        # Hardening options
+        LockPersonality = true;
+        MemoryDenyWriteExecute = true;
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProcSubset = "pid";
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectProc = "invisible";
+        ProtectSystem = "strict";
+        RemoveIPC = true;
+        RestrictAddressFamilies = "AF_INET AF_INET6 AF_UNIX";
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [ "@system-service" "~@resources" "~@privileged" ];
+        UMask = "0002";
+        CapabilityBoundingSet = [
+          "~CAP_BLOCK_SUSPEND"
+          "~CAP_BPF"
+          "~CAP_CHOWN"
+          "~CAP_IPC_LOCK"
+          "~CAP_KILL"
+          "~CAP_LEASE"
+          "~CAP_LINUX_IMMUTABLE"
+          "~CAP_NET_ADMIN"
+          "~CAP_SYS_ADMIN"
+          "~CAP_SYS_BOOT"
+          "~CAP_SYS_CHROOT"
+          "~CAP_SYS_NICE"
+          "~CAP_SYS_PACCT"
+          "~CAP_SYS_PTRACE"
+          "~CAP_SYS_RESOURCE"
+          "~CAP_SYS_TTY_CONFIG"
+        ];
+      };
+    };
+  };
+}

overlays/pyload-declarative-user-management/declarative-config.patch

@@ -0,0 +1,18 @@
+diff --git a/src/pyload/core/__init__.py b/src/pyload/core/__init__.py
+index 4324fc700..5d915a85e 100644
+--- a/src/pyload/core/__init__.py
++++ b/src/pyload/core/__init__.py
+@@ -128,6 +128,13 @@ class Core:
+         else:
+             self._debug = max(0, int(debug))
+
++        # Allow setting any option declaratively, for the NixOS module
++        for env, value in os.environ.items():
++            if not env.startswith("PYLOAD__"):
++                continue
++            section, opt = env.removeprefix("PYLOAD__").lower().split("__")
++            self.config.set(section, opt, value)
++
+         # If no argument set, read storage dir from config file,
+         # otherwise save setting to config dir
+         if storagedir is None:

overlays/pyload-declarative-user-management/declarative-default-user.patch

@@ -0,0 +1,15 @@
+diff --git a/src/pyload/core/__init__.py b/src/pyload/core/__init__.py
+index 4324fc700..f7fcd66ec 100644
+--- a/src/pyload/core/__init__.py
++++ b/src/pyload/core/__init__.py
+@@ -46,8 +46,8 @@ class Exit(Exception):
+ #  improve external scripts
+ class Core:
+     LOCALE_DOMAIN = APPID
+-    DEFAULT_USERNAME = APPID
+-    DEFAULT_PASSWORD = APPID
++    DEFAULT_USERNAME = os.getenv("PYLOAD_DEFAULT_USERNAME", APPID)
++    DEFAULT_PASSWORD = os.getenv("PYLOAD_DEFAULT_PASSWORD", APPID)
+     DEFAULT_DATADIR = os.path.join(
+         os.getenv("APPDATA") or USERHOMEDIR, "pyLoad" if os.name == "nt" else ".pyload"
+     )

overlays/pyload-declarative-user-management/default.nix

@@ -0,0 +1,4 @@
+self: _super:
+{
+  pyload-ng = self.callPackage ./package.nix { };
+}

overlays/pyload-declarative-user-management/package.nix

@@ -0,0 +1,60 @@
+{ lib, fetchPypi, python3 }:
+
+python3.pkgs.buildPythonApplication rec {
+  version = "0.5.0b3.dev75";
+  pname = "pyload-ng";
+  pyproject = true;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-1lPIKkZESonDaVCnac0iUu/gCqXVDBhNZrk5S0eC6F0=";
+  };
+
+  patches = [
+    # Makes it possible to change the default username/password in the module
+    ./declarative-default-user.patch
+    ./declarative-config.patch
+  ];
+
+  postPatch = ''
+    # relax version bounds
+    sed -i 's/\([A-z0-9]*\)~=.*$/\1/' setup.cfg
+    # not sure what Flask-Session2 is but flask-session works just fine
+    sed -i '/Flask-Session2/d' setup.cfg
+  '';
+
+  propagatedBuildInputs = with python3.pkgs; [
+    bitmath
+    certifi
+    cheroot
+    cryptography
+    filetype
+    flask
+    flask-babel
+    flask-caching
+    flask-compress
+    flask-session
+    flask-themes2
+    js2py
+    pycurl
+    semver
+    setuptools
+  ];
+
+  passthru.optional-dependencies = {
+    plugins = with python3.pkgs; [
+      beautifulsoup4 # for some plugins
+      colorlog # colorful console logging
+      pillow # for some CAPTCHA plugin
+      send2trash # send some files to trash instead of deleting them
+      slixmpp # XMPP plugin
+    ];
+  };
+
+  meta = with lib; {
+    description = "Free and open-source download manager with support for 1-click-hosting sites";
+    homepage = "https://github.com/pyload/pyload";
+    license = licenses.agpl3Plus;
+    maintainers = with maintainers; [ ruby0b ];
+  };
+}

overlays/woodpecker-2.0.0-rc.0/agent.nix

@@ -0,0 +1,17 @@
+{ buildGoModule, callPackage }:
+let
+  common = callPackage ./common.nix { };
+in
+buildGoModule {
+  pname = "woodpecker-agent";
+  inherit (common) version src ldflags postInstall vendorHash;
+
+  subPackages = "cmd/agent";
+
+  CGO_ENABLED = 0;
+
+  meta = common.meta // {
+    description = "Woodpecker Continuous Integration agent";
+    mainProgram = "woodpecker-agent";
+  };
+}

overlays/woodpecker-2.0.0-rc.0/cli.nix

@@ -0,0 +1,17 @@
+{ buildGoModule, callPackage }:
+let
+  common = callPackage ./common.nix { };
+in
+buildGoModule {
+  pname = "woodpecker-cli";
+  inherit (common) version src ldflags postInstall vendorHash;
+
+  subPackages = "cmd/cli";
+
+  CGO_ENABLED = 0;
+
+  meta = common.meta // {
+    description = "Command line client for the Woodpecker Continuous Integration server";
+    mainProgram = "woodpecker-cli";
+  };
+}

overlays/woodpecker-2.0.0-rc.0/common.nix

@@ -0,0 +1,38 @@
+{ lib, fetchFromGitHub }:
+let
+  version = "2.0.0-rc.0";
+  srcHash = "sha256-n4rJNBgOQV4rKoFirHW63q520HF8PYBmtwx7iEmht+E=";
+  vendorHash = "sha256-/GOwsrYzVoYjJZ27sp/gzp1N3de6P16v+kHY1u1wTKs=";
+  yarnHash = "sha256-IZbhn95wBCeh0DJDe+S/9Vqx4CHAQ+4QA5jp6xc9hmY=";
+in
+{
+  inherit version yarnHash vendorHash;
+
+  src = fetchFromGitHub {
+    owner = "woodpecker-ci";
+    repo = "woodpecker";
+    rev = "v${version}";
+    hash = srcHash;
+  };
+
+  postInstall = ''
+    cd $out/bin
+    for f in *; do
+      mv -- "$f" "woodpecker-$f"
+    done
+    cd -
+  '';
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X go.woodpecker-ci.org/woodpecker/version.Version=${version}"
+  ];
+
+  meta = with lib; {
+    homepage = "https://woodpecker-ci.org/";
+    changelog = "https://github.com/woodpecker-ci/woodpecker/blob/v${version}/CHANGELOG.md";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ ambroisie techknowlogick adamcstephens ];
+  };
+}

overlays/woodpecker-2.0.0-rc.0/default.nix

@@ -0,0 +1,10 @@
+self: _super:
+{
+  woodpecker-agent = self.callPackage ./agent.nix { };
+
+  woodpecker-cli = self.callPackage ./cli.nix { };
+
+  woodpecker-server = self.callPackage ./server.nix {
+    woodpecker-frontend = self.callPackage ./frontend.nix { };
+  };
+}

overlays/woodpecker-2.0.0-rc.0/frontend.nix

@@ -0,0 +1,63 @@
+{ lib, buildPackages, callPackage, fetchFromGitHub, fetchYarnDeps, mkYarnPackage }:
+let
+  common = callPackage ./common.nix { };
+
+  esbuild_0_18_20 = buildPackages.esbuild.overrideAttrs (_: rec {
+    version = "0.18.20";
+
+    src = fetchFromGitHub {
+      owner = "evanw";
+      repo = "esbuild";
+      rev = "v${version}";
+      hash = "sha256-mED3h+mY+4H465m02ewFK/BgA1i/PQ+ksUNxBlgpUoI=";
+    };
+
+    vendorHash = "sha256-+BfxCyg0KkDQpHt/wycy/8CTG6YBA/VJvJFhhzUnSiQ=";
+  });
+in
+mkYarnPackage {
+  pname = "woodpecker-frontend";
+  inherit (common) version;
+
+  src = "${common.src}/web";
+
+  packageJSON = ./woodpecker-package.json;
+  yarnLock = ./yarn.lock;
+
+  offlineCache = fetchYarnDeps {
+    yarnLock = ./yarn.lock;
+    hash = common.yarnHash;
+  };
+
+  ESBUILD_BINARY_PATH = lib.getExe esbuild_0_18_20;
+
+  postPatch = ''
+    substituteInPlace vite.config.ts \
+      --replace 'src/' '/build/web/deps/woodpecker-ci/src/' \
+      --replace 'node_modules/' '/build/web/node_modules/'
+  '';
+
+  buildPhase = ''
+    runHook preBuild
+
+    yarn --offline build
+
+    runHook postBuild
+  '';
+
+  installPhase = ''
+    runHook preInstall
+
+    cp -R deps/woodpecker-ci/dist $out
+    echo "${common.version}" > "$out/version"
+
+    runHook postInstall
+  '';
+
+  # Do not attempt generating a tarball for woodpecker-frontend again.
+  doDist = false;
+
+  meta = common.meta // {
+    description = "Woodpecker Continuous Integration server frontend";
+  };
+}

overlays/woodpecker-2.0.0-rc.0/server.nix

@@ -0,0 +1,27 @@
+{ buildGoModule, callPackage, woodpecker-frontend }:
+let
+  common = callPackage ./common.nix { };
+in
+buildGoModule {
+  pname = "woodpecker-server";
+  inherit (common) version src ldflags postInstall vendorHash;
+
+  postPatch = ''
+    cp -r ${woodpecker-frontend} web/dist
+  '';
+
+  subPackages = "cmd/server";
+
+  CGO_ENABLED = 1;
+
+  passthru = {
+    inherit woodpecker-frontend;
+
+    updateScript = ./update.sh;
+  };
+
+  meta = common.meta // {
+    description = "Woodpecker Continuous Integration server";
+    mainProgram = "woodpecker-server";
+  };
+}

overlays/woodpecker-2.0.0-rc.0/update.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p nix wget prefetch-yarn-deps nix-prefetch-github jq nix-prefetch pnpm-lock-export
+
+# shellcheck shell=bash
+
+if [ -n "$GITHUB_TOKEN" ]; then
+    TOKEN_ARGS=(--header "Authorization: token $GITHUB_TOKEN")
+fi
+
+if [[ $# -gt 1 || $1 == -* ]]; then
+    echo "Regenerates packaging data for the woodpecker packages."
+    echo "Usage: $0 [git release tag]"
+    exit 1
+fi
+
+set -x
+
+cd "$(dirname "$0")"
+version="$1"
+
+set -euo pipefail
+
+if [ -z "$version" ]; then
+    version="$(wget -O- "${TOKEN_ARGS[@]}" "https://api.github.com/repos/woodpecker-ci/woodpecker/releases?per_page=1" | jq -r '.[0].tag_name')"
+fi
+
+# strip leading "v"
+version="${version#v}"
+sed -i -E -e "s#version = \".*\"#version = \"$version\"#" common.nix
+
+# Woodpecker repository
+src_hash=$(nix-prefetch-github woodpecker-ci woodpecker --rev "v${version}" | jq -r .hash)
+src_hash=$(nix hash to-sri --type sha256 "$src_hash")
+sed -i -E -e "s#srcHash = \".*\"#srcHash = \"$src_hash\"#" common.nix
+
+# Go modules
+vendorHash=$(nix-prefetch '{ sha256 }: (callPackage (import ./cli.nix) { }).goModules.overrideAttrs (_: { modHash = sha256; })')
+vendorHash=$(nix hash to-sri --type sha256 "$vendorHash")
+sed -i -E -e "s#vendorHash = \".*\"#vendorHash = \"$vendorHash\"#" common.nix
+
+# Front-end dependencies
+woodpecker_src="https://raw.githubusercontent.com/woodpecker-ci/woodpecker/v$version"
+wget "${TOKEN_ARGS[@]}" "$woodpecker_src/web/package.json" -O woodpecker-package.json
+
+trap 'rm -rf pnpm-lock.yaml' EXIT
+wget "${TOKEN_ARGS[@]}" "$woodpecker_src/web/pnpm-lock.yaml"
+pnpm-lock-export --schema yarn.lock@v1
+yarn_hash=$(prefetch-yarn-deps yarn.lock)
+yarn_hash=$(nix hash to-sri --type sha256 "$yarn_hash")
+sed -i -E -e "s#yarnHash = \".*\"#yarnHash = \"$yarn_hash\"#" common.nix

overlays/woodpecker-2.0.0-rc.0/woodpecker-package.json

@@ -0,0 +1,76 @@
+{
+  "name": "woodpecker-ci",
+  "author": "Woodpecker CI",
+  "version": "0.0.0",
+  "license": "Apache-2.0",
+  "engines": {
+    "node": ">=14"
+  },
+  "scripts": {
+    "start": "vite",
+    "build": "vite build --base=/BASE_PATH",
+    "serve": "vite preview",
+    "lint": "eslint --max-warnings 0 --ext .js,.ts,.vue,.json .",
+    "format": "prettier --write .",
+    "format:check": "prettier -c .",
+    "typecheck": "vue-tsc --noEmit",
+    "test": "echo 'No tests configured' && exit 0"
+  },
+  "dependencies": {
+    "@intlify/unplugin-vue-i18n": "^1.0.0",
+    "@kyvg/vue3-notification": "^3.0.0",
+    "@vueuse/core": "^10.5.0",
+    "ansi_up": "^6.0.0",
+    "dayjs": "^1.11.9",
+    "fuse.js": "^7.0.0",
+    "humanize-duration": "^3.28.0",
+    "javascript-time-ago": "^2.5.9",
+    "lodash": "^4.17.21",
+    "node-emoji": "^2.0.0",
+    "pinia": "^2.1.4",
+    "prismjs": "^1.29.0",
+    "vue": "^3.3.4",
+    "vue-i18n": "^9.2.2",
+    "vue-router": "^4.2.2"
+  },
+  "devDependencies": {
+    "@iconify/json": "^2.2.131",
+    "@types/humanize-duration": "^3.27.1",
+    "@types/javascript-time-ago": "^2.0.3",
+    "@types/lodash": "^4.14.195",
+    "@types/node": "^20.0.0",
+    "@types/node-emoji": "^1.8.2",
+    "@types/prismjs": "^1.26.0",
+    "@typescript-eslint/eslint-plugin": "^6.8.0",
+    "@typescript-eslint/parser": "^6.8.0",
+    "@vitejs/plugin-vue": "^4.2.3",
+    "@vue/compiler-sfc": "^3.3.6",
+    "eslint": "^8.44.0",
+    "eslint-config-airbnb-base": "^15.0.0",
+    "eslint-config-airbnb-typescript": "^17.0.0",
+    "eslint-config-prettier": "^9.0.0",
+    "eslint-plugin-import": "^2.29.0",
+    "eslint-plugin-prettier": "^5.0.0",
+    "eslint-plugin-promise": "^6.1.1",
+    "eslint-plugin-simple-import-sort": "^10.0.0",
+    "eslint-plugin-vue": "^9.17.0",
+    "eslint-plugin-vue-scoped-css": "^2.5.0",
+    "prettier": "^3.0.0",
+    "tinycolor2": "^1.6.0",
+    "typescript": "5.2.2",
+    "unplugin-icons": "^0.17.0",
+    "unplugin-vue-components": "^0.25.0",
+    "vite": "^4.3.9",
+    "vite-plugin-prismjs": "^0.0.8",
+    "vite-plugin-windicss": "^1.9.0",
+    "vite-svg-loader": "^4.0.0",
+    "vue-eslint-parser": "^9.3.1",
+    "vue-tsc": "^1.8.19",
+    "windicss": "^3.5.6"
+  },
+  "pnpm": {
+    "overrides": {
+      "semver@<7.5.2": ">=7.5.2"
+    }
+  }
+}