From 30b2f192e4ad6a55fcda27cd8d27e5671aa417d9 Mon Sep 17 00:00:00 2001 From: Gabriel Arazas Date: Thu, 22 Aug 2024 19:15:48 +0800 Subject: [PATCH] hosts/ni/services/download-media: add wrapper-manager and update config --- configs/flake-parts/nixos.nix | 1 + .../download-media/data/jobs.schema.json | 14 ++++- .../services/download-media/default.nix | 62 +++++++++++-------- 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/configs/flake-parts/nixos.nix b/configs/flake-parts/nixos.nix index bca7f256..111e5763 100644 --- a/configs/flake-parts/nixos.nix +++ b/configs/flake-parts/nixos.nix @@ -16,6 +16,7 @@ modules = [ inputs.disko.nixosModules.disko inputs.sops-nix.nixosModules.sops + inputs.self.nixosModules.wrapper-manager ]; home-manager = { branch = "home-manager-unstable"; diff --git a/configs/nixos/ni/modules/services/download-media/data/jobs.schema.json b/configs/nixos/ni/modules/services/download-media/data/jobs.schema.json index 87f32058..0c91ea01 100644 --- a/configs/nixos/ni/modules/services/download-media/data/jobs.schema.json +++ b/configs/nixos/ni/modules/services/download-media/data/jobs.schema.json @@ -9,12 +9,17 @@ "required": true, "properties": { "extraArgs": { + "$comment": "Extra arguments to be passed to the associated service.", "type": "array", "items": { "type": "string" }, "uniqueItems": true }, + "downloadPath": { + "$comment": "Job-specific download path of the associated service.", + "type": "string" + }, "subscriptions": { "$comment": "While it is easy to think this could be an object, some exports and applications allow the data to have the same name but points to different URLs. For example, NewPipe has support for multiple services other than YouTube which the same creator could have accounts on multiple platforms. Overriding it would be troublesome in case I want to follow the same creator on multiple platforms.", "type": "array", @@ -28,11 +33,16 @@ "type": "string" } }, - "required": [ "name", "url" ] + "required": [ + "name", + "url" + ] } } }, - "required": [ "subscriptions" ] + "required": [ + "subscriptions" + ] } } } diff --git a/configs/nixos/ni/modules/services/download-media/default.nix b/configs/nixos/ni/modules/services/download-media/default.nix index 38f7fa4b..2d9fb646 100644 --- a/configs/nixos/ni/modules/services/download-media/default.nix +++ b/configs/nixos/ni/modules/services/download-media/default.nix @@ -29,8 +29,7 @@ in # Write the subtitle file with the preferred languages. "--write-subs" - "--sub-langs" - "en.*,ja,ko,zh.*,fr,pt.*" + "--sub-langs" "en.*,ja,ko,zh.*,fr,pt.*" # Write the description in a separate file. "--write-description" @@ -44,15 +43,12 @@ in "(webm,mkv,mp4)[height<=?1280]" # Prefer MKV whenever possible for video formats. - "--merge-output-format" - "mkv" + "--merge-output-format" "mkv" # Don't download any videos that are originally live streams. - "--match-filters" - "!was_live" + "--match-filters" "!was_live" - "--audio-quality" - "1" + "--audio-quality" "1" # Not much error since it will always fail. "--no-abort-on-error" @@ -60,9 +56,17 @@ in "--ignore-no-formats-error" ]; - ytdlpArchiveVariant = pkgs.writeScriptBin "yt-dlp-archive-variant" '' - ${pkgs.yt-dlp}/bin/yt-dlp ${lib.escapeShellArgs ytdlpArgs} $@ - ''; + galleryDlArgs = [ + # Write metadata to separate JSON files. + "--write-metadata" + + # The config file that contains the secrets for various services. + # We're putting as a separate config file instead of configuring it + # in the service properly since secrets decrypted by sops-nix cannot + # be read in Nix. + "--config" + "${config.sops.secrets."${pathPrefix}/secrets-config".path}" + ]; # Given an attribute set of jobs that contains a list of objects with # their names and URL, create an attrset suitable for declaring the @@ -93,25 +97,27 @@ in lib.listToAttrs jobsList; in { - environment.systemPackages = [ ytdlpArchiveVariant ]; - sops.secrets = foodogsquaredLib.sops-nix.getSecrets ./secrets.yaml (lib.attachSopsPathPrefix pathPrefix { "secrets-config" = { }; }); + # This is to make an exception for Archivebox. + nixpkgs.config.permittedInsecurePackages = [ + "python3.12-django-3.1.14" + ]; + suites.filesystem.setups.archive.enable = true; services.yt-dlp = { enable = true; - archivePath = "${mountName}/yt-dlp-service"; + downloadPath = "${mountName}/yt-dlp-service"; # This is applied on all jobs. It is best to be minimal as much as # possible for this. extraArgs = ytdlpArgs ++ [ # Make a global list of successfully downloaded videos as a cache for yt-dlp. - "--download-archive" - "${config.services.yt-dlp.archivePath}/videos" + "--download-archive" "videos" ]; jobs = mkJobs { @@ -150,20 +156,10 @@ in enable = true; archivePath = "${mountName}/gallery-dl-service"; - extraArgs = [ + extraArgs = galleryDlArgs ++ [ # Record all downloaded files in an archive file. "--download-archive" "${config.services.gallery-dl.archivePath}/photos" - - # Write metadata to separate JSON files. - "--write-metadata" - - # The config file that contains the secrets for various services. - # We're putting as a separate config file instead of configuring it - # in the service properly since secrets decrypted by sops-nix cannot - # be read in Nix. - "--config" - "${config.sops.secrets."${pathPrefix}/secrets-config".path}" ]; settings.extractor = { @@ -203,6 +199,18 @@ in }; }; }; + + wrapper-manager.packages.download-media-variants = { + wrappers."yt-dlp-${pathPrefix}" = { + arg0 = lib.getExe' config.services.yt-dlp.package "yt-dlp"; + prependArgs = ytdlpArgs; + }; + + wrappers."gallery-dl-${pathPrefix}" = { + arg0 = lib.getExe' config.services.gallery-dl.package "gallery-dl"; + prependArgs = galleryDlArgs; + }; + }; } ); }