From 41f4cdcd7c0a6535df456754788047ddb8779926 Mon Sep 17 00:00:00 2001 From: Gabriel Arazas Date: Sat, 20 Jul 2024 13:33:51 +0800 Subject: [PATCH] wrapper-manager-fds/docs: update build step for Hugo site For now, there is a broken build hook for building Hugo sites but we'll figure it out. --- docs/default.nix | 47 ++++-- docs/hugo-build-module.nix | 318 +++++++++++++++++++++++++++++++++++++ npins/sources.json | 12 ++ 3 files changed, 364 insertions(+), 13 deletions(-) create mode 100644 docs/hugo-build-module.nix diff --git a/docs/default.nix b/docs/default.nix index 6e5f512..bf3a287 100644 --- a/docs/default.nix +++ b/docs/default.nix @@ -1,3 +1,9 @@ +# I forgot about the fact Hugo also uses Go modules for its Hugo modules +# feature. For now, this is considered broken up until that is working. Also, +# Hugo has several features such as embedding metadata from VCS which doesn't +# play well with Nix that is requiring a clean source. +# +# For now, we're just relying on nix-shell to build it for us. let sources = import ../npins; in @@ -5,6 +11,7 @@ in let inherit (pkgs) nixosOptionsDoc stdenv lib; + buildHugoSite = pkgs.callPackage ./hugo-build-module.nix { }; wrapperManagerLib = import ../lib/env.nix; wrapperManagerEval = wrapperManagerLib.eval { inherit pkgs; }; @@ -16,8 +23,24 @@ let ruby = pkgs.ruby_3_1; gemdir = ./.; }; + + # Now this is some dogfooding. + asciidoctorWrapped = + wrapperManagerLib.build { + inherit pkgs; + modules = [ + ({ config, lib, pkgs, ... }: { + wrappers.asciidoctor = { + arg0 = lib.getExe' gems "asciidoctor"; + appendArgs = [ + "-T" "${sources.website}/templates" + ]; + }; + }) + ]; + }; in -stdenv.mkDerivation (finalAttrs: { +buildHugoSite { pname = "wrapper-manager-docs"; version = "2024-07-13"; @@ -28,27 +51,25 @@ stdenv.mkDerivation (finalAttrs: { ./config ./content ./layouts + ./go.mod + ./go.sum ]; }; + vendorHash = "sha256-vMLi8of2eF/s60B/lM3FDfSntEyieGkvJbTSMuI7Wws="; + buildInputs = with pkgs; [ + asciidoctorWrapped hugo - go git gems gems.wrappedRuby ]; - patchPhase = '' - runHook prePatch - install -Dm0644 "${optionsDoc.optionsJSON}/share/doc/nixos/options.json" "${finalAttrs.src}/content/en-US/nix-module-options/module-environment/content.json" - runHook postPatch - ''; - - buildPhase = '' - runHook preBuild - hugo build - runHook postBuild + installPhase = '' + runHook preInstall + cp --reflink=auto "$src/public" "$out" + runHook postInstall ''; meta = with lib; { @@ -60,4 +81,4 @@ stdenv.mkDerivation (finalAttrs: { ]; platforms = platforms.all; }; -}) +} diff --git a/docs/hugo-build-module.nix b/docs/hugo-build-module.nix new file mode 100644 index 0000000..1226d2a --- /dev/null +++ b/docs/hugo-build-module.nix @@ -0,0 +1,318 @@ +{ hugo, go, cacert, git, lib, stdenv }: + +{ name ? "${args'.pname}-${args'.version}" +, src +, nativeBuildInputs ? [ ] +, passthru ? { } +, patches ? [ ] + + # A function to override the goModules derivation +, overrideModAttrs ? (_oldAttrs: { }) + + # path to go.mod and go.sum directory +, modRoot ? "./" + + # vendorHash is the SRI hash of the vendored dependencies + # + # if vendorHash is null, then we won't fetch any dependencies and + # rely on the vendor folder within the source. +, vendorHash ? throw ( + if args'?vendorSha256 then + "buildGoModule: Expect vendorHash instead of vendorSha256" + else + "buildGoModule: vendorHash is missing" + ) + # Whether to delete the vendor folder supplied with the source. +, deleteVendor ? false + # Whether to fetch (go mod download) and proxy the vendor directory. + # This is useful if your code depends on c code and go mod tidy does not + # include the needed sources to build or if any dependency has case-insensitive + # conflicts which will produce platform dependant `vendorHash` checksums. +, proxyVendor ? false + + # We want parallel builds by default +, enableParallelBuilding ? true + + # Do not enable this without good reason + # IE: programs coupled with the compiler +, allowGoReference ? false + +, CGO_ENABLED ? go.CGO_ENABLED + +, meta ? { } + + # Not needed with buildGoModule +, goPackagePath ? "" + +, ldflags ? [ ] + +, GOFLAGS ? [ ] + + # needed for buildFlags{,Array} warning +, buildFlags ? "" +, buildFlagsArray ? "" + +, ... +}@args': + +assert goPackagePath != "" -> throw "`goPackagePath` is not needed with `buildGoModule`"; + +let + args = removeAttrs args' [ "overrideModAttrs" "vendorSha256" "vendorHash" ]; + + GO111MODULE = "on"; + GOTOOLCHAIN = "local"; + + hugoModules = if (vendorHash == null) then "" else + (stdenv.mkDerivation { + name = "${name}-hugo-modules"; + + nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ hugo go git cacert ]; + + inherit (args) src; + inherit (go) GOOS GOARCH; + inherit GO111MODULE GOTOOLCHAIN; + + # The following inheritence behavior is not trivial to expect, and some may + # argue it's not ideal. Changing it may break vendor hashes in Nixpkgs and + # out in the wild. In anycase, it's documented in: + # doc/languages-frameworks/go.section.md + prePatch = args.prePatch or ""; + patches = args.patches or [ ]; + patchFlags = args.patchFlags or [ ]; + postPatch = args.postPatch or ""; + preBuild = args.preBuild or ""; + postBuild = args.modPostBuild or ""; + sourceRoot = args.sourceRoot or ""; + setSourceRoot = args.setSourceRoot or ""; + env = args.env or { }; + + impureEnvVars = lib.fetchers.proxyImpureEnvVars ++ [ + "GIT_PROXY_COMMAND" + "SOCKS_SERVER" + "GOPROXY" + ]; + + configurePhase = args.modConfigurePhase or '' + runHook preConfigure + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + cd "${modRoot}" + runHook postConfigure + ''; + + buildPhase = args.modBuildPhase or ('' + runHook preBuild + '' + lib.optionalString deleteVendor '' + if [ ! -d _vendor ]; then + echo "_vendor folder does not exist, 'deleteVendor' is not needed" + exit 10 + else + rm -rf _vendor + fi + '' + '' + if [ -d _vendor ]; then + echo "_vendor folder exists, please set 'vendorHash = null;' in your expression" + exit 10 + fi + + ${if proxyVendor then '' + mkdir -p "''${GOPATH}/pkg/mod/cache/download" + hugo mod vendor + '' else '' + if (( "''${NIX_DEBUG:-0}" >= 1 )); then + hugoModVendorFlags+=(-v) + fi + hugo mod vendor "''${hugoModVendorFlags[@]}" + ''} + + mkdir -p _vendor + + runHook postBuild + ''); + + installPhase = args.modInstallPhase or '' + runHook preInstall + + ${if proxyVendor then '' + rm -rf "''${GOPATH}/pkg/mod/cache/download/sumdb" + cp -r --reflink=auto "''${GOPATH}/pkg/mod/cache/download" $out + '' else '' + cp -r --reflink=auto _vendor $out + ''} + + if ! [ "$(ls -A $out)" ]; then + echo "_vendor folder is empty, please set 'vendorHash = null;' in your expression" + exit 10 + fi + + runHook postInstall + ''; + + dontFixup = true; + + outputHashMode = "recursive"; + outputHash = vendorHash; + # Handle empty vendorHash; avoid + # error: empty hash requires explicit hash algorithm + outputHashAlgo = if vendorHash == "" then "sha256" else null; + }).overrideAttrs overrideModAttrs; + + package = stdenv.mkDerivation (args // { + nativeBuildInputs = [ hugo git go ] ++ nativeBuildInputs; + + inherit (go) GOOS GOARCH; + + GOFLAGS = GOFLAGS + ++ lib.warnIf (lib.any (lib.hasPrefix "-mod=") GOFLAGS) "use `proxyVendor` to control Go module/vendor behavior instead of setting `-mod=` in GOFLAGS" + (lib.optional (!proxyVendor) "-mod=vendor") + ++ lib.warnIf (builtins.elem "-trimpath" GOFLAGS) "`-trimpath` is added by default to GOFLAGS by buildGoModule when allowGoReference isn't set to true" + (lib.optional (!allowGoReference) "-trimpath"); + inherit CGO_ENABLED enableParallelBuilding GO111MODULE GOTOOLCHAIN; + + # If not set to an explicit value, set the buildid empty for reproducibility. + ldflags = ldflags ++ lib.optional (!lib.any (lib.hasPrefix "-buildid=") ldflags) "-buildid="; + + configurePhase = args.configurePhase or ('' + runHook preConfigure + + export GOCACHE=$TMPDIR/go-cache + export GOPATH="$TMPDIR/go" + export GOPROXY=off + export GOSUMDB=off + cd "$modRoot" + '' + lib.optionalString (vendorHash != null) '' + ${if proxyVendor then '' + export GOPROXY=file://${hugoModules} + '' else '' + rm -rf _vendor + cp -r --reflink=auto ${hugoModules} _vendor + ''} + '' + '' + + # currently pie is only enabled by default in pkgsMusl + # this will respect the `hardening{Disable,Enable}` flags if set + if [[ $NIX_HARDENING_ENABLE =~ "pie" ]]; then + export GOFLAGS="-buildmode=pie $GOFLAGS" + fi + + runHook postConfigure + ''); + + buildPhase = args.buildPhase or ( + lib.warnIf (buildFlags != "" || buildFlagsArray != "") + "`buildFlags`/`buildFlagsArray` are deprecated and will be removed in the 24.11 release. Use the `ldflags` and/or `tags` attributes instead of `buildFlags`/`buildFlagsArray`" + lib.warnIf (builtins.elem "-buildid=" ldflags) + "`-buildid=` is set by default as ldflag by buildGoModule" + '' + runHook preBuild + + exclude='\(/_\|examples\|Godeps\|testdata' + if [[ -n "$excludedPackages" ]]; then + IFS=' ' read -r -a excludedArr <<<$excludedPackages + printf -v excludedAlternates '%s\\|' "''${excludedArr[@]}" + excludedAlternates=''${excludedAlternates%\\|} # drop final \| added by printf + exclude+='\|'"$excludedAlternates" + fi + exclude+='\)' + + buildGoDir() { + local cmd="$1" dir="$2" + + declare -ga buildFlagsArray + declare -a flags + flags+=($buildFlags "''${buildFlagsArray[@]}") + flags+=(''${tags:+-tags=''${tags// /,}}) + flags+=(''${ldflags:+-ldflags="$ldflags"}) + flags+=("-p" "$NIX_BUILD_CORES") + + if [ "$cmd" = "test" ]; then + flags+=(-vet=off) + flags+=($checkFlags) + fi + + local OUT + if ! OUT="$(go $cmd "''${flags[@]}" $dir 2>&1)"; then + if ! echo "$OUT" | grep -qE '(no( buildable| non-test)?|build constraints exclude all) Go (source )?files'; then + echo "$OUT" >&2 + return 1 + fi + fi + if [ -n "$OUT" ]; then + echo "$OUT" >&2 + fi + return 0 + } + + getGoDirs() { + local type; + type="$1" + if [ -n "$subPackages" ]; then + echo "$subPackages" | sed "s,\(^\| \),\1./,g" + else + find . -type f -name \*$type.go -exec dirname {} \; | grep -v "/_vendor/" | sort --unique | grep -v "$exclude" + fi + } + + if (( "''${NIX_DEBUG:-0}" >= 1 )); then + buildFlagsArray+=(-x) + fi + + if [ -z "$enableParallelBuilding" ]; then + export NIX_BUILD_CORES=1 + fi + for pkg in $(getGoDirs ""); do + echo "Building subPackage $pkg" + buildGoDir install "$pkg" + done + '' + lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' + # normalize cross-compiled builds w.r.t. native builds + ( + dir=$GOPATH/bin/${go.GOOS}_${go.GOARCH} + if [[ -n "$(shopt -s nullglob; echo $dir/*)" ]]; then + mv $dir/* $dir/.. + fi + if [[ -d $dir ]]; then + rmdir $dir + fi + ) + '' + '' + runHook postBuild + ''); + + doCheck = args.doCheck or true; + checkPhase = args.checkPhase or '' + runHook preCheck + # We do not set trimpath for tests, in case they reference test assets + export GOFLAGS=''${GOFLAGS//-trimpath/} + + for pkg in $(getGoDirs test); do + buildGoDir test "$pkg" + done + + runHook postCheck + ''; + + installPhase = args.installPhase or '' + runHook preInstall + + mkdir -p $out + dir="$GOPATH/bin" + [ -e "$dir" ] && cp -r $dir $out + + runHook postInstall + ''; + + strictDeps = true; + + disallowedReferences = lib.optional (!allowGoReference) go; + + passthru = passthru // { inherit go hugo hugoModules vendorHash; }; + + meta = { + # Add default meta information + platforms = go.meta.platforms or lib.platforms.all; + } // meta; + }); +in +package diff --git a/npins/sources.json b/npins/sources.json index ad8521f..5dc591c 100644 --- a/npins/sources.json +++ b/npins/sources.json @@ -47,6 +47,18 @@ "revision": "693bc46d169f5af9c992095736e82c3488bf7dbb", "url": "https://github.com/NixOS/nixpkgs/archive/693bc46d169f5af9c992095736e82c3488bf7dbb.tar.gz", "hash": "0q3shwwdfkml1nxshd5mfmvpla7lbs986qfw9jz4d3193kcn3rx1" + }, + "website": { + "type": "Git", + "repository": { + "type": "GitHub", + "owner": "foo-dogsquared", + "repo": "website" + }, + "branch": "master", + "revision": "799cfa313e36eaba67862b872a8b350d50ec3617", + "url": "https://github.com/foo-dogsquared/website/archive/799cfa313e36eaba67862b872a8b350d50ec3617.tar.gz", + "hash": "0kcmcsrfs0fxx9pyx7dnpyk2wg1r6kdb0vrl8b3vlhcirz0nlmlx" } }, "version": 3