bring back compress-drv tests
This commit is contained in:
parent
9ec4e18bab
commit
e4eeaa15f2
11
flake.nix
11
flake.nix
|
|
@ -104,7 +104,8 @@
|
||||||
deploy-rs-pkg = null;
|
deploy-rs-pkg = null;
|
||||||
})
|
})
|
||||||
(_: super: {
|
(_: super: {
|
||||||
inherit (super.callPackage ./pkgs/compress-drv.nix {}) compressDrvWeb;
|
compressDrv = super.callPackage ./pkgs/compress-drv {};
|
||||||
|
compressDrvWeb = super.callPackage ./pkgs/compress-drv/web.nix {};
|
||||||
|
|
||||||
tmuxbash = super.callPackage ./pkgs/tmuxbash.nix {};
|
tmuxbash = super.callPackage ./pkgs/tmuxbash.nix {};
|
||||||
btrfs-auto-snapshot = super.callPackage ./pkgs/btrfs-auto-snapshot.nix {};
|
btrfs-auto-snapshot = super.callPackage ./pkgs/btrfs-auto-snapshot.nix {};
|
||||||
|
|
@ -357,10 +358,10 @@
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
#compress-drv-tests = let
|
compress-drv-test = let
|
||||||
# pkgs = import nixpkgs {inherit system;};
|
pkgs = import nixpkgs {inherit system overlays;};
|
||||||
#in
|
in
|
||||||
# pkgs.callPackage ./pkgs/compress-drv-tests.nix {};
|
pkgs.callPackage ./pkgs/compress-drv/test.nix {};
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
deploy-rs.lib;
|
deploy-rs.lib;
|
||||||
|
|
|
||||||
|
|
@ -1,184 +0,0 @@
|
||||||
{
|
|
||||||
lib,
|
|
||||||
runCommand,
|
|
||||||
xorg,
|
|
||||||
zopfli,
|
|
||||||
brotli,
|
|
||||||
}: let
|
|
||||||
compressDrv = drv: {
|
|
||||||
formats,
|
|
||||||
compressors,
|
|
||||||
...
|
|
||||||
} @ args: let
|
|
||||||
compressorMap = lib.filterAttrs (k: _: (lib.hasPrefix "compressor-" k)) args;
|
|
||||||
compressCommands =
|
|
||||||
map
|
|
||||||
(ext: let
|
|
||||||
prog = builtins.getAttr "compressor-${ext}" compressorMap;
|
|
||||||
in "tee >(xargs -I{} -n1 -P$NIX_BUILD_CORES ${prog})")
|
|
||||||
compressors;
|
|
||||||
formatsbar = builtins.concatStringsSep "|" formats;
|
|
||||||
in
|
|
||||||
runCommand "${drv.name}-compressed" {} ''
|
|
||||||
mkdir $out
|
|
||||||
${xorg.lndir}/bin/lndir ${drv}/ $out/
|
|
||||||
|
|
||||||
find -L $out -type f -regextype posix-extended \
|
|
||||||
-iregex '.*\.(${formatsbar})' | \
|
|
||||||
${builtins.concatStringsSep " | \\\n " compressCommands}
|
|
||||||
find $out/
|
|
||||||
'';
|
|
||||||
in {
|
|
||||||
/*
|
|
||||||
compressDrv compresses files in a given derivation.
|
|
||||||
|
|
||||||
Inputs:
|
|
||||||
|
|
||||||
- formats :: [String]
|
|
||||||
|
|
||||||
List of file extensions to compress.
|
|
||||||
|
|
||||||
Example: ["txt" "svg" "xml"]
|
|
||||||
|
|
||||||
- compressors :: [String]
|
|
||||||
|
|
||||||
A list of compressor names to use. Each element will need to have
|
|
||||||
an associated compressor in the same arguments (see below).
|
|
||||||
|
|
||||||
Example: ["gz" "zstd"]
|
|
||||||
|
|
||||||
- compressor-<EXTENSION> :: String
|
|
||||||
|
|
||||||
Map a desired extension (e.g. `gz`) to a compress program.
|
|
||||||
|
|
||||||
The compressor program that will be executed to get the `COMPRESSOR`
|
|
||||||
extension. The program is passed to xargs like this:
|
|
||||||
|
|
||||||
xargs -I{} -n1 ${prog}
|
|
||||||
|
|
||||||
Compressor must:
|
|
||||||
- read symlinks (thus --force is needed to gzip, zstd, xz).
|
|
||||||
- keep the original file in place (--keep).
|
|
||||||
|
|
||||||
Example compressor:
|
|
||||||
|
|
||||||
compressor-xz = "${xz}/bin/xz --force --keep {}";
|
|
||||||
|
|
||||||
See compressDrvWeb, which is a wrapper on top of compressDrv, for broader
|
|
||||||
use examples.
|
|
||||||
*/
|
|
||||||
|
|
||||||
compressDrv = compressDrv;
|
|
||||||
|
|
||||||
/*
|
|
||||||
compressDrvWeb compresses a derivation for "web server" usage.
|
|
||||||
|
|
||||||
Useful when one wants to pre-compress certain static assets and pass them to
|
|
||||||
the web server. For example, `pkgs.gamja` creates this derivation:
|
|
||||||
|
|
||||||
/nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/
|
|
||||||
├── index.2fd01148.js
|
|
||||||
├── index.2fd01148.js.map
|
|
||||||
├── index.37aa9a8a.css
|
|
||||||
├── index.37aa9a8a.css.map
|
|
||||||
├── index.html
|
|
||||||
└── manifest.webmanifest
|
|
||||||
|
|
||||||
`pkgs.compressDrvWeb pkgs.gamja`:
|
|
||||||
|
|
||||||
/nix/store/f5ryid7zrw2hid7h9kil5g5j29q5r2f7-gamja-1.0.0-beta.9-compressed
|
|
||||||
├── index.2fd01148.js -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js
|
|
||||||
├── index.2fd01148.js.br
|
|
||||||
├── index.2fd01148.js.gz
|
|
||||||
├── index.2fd01148.js.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js.map
|
|
||||||
├── index.2fd01148.js.map.br
|
|
||||||
├── index.2fd01148.js.map.gz
|
|
||||||
├── index.37aa9a8a.css -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css
|
|
||||||
├── index.37aa9a8a.css.br
|
|
||||||
├── index.37aa9a8a.css.gz
|
|
||||||
├── index.37aa9a8a.css.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css.map
|
|
||||||
├── index.37aa9a8a.css.map.br
|
|
||||||
├── index.37aa9a8a.css.map.gz
|
|
||||||
├── index.html -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.html
|
|
||||||
├── index.html.br
|
|
||||||
├── index.html.gz
|
|
||||||
├── manifest.webmanifest -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/manifest.webmanifest
|
|
||||||
├── manifest.webmanifest.br
|
|
||||||
└── manifest.webmanifest.gz
|
|
||||||
|
|
||||||
|
|
||||||
When this `-compressed` directory is passed to a properly configured web
|
|
||||||
server, it will serve those pre-compressed files:
|
|
||||||
|
|
||||||
$ curl -I -H 'Accept-Encoding: br' https://irc.example.org/
|
|
||||||
<...>
|
|
||||||
content-encoding: br
|
|
||||||
<...>
|
|
||||||
|
|
||||||
For example, a caddy configuration snippet for gamja to serve
|
|
||||||
the static assets (JS, CSS files) pre-compressed:
|
|
||||||
|
|
||||||
virtualHosts."irc.example.org".extraConfig = ''
|
|
||||||
root * ${pkgs.compressDrvWeb pkgs.gamja {}}
|
|
||||||
file_server browse {
|
|
||||||
precompressed br gzip
|
|
||||||
}
|
|
||||||
'';
|
|
||||||
|
|
||||||
This feature is also available in nginx via `ngx_brotli` and
|
|
||||||
`ngx_http_gzip_static_module`.
|
|
||||||
|
|
||||||
Inputs:
|
|
||||||
- formats :: [String]
|
|
||||||
|
|
||||||
List of file extensions to compress.
|
|
||||||
|
|
||||||
Default: common formats that compress well. The list may be appended
|
|
||||||
expanded.
|
|
||||||
|
|
||||||
- extraFormats :: [String]
|
|
||||||
|
|
||||||
Extra extensions to compress in addition to `formats`.
|
|
||||||
|
|
||||||
- compressors :: [String]
|
|
||||||
|
|
||||||
A list of compressor names to use. Each element will need to have
|
|
||||||
an associated compressor in the same arguments (see below).
|
|
||||||
|
|
||||||
Default: ["gz" "br"]
|
|
||||||
|
|
||||||
- extraCompressors :: [String]
|
|
||||||
|
|
||||||
Extra compressors in addition to `compressors`.
|
|
||||||
|
|
||||||
- compressor-<COMPRESSOR> :: String
|
|
||||||
|
|
||||||
Map a desired extension (e.g. `gz`) to a compress program.
|
|
||||||
|
|
||||||
The compressor program that will be executed to get the `COMPRESSOR`
|
|
||||||
extension. The program is passed to xargs like this:
|
|
||||||
|
|
||||||
xargs -I{} -n1 ${prog}
|
|
||||||
|
|
||||||
Default:
|
|
||||||
|
|
||||||
compressor-gz = "${zopfli}/bin/zopfli --keep {}";
|
|
||||||
compressor-br = "${brotli}/bin/brotli --keep --no-copy-stat {}";
|
|
||||||
|
|
||||||
*/
|
|
||||||
|
|
||||||
compressDrvWeb = drv: {
|
|
||||||
formats ? ["css" "js" "svg" "ttf" "eot" "txt" "xml" "map" "html" "json" "webmanifest"],
|
|
||||||
extraFormats ? [],
|
|
||||||
compressors ? ["gz" "br"],
|
|
||||||
extraCompressors ? [],
|
|
||||||
...
|
|
||||||
} @ args:
|
|
||||||
compressDrv drv ({
|
|
||||||
formats = formats ++ extraFormats;
|
|
||||||
compressors = compressors ++ extraCompressors;
|
|
||||||
compressor-gz = "${zopfli}/bin/zopfli --keep {}";
|
|
||||||
compressor-br = "${brotli}/bin/brotli --keep --no-copy-stat {}";
|
|
||||||
}
|
|
||||||
// lib.filterAttrs (k: _: (lib.hasPrefix "compressor-" k)) args);
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,76 @@
|
||||||
|
/*
|
||||||
|
compressDrv compresses files in a given derivation.
|
||||||
|
|
||||||
|
Inputs:
|
||||||
|
|
||||||
|
- formats :: [String]
|
||||||
|
|
||||||
|
List of file extensions to compress.
|
||||||
|
|
||||||
|
Example: ["txt" "svg" "xml"]
|
||||||
|
|
||||||
|
- compressors :: [String]
|
||||||
|
|
||||||
|
A list of compressor names to use. Each element will need to have
|
||||||
|
an associated compressor in the same arguments (see below).
|
||||||
|
|
||||||
|
Example: ["gz" "zstd"]
|
||||||
|
|
||||||
|
- compressor-<EXTENSION> :: String
|
||||||
|
|
||||||
|
Map a desired extension (e.g. `gz`) to a compress program.
|
||||||
|
|
||||||
|
The compressor program that will be executed to get the `COMPRESSOR`
|
||||||
|
extension. The program should have a single " {}", which will be the
|
||||||
|
replaced with the target filename.
|
||||||
|
|
||||||
|
Compressor must:
|
||||||
|
- read symlinks (thus --force is needed to gzip, zstd, xz).
|
||||||
|
- keep the original file in place (--keep).
|
||||||
|
|
||||||
|
Example compressor:
|
||||||
|
|
||||||
|
compressor-xz = "${xz}/bin/xz --force --keep {}";
|
||||||
|
|
||||||
|
See compressDrvWeb, which is a wrapper on top of compressDrv, for broader
|
||||||
|
use examples.
|
||||||
|
*/
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
runCommand,
|
||||||
|
parallel,
|
||||||
|
xorg,
|
||||||
|
}: drv: {
|
||||||
|
formats,
|
||||||
|
compressors,
|
||||||
|
...
|
||||||
|
} @ args: let
|
||||||
|
validProg = ext: prog: let
|
||||||
|
matches = (builtins.length (builtins.split "\\{}" prog) - 1) / 2;
|
||||||
|
in
|
||||||
|
lib.assertMsg
|
||||||
|
(matches == 1)
|
||||||
|
"compressor-${ext} needs to have exactly one '{}', found ${builtins.toString matches}";
|
||||||
|
compressorMap = lib.filterAttrs (k: _: (lib.hasPrefix "compressor-" k)) args;
|
||||||
|
mkCmd = ext: prog: let
|
||||||
|
fname = builtins.replaceStrings ["{}"] ["\"$fname\""] prog;
|
||||||
|
in
|
||||||
|
assert validProg ext prog; "${parallel}/bin/sem --id $$ -P$NIX_BUILD_CORES ${fname}";
|
||||||
|
formatsPipe = builtins.concatStringsSep "|" formats;
|
||||||
|
in
|
||||||
|
runCommand "${drv.name}-compressed" {} ''
|
||||||
|
mkdir $out
|
||||||
|
export PARALLEL_HOME=$(mktemp -d)
|
||||||
|
${xorg.lndir}/bin/lndir ${drv}/ $out/
|
||||||
|
|
||||||
|
while IFS= read -d "" -r fname; do
|
||||||
|
${
|
||||||
|
lib.concatMapStringsSep
|
||||||
|
"\n"
|
||||||
|
(ext: mkCmd ext (builtins.getAttr "compressor-${ext}" compressorMap))
|
||||||
|
compressors
|
||||||
|
}
|
||||||
|
done < <(find -L $out -print0 -type f -regextype posix-extended -iregex '.*\.(${formatsPipe})')
|
||||||
|
|
||||||
|
${parallel}/bin/sem --id ''$$ --wait
|
||||||
|
''
|
||||||
|
|
@ -1,13 +1,11 @@
|
||||||
{
|
{
|
||||||
gzip,
|
gzip,
|
||||||
callPackage,
|
runCommand,
|
||||||
runCommandNoCC,
|
compressDrv,
|
||||||
}: let
|
}: let
|
||||||
compressDrv = (callPackage ./compress-drv.nix {}).compressDrv;
|
example = runCommand "sample-drv" {} ''
|
||||||
|
|
||||||
example = runCommandNoCC "sample-drv" {} ''
|
|
||||||
mkdir $out
|
mkdir $out
|
||||||
echo 1 > $out/1.txt
|
echo 42 > $out/1.txt
|
||||||
touch $out/2.png
|
touch $out/2.png
|
||||||
'';
|
'';
|
||||||
drv = compressDrv example {
|
drv = compressDrv example {
|
||||||
|
|
@ -16,7 +14,7 @@
|
||||||
compressor-gz = "${gzip}/bin/gzip --force --keep --fast {}";
|
compressor-gz = "${gzip}/bin/gzip --force --keep --fast {}";
|
||||||
};
|
};
|
||||||
in
|
in
|
||||||
runCommandNoCC "test-compressDrv" {} ''
|
runCommand "test-compressDrv" {} ''
|
||||||
set -ex
|
set -ex
|
||||||
find ${drv}
|
find ${drv}
|
||||||
test -h ${drv}/1.txt
|
test -h ${drv}/1.txt
|
||||||
|
|
@ -0,0 +1,113 @@
|
||||||
|
/*
|
||||||
|
compressDrvWeb compresses a derivation for common web server use.
|
||||||
|
|
||||||
|
Useful when one wants to pre-compress certain static assets and pass them to
|
||||||
|
the web server. For example, `pkgs.gamja` creates this derivation:
|
||||||
|
|
||||||
|
/nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/
|
||||||
|
├── index.2fd01148.js
|
||||||
|
├── index.2fd01148.js.map
|
||||||
|
├── index.37aa9a8a.css
|
||||||
|
├── index.37aa9a8a.css.map
|
||||||
|
├── index.html
|
||||||
|
└── manifest.webmanifest
|
||||||
|
|
||||||
|
`pkgs.compressDrvWeb pkgs.gamja`:
|
||||||
|
|
||||||
|
/nix/store/f5ryid7zrw2hid7h9kil5g5j29q5r2f7-gamja-1.0.0-beta.9-compressed
|
||||||
|
├── index.2fd01148.js -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js
|
||||||
|
├── index.2fd01148.js.br
|
||||||
|
├── index.2fd01148.js.gz
|
||||||
|
├── index.2fd01148.js.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js.map
|
||||||
|
├── index.2fd01148.js.map.br
|
||||||
|
├── index.2fd01148.js.map.gz
|
||||||
|
├── index.37aa9a8a.css -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css
|
||||||
|
├── index.37aa9a8a.css.br
|
||||||
|
├── index.37aa9a8a.css.gz
|
||||||
|
├── index.37aa9a8a.css.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css.map
|
||||||
|
├── index.37aa9a8a.css.map.br
|
||||||
|
├── index.37aa9a8a.css.map.gz
|
||||||
|
├── index.html -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.html
|
||||||
|
├── index.html.br
|
||||||
|
├── index.html.gz
|
||||||
|
├── manifest.webmanifest -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/manifest.webmanifest
|
||||||
|
├── manifest.webmanifest.br
|
||||||
|
└── manifest.webmanifest.gz
|
||||||
|
|
||||||
|
|
||||||
|
When this `-compressed` directory is passed to a properly configured web
|
||||||
|
server, it will serve those pre-compressed files:
|
||||||
|
|
||||||
|
$ curl -I -H 'Accept-Encoding: br' https://irc.example.org/
|
||||||
|
<...>
|
||||||
|
content-encoding: br
|
||||||
|
<...>
|
||||||
|
|
||||||
|
For example, a caddy configuration snippet for gamja to serve
|
||||||
|
the static assets (JS, CSS files) pre-compressed:
|
||||||
|
|
||||||
|
virtualHosts."irc.example.org".extraConfig = ''
|
||||||
|
root * ${pkgs.compressDrvWeb pkgs.gamja {}}
|
||||||
|
file_server browse {
|
||||||
|
precompressed br gzip
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
This feature is also available in nginx via `ngx_brotli` and
|
||||||
|
`ngx_http_gzip_static_module`.
|
||||||
|
|
||||||
|
Inputs:
|
||||||
|
- formats :: [String]
|
||||||
|
|
||||||
|
List of file extensions to compress.
|
||||||
|
|
||||||
|
Default: common formats that compress well. The list may be expanded.
|
||||||
|
|
||||||
|
- extraFormats :: [String]
|
||||||
|
|
||||||
|
Extra extensions to compress in addition to `formats`.
|
||||||
|
|
||||||
|
- compressors :: [String]
|
||||||
|
|
||||||
|
A list of compressor names to use. Each element will need to have
|
||||||
|
an associated compressor in the same arguments (see below).
|
||||||
|
|
||||||
|
Default: ["gz" "br"]
|
||||||
|
|
||||||
|
- extraCompressors :: [String]
|
||||||
|
|
||||||
|
Extra compressors in addition to `compressors`.
|
||||||
|
|
||||||
|
- compressor-<COMPRESSOR> :: String
|
||||||
|
|
||||||
|
Map a desired extension (e.g. `gz`) to a compress program.
|
||||||
|
|
||||||
|
The compressor program that will be executed to get the `COMPRESSOR`
|
||||||
|
extension. The program is passed to xargs like this:
|
||||||
|
|
||||||
|
xargs -I{} -n1 ${prog}
|
||||||
|
|
||||||
|
Default:
|
||||||
|
|
||||||
|
compressor-gz = "${zopfli}/bin/zopfli --keep {}";
|
||||||
|
compressor-br = "${brotli}/bin/brotli --keep --no-copy-stat {}";
|
||||||
|
*/
|
||||||
|
{
|
||||||
|
lib,
|
||||||
|
zopfli,
|
||||||
|
brotli,
|
||||||
|
compressDrv,
|
||||||
|
}: drv: {
|
||||||
|
formats ? ["css" "js" "svg" "ttf" "eot" "txt" "xml" "map" "html" "json" "webmanifest"],
|
||||||
|
extraFormats ? [],
|
||||||
|
compressors ? ["gz" "br"],
|
||||||
|
extraCompressors ? [],
|
||||||
|
...
|
||||||
|
} @ args:
|
||||||
|
compressDrv drv ({
|
||||||
|
formats = formats ++ extraFormats;
|
||||||
|
compressors = compressors ++ extraCompressors;
|
||||||
|
compressor-gz = "${zopfli}/bin/zopfli --keep {}";
|
||||||
|
compressor-br = "${brotli}/bin/brotli --keep --no-copy-stat {}";
|
||||||
|
}
|
||||||
|
// lib.filterAttrs (k: _: (lib.hasPrefix "compressor-" k)) args)
|
||||||
Loading…
Reference in New Issue