138 lines
4.8 KiB
Nix
138 lines
4.8 KiB
Nix
/*
|
|
Creating a new ZFS pool:
|
|
|
|
% POOLNAME=tank
|
|
|
|
Find out the best recordsize value. 64K for databases and configs, 1M for
|
|
large data chunks such as photos and videos.
|
|
|
|
% RECORDSIZE=64K
|
|
|
|
Create a key:
|
|
|
|
% KEYPATH=$(mktemp)
|
|
% od -Anone -x -N 32 -w64 /dev/random | tr -d [:blank:] > "${KEYPATH?}"
|
|
|
|
# cat "${KEYPATH?}" | zpool create \
|
|
-O compression=lz4 \
|
|
-O xattr=sa \
|
|
-O atime=off \
|
|
-O dedup=off \
|
|
-O acltype=off \
|
|
-O encryption=on \
|
|
-O keyformat=hex \
|
|
-o autoexpand=off \
|
|
-O mountpoint=legacy \
|
|
-O recordsize="${RECORDSIZE?}" \
|
|
-o ashift=12 \
|
|
"${POOLNAME?}" mirror /dev/disk/by-id/{id1,id2}
|
|
|
|
compression: lz4 is fast enough.
|
|
xattr: xattr must be enabled for "chattr" (set immutability) and "setcap"
|
|
(ping) to work. "sa" is fast, "on" is slow.
|
|
atime: faster access time when disabled. Can be specified as mount option
|
|
instead.
|
|
dedup: very slow, disable.
|
|
acltype: this is for having a list of granular access with setfacl, instead
|
|
of the owner/group/others UNIX model. We don't use that, disable.
|
|
If you happened to enable it before, change to off, but this isn't
|
|
retroactive - you have to "setfacl --remove-all --recursive /path".
|
|
See also https://www.truenas.com/community/threads/37046/
|
|
encryption: enable encryption at the pool level, which does nothing else
|
|
than storing the parameters. Only datasets themselves are
|
|
encrypted, and can be chosen for each of them.
|
|
autoexpand: do not automatically expand the pool to fit the size of attached
|
|
devices. This will hurt if you detach a smaller device and then
|
|
try to re-attach it! zpool size CAN NOT be reduced.
|
|
ashift: use 12 because all SSDs are tuned for 4096 block size.
|
|
https://openzfs.github.io/openzfs-docs/Performance%20and%20Tuning/Hardware.html#flash-pages
|
|
Previously, this manual recommended ashift = log2 PHY-SEC from
|
|
lsblk -o NAME,PHY-SEC.
|
|
mountpoint: we mount via NixOS config, not automatically in ZFS.
|
|
|
|
Default options:
|
|
autotrim=off because we run a trim service instead. Continuous trimming
|
|
can be destructive for some SSD drives.
|
|
|
|
Use "zdb -C" to get current values such as ashift.
|
|
|
|
Upon creating the pool:
|
|
1. "chattr +i" the directory where you are mounting it, to avoid writing
|
|
data there before the pool is mounted.
|
|
2. create a 20% reservation to prevent performance deterioration (on zfs
|
|
occurs when free space goes below 20%):
|
|
|
|
# zfs create \
|
|
-o encryption=off \
|
|
-o refreservation=10G \
|
|
-o mountpoint=none \
|
|
-o canmount=off \
|
|
"${POOLNAME?}/reserved"
|
|
3. add the pool to fileSystems
|
|
4. consider adding the pool to services.sanoid.datasets for auto-snapshot
|
|
|
|
Creating a new ZFS filesystem:
|
|
|
|
# zfs create "${POOLNAME?}/myfs"
|
|
|
|
mountpoint: see above.
|
|
encryption: set to "off" to disable encryption for this dataset
|
|
specifically.
|
|
|
|
Remember to "chattr +i" (see above) and add to fileSystems.
|
|
|
|
To check config of an existing pool or filesystem, use:
|
|
% zfs get all -s local -t filesystem -r "${POOLNAME?}"
|
|
*/
|
|
|
|
{ lib, pkgs, ... }:
|
|
{
|
|
services.zfs = {
|
|
trim = {
|
|
enable = true;
|
|
interval = "Mon 22:00";
|
|
randomizedDelaySec = "1min";
|
|
};
|
|
autoScrub = {
|
|
enable = true;
|
|
interval = "*-*-01 23:00";
|
|
randomizedDelaySec = "1min";
|
|
};
|
|
|
|
zed.settings = {
|
|
ZED_EMAIL_ADDR = [ "root" ];
|
|
ZED_NOTIFY_VERBOSE = true;
|
|
|
|
ZED_USE_ENCLOSURE_LEDS = true;
|
|
ZED_SCRUB_AFTER_RESILVER = true;
|
|
};
|
|
};
|
|
# TODO: raise a PR to nixpkgs to add options to services.zfs.trim
|
|
systemd.services.zpool-trim.serviceConfig.ExecStart =
|
|
lib.mkForce "${pkgs.runtimeShell} -c 'for pool in $(zpool list -H -o name); do zpool trim -r 100MB $pool; done || true' ";
|
|
|
|
services.sanoid = {
|
|
enable = true;
|
|
interval = "*:0/15"; # For "frequently" snapshots.
|
|
templates = {
|
|
"default" = {
|
|
frequently = 8; # Keep this many snapshots @15min frequency.
|
|
hourly = 48;
|
|
daily = 90;
|
|
weekly = 30;
|
|
monthly = 24;
|
|
yearly = 100;
|
|
};
|
|
};
|
|
};
|
|
|
|
# https://openzfs.github.io/openzfs-docs/Project%20and%20Community/FAQ.html#selecting-dev-names-when-creating-a-pool-linux
|
|
# Default /dev/disk/by-id doesn't work on VM, but /dev/disk/by-uuid could.
|
|
boot.zfs.devNodes = lib.mkDefault "/dev/disk/by-path";
|
|
# We set forceImportRoot to true because our servers have exclusive access to
|
|
# their local disks (no SANs/iSCSI). This guarantees they can recover and boot
|
|
# unattended after a sudden power loss or hard crash, instead of hanging
|
|
# indefinitely in an emergency shell waiting for manual intervention.
|
|
boot.zfs.forceImportRoot = true;
|
|
}
|