Add iso/e2b-contig.sh: make an ISO contiguous on a USB drive from Linux
Easy2Boot's UEFI menu returned straight to the menu for the new kiosk ISO (a 3.5 GB copy on a well-used drive). Some multiboot modes need the ISO in one unbroken piece, and Linux has no defragmenter for NTFS/FAT32/exFAT - Easy2Boot's own Linux helpers are a 32-bit udefrag needing i386 libraries and a FAT32-only perl script. e2b-contig.sh rewrites just the one file, using only ordinary file operations - never filesystem structures - so the worst case is "still not contiguous", never a damaged drive: 1. fill the drive's free space with 4 MB filler files (plugs every gap; a plain copy, or fallocate where it exists at all, just fills the gaps again - measured on ntfs-3g: 7-8 pieces either way); 2. map the fillers (filefrag -e, which also works on ntfs-3g via FIBMAP) and find the longest physically unbroken run of single-piece ones; 3. free just enough of that run and write the copy - the only free space left - then delete the fillers; 4. verify the copy's SHA-256 against the original and rename it into place (same directory, so the data doesn't move). If there's no run big enough while the original is on the drive, --move-out parks it on this computer (checksummed), retries, and puts it back verified if that still fails. --check only reports. Interrupts and failures remove fillers and temp copies. Tested on NTFS images through ntfs-3g: a 60 MB file in 7 pieces became 1 piece with identical content, no leftovers, free space unchanged (27 s for ~900 MB free); drives with no big enough free run (with and without --move-out) report so and leave the original byte-identical, parked copy cleaned up. FAT32/exFAT through their FUSE drivers can't report layout, which the tool detects and stops on, naming the kernel driver to remount with (vfat/exfat/ntfs3); the kernel drivers aren't available in the test environment, so those paths are untested. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Cjz8a58eB5P9sUweqd3NHm
This commit is contained in:
Claude
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/bin/bash
|
||||
################################################################################
|
||||
# iso/e2b-contig.sh - Make a file on a USB drive contiguous, from Linux.
|
||||
#
|
||||
# Easy2Boot (and other multiboot loaders) can only boot some ISO files when
|
||||
# they sit on the drive in one unbroken piece. A big ISO copied onto a
|
||||
# well-used drive usually lands in several pieces, and Linux has no
|
||||
# defragmenter for NTFS/FAT32/exFAT. This rewrites the one file so it is
|
||||
# contiguous, using only ordinary file operations - it never writes
|
||||
# filesystem structures directly, so the worst case is "still not
|
||||
# contiguous", not a damaged drive.
|
||||
#
|
||||
# How: a plain copy just fills the drive's scattered free gaps again. So
|
||||
# 1. fill the free space with temporary filler files (plugging every gap),
|
||||
# 2. find the longest physically unbroken run among them and free exactly
|
||||
# that run,
|
||||
# 3. write the copy - the only free space left is that run,
|
||||
# 4. delete the fillers, verify the copy's SHA-256, swap it in.
|
||||
# This writes about as much data as the drive has free (progress shown) -
|
||||
# minutes on USB 3, longer on USB 2 or a nearly empty big drive.
|
||||
#
|
||||
# If there's no unbroken run as big as the file while the original is still
|
||||
# on the drive, --move-out first parks the original on this computer's disk
|
||||
# (checksummed), deletes it from the drive to free its pieces, and retries.
|
||||
#
|
||||
# Usage:
|
||||
# sudo iso/e2b-contig.sh [--check] [--move-out] [--temp-dir DIR] [--yes] FILE...
|
||||
#
|
||||
# --check Only report whether each file is contiguous.
|
||||
# --move-out Allowed to park the original on this computer if needed
|
||||
# (default temp dir: /var/tmp).
|
||||
# --temp-dir D Where to park it.
|
||||
# --yes Don't ask for confirmation.
|
||||
#
|
||||
# Works with NTFS (ntfs-3g or the ntfs3 kernel driver), FAT32 and exFAT -
|
||||
# anything filefrag can map. Run as root (filefrag needs it on most of
|
||||
# these filesystems).
|
||||
################################################################################
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
CHECK_ONLY=0
|
||||
MOVE_OUT=0
|
||||
ASSUME_YES=0
|
||||
TEMP_DIR=/var/tmp
|
||||
# Filler size. Small enough that fillers fit inside the drive's free gaps
|
||||
# (a filler spanning two gaps can't be part of an unbroken run); kept in
|
||||
# subfolders of 1000 because FAT32 caps the number of files per folder.
|
||||
CHUNK_MB=4
|
||||
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
info() { echo "==> $*"; }
|
||||
|
||||
FILES=()
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--check) CHECK_ONLY=1; shift ;;
|
||||
--move-out) MOVE_OUT=1; shift ;;
|
||||
--temp-dir) TEMP_DIR="$2"; shift 2 ;;
|
||||
--yes|-y) ASSUME_YES=1; shift ;;
|
||||
-h|--help) sed -n '/^# Usage:/,/^################/p' "$0" | sed '$d; s/^# \{0,1\}//'; exit 0 ;;
|
||||
-*) die "unknown option: $1" ;;
|
||||
*) FILES+=("$1"); shift ;;
|
||||
esac
|
||||
done
|
||||
(( ${#FILES[@]} )) || die "no file given - usage: sudo $0 [--check] FILE..."
|
||||
[[ $EUID -eq 0 ]] || die "run with sudo (filefrag needs root on NTFS/FAT/exFAT)"
|
||||
for cmd in filefrag findmnt blkid sha256sum dd awk df stat; do
|
||||
command -v "$cmd" &>/dev/null || die "'$cmd' not found (packages: e2fsprogs, util-linux, coreutils)"
|
||||
done
|
||||
|
||||
# extents FILE -> "physical_start physical_end" per extent (filesystem
|
||||
# blocks), in file order. Empty output = couldn't map (unsupported).
|
||||
extents() {
|
||||
filefrag -e "$1" 2>/dev/null | awk '
|
||||
/^ *[0-9]+:/ {
|
||||
gsub(/\.\./, " "); gsub(/:/, " ")
|
||||
# fields: idx logical_start logical_end physical_start physical_end length ...
|
||||
print $4, $5
|
||||
}'
|
||||
}
|
||||
|
||||
# Block size filefrag reports in.
|
||||
fs_block_size() {
|
||||
filefrag -e "$1" 2>/dev/null | sed -n 's/.*(\([0-9]*\) blocks of \([0-9]*\) bytes).*/\2/p' | head -1
|
||||
}
|
||||
|
||||
# pieces FILE -> number of physically separate pieces (adjacent extents
|
||||
# count as one; filefrag splits long runs into several extents).
|
||||
pieces() {
|
||||
extents "$1" | awk 'NR==1 {n=1; prev=$2; next} { if ($1 != prev + 1) n++; prev=$2 } END { print n+0 }'
|
||||
}
|
||||
|
||||
confirm() {
|
||||
(( ASSUME_YES )) && return 0
|
||||
local reply
|
||||
read -r -p "$1 [y/N] " reply
|
||||
[[ "${reply,,}" == y || "${reply,,}" == yes ]]
|
||||
}
|
||||
|
||||
FILL_DIR=""
|
||||
TMP_COPY=""
|
||||
cleanup() {
|
||||
[[ -n "$FILL_DIR" && -d "$FILL_DIR" ]] && rm -rf -- "$FILL_DIR"
|
||||
[[ -n "$TMP_COPY" && -f "$TMP_COPY" ]] && rm -f -- "$TMP_COPY"
|
||||
sync
|
||||
}
|
||||
trap cleanup EXIT
|
||||
trap 'echo; echo "Interrupted - removing temporary files..."; exit 130' INT TERM
|
||||
|
||||
# fill_and_carve MOUNT NEED_BYTES -> 0 if it left exactly one free run of at
|
||||
# least NEED_BYTES (fillers stay in $FILL_DIR until cleanup), 1 if no such run.
|
||||
fill_and_carve() {
|
||||
local mnt="$1" need="$2"
|
||||
FILL_DIR="$mnt/.e2b-contig-fill.$$"
|
||||
mkdir -p "$FILL_DIR" || return 1
|
||||
local free_mb
|
||||
free_mb=$(( $(df -B1M --output=avail "$mnt" | tail -1) ))
|
||||
info "Filling $free_mb MB of free space with temporary files to plug the gaps..."
|
||||
local n=0 written=0 fname
|
||||
while :; do
|
||||
n=$((n + 1))
|
||||
fname="$FILL_DIR/d$(printf %04d $((n / 1000)))/f$(printf %07d $n)"
|
||||
mkdir -p "${fname%/*}" 2>/dev/null || break
|
||||
# dd stops (keeping what it wrote) when the drive is full.
|
||||
dd if=/dev/zero of="$fname" bs=1M count="$CHUNK_MB" status=none 2>/dev/null
|
||||
local rc=$?
|
||||
local sz
|
||||
sz=$(stat -c %s "$fname" 2>/dev/null || echo 0)
|
||||
written=$((written + sz))
|
||||
(( n % 25 == 0 )) && printf '\r %d / %d MB' "$((written / 1048576))" "$free_mb"
|
||||
(( rc != 0 || sz < CHUNK_MB * 1048576 )) && break
|
||||
done
|
||||
printf '\r %d / %d MB\n' "$((written / 1048576))" "$free_mb"
|
||||
sync
|
||||
|
||||
# Single-piece fillers sorted by physical position; the longest chain
|
||||
# where each starts right after the previous ends is the best run.
|
||||
local bs
|
||||
bs=$(fs_block_size "$FILL_DIR/d0000/f0000001")
|
||||
[[ -n "$bs" ]] || { echo " Couldn't map the filler files (filefrag unsupported here)."; return 1; }
|
||||
# The map lives on this computer, not on the (now full) drive.
|
||||
local f list
|
||||
list=$(mktemp) || return 1
|
||||
info "Mapping where the filler files landed..."
|
||||
for f in "$FILL_DIR"/d*/f*; do
|
||||
[[ -s "$f" ]] || continue
|
||||
# One filefrag call per filler (there can be thousands): keep it only
|
||||
# if it's a single piece, as "start end path".
|
||||
extents "$f" | awk -v f="$f" '
|
||||
NR == 1 { s = $1; e = $2; n = 1; next }
|
||||
{ if ($1 == e + 1) e = $2; else n++ }
|
||||
END { if (NR > 0 && n == 1) print s, e, f }' >> "$list"
|
||||
done
|
||||
local best
|
||||
best=$(sort -n "$list" | awk -v bs="$bs" '
|
||||
{ if (NR == 1 || $1 != prev_end + 1) { start_line = NR; run_bytes = 0; files = "" }
|
||||
run_bytes += ($2 - $1 + 1) * bs; files = files " " $3; prev_end = $2
|
||||
if (run_bytes > best_bytes) { best_bytes = run_bytes; best_files = files } }
|
||||
END { print best_bytes + 0, best_files }')
|
||||
rm -f -- "$list"
|
||||
local best_bytes=${best%% *}
|
||||
info "Longest unbroken free run found: $((best_bytes / 1048576)) MB (need $((need / 1048576)) MB)"
|
||||
if (( best_bytes < need )); then
|
||||
return 1
|
||||
fi
|
||||
# Free just enough of that run (from its start) for the file - keeping the
|
||||
# rest plugged so the copy can't wander elsewhere.
|
||||
local freed=0 g
|
||||
for g in ${best#* }; do
|
||||
(( freed >= need )) && break
|
||||
freed=$((freed + $(stat -c %s "$g")))
|
||||
rm -f -- "$g"
|
||||
done
|
||||
sync
|
||||
return 0
|
||||
}
|
||||
|
||||
# restore_parked PARKED FILE SUM - copy a parked original back onto the
|
||||
# drive (not contiguous - just safe) and only delete the parked copy once
|
||||
# the restored one verifies.
|
||||
restore_parked() {
|
||||
local parked="$1" file="$2" sum="$3"
|
||||
info "Putting the original back..."
|
||||
if cp -- "$parked" "$file" && [[ "$(sha256sum "$file" | cut -d' ' -f1)" == "$sum" ]]; then
|
||||
rm -f -- "$parked"
|
||||
echo " Original restored (unchanged, still in pieces)."
|
||||
else
|
||||
echo " !! Couldn't verify the restored file - the original is kept safe at $parked"
|
||||
fi
|
||||
}
|
||||
|
||||
# make_contiguous FILE
|
||||
make_contiguous() {
|
||||
local file="$1"
|
||||
[[ -f "$file" ]] || { echo "$file: not a file"; return 1; }
|
||||
file=$(realpath "$file")
|
||||
local mnt fstype dev
|
||||
read -r dev fstype mnt < <(findmnt -n -o SOURCE,FSTYPE,TARGET --target "$file")
|
||||
local size
|
||||
size=$(stat -c %s "$file")
|
||||
local p
|
||||
p=$(pieces "$file")
|
||||
echo
|
||||
echo "$file"
|
||||
echo " drive: $dev ($fstype) mounted at $mnt, size $((size / 1048576)) MB"
|
||||
if [[ -z "$(extents "$file")" ]]; then
|
||||
echo " Can't read this file's layout through this driver ($fstype)."
|
||||
# FUSE drivers (fusefat, exfat-fuse) can't map files; the kernel
|
||||
# drivers can - and ntfs-3g usually can, but ntfs3 always can.
|
||||
local real kdrv=""
|
||||
real=$(blkid -o value -s TYPE "$dev" 2>/dev/null)
|
||||
case "$real" in
|
||||
ntfs) kdrv=ntfs3 ;;
|
||||
exfat) kdrv=exfat ;;
|
||||
vfat|msdos) kdrv=vfat ;;
|
||||
esac
|
||||
if [[ -n "$kdrv" ]]; then
|
||||
echo " Remount it with the kernel's $kdrv driver and retry:"
|
||||
echo " sudo umount $dev && sudo mount -t $kdrv $dev $mnt"
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
if [[ "$p" == 1 ]]; then
|
||||
echo " Contiguous (1 piece) - nothing to do."
|
||||
return 0
|
||||
fi
|
||||
echo " NOT contiguous: $p pieces."
|
||||
(( CHECK_ONLY )) && return 2
|
||||
|
||||
confirm " Rewrite it in one piece? (temporarily fills the drive's free space)" || { echo " Skipped."; return 2; }
|
||||
|
||||
info "Checksumming the original..."
|
||||
local sum
|
||||
sum=$(sha256sum "$file" | cut -d' ' -f1)
|
||||
|
||||
local source="$file" parked=""
|
||||
if ! fill_and_carve "$mnt" "$size"; then
|
||||
rm -rf -- "$FILL_DIR"; FILL_DIR=""; sync
|
||||
if (( ! MOVE_OUT )); then
|
||||
echo " Not enough unbroken free space with the original still on the drive."
|
||||
echo " Re-run with --move-out to park the original on this computer meanwhile,"
|
||||
echo " or free up space on the drive (move a few other large files off)."
|
||||
return 1
|
||||
fi
|
||||
local avail
|
||||
avail=$(( $(df -B1 --output=avail "$TEMP_DIR" | tail -1) ))
|
||||
(( avail > size )) || { echo " Not enough room in $TEMP_DIR to park it ($((size / 1048576)) MB needed)."; return 1; }
|
||||
parked="$TEMP_DIR/$(basename "$file").e2b-contig-parked"
|
||||
info "Parking the original in $parked..."
|
||||
cp -- "$file" "$parked" && [[ "$(sha256sum "$parked" | cut -d' ' -f1)" == "$sum" ]] \
|
||||
|| { rm -f -- "$parked"; echo " Parking copy failed its checksum - nothing changed."; return 1; }
|
||||
rm -f -- "$file"; sync
|
||||
source="$parked"
|
||||
if ! fill_and_carve "$mnt" "$size"; then
|
||||
rm -rf -- "$FILL_DIR"; FILL_DIR=""; sync
|
||||
echo " Still no unbroken run big enough."
|
||||
restore_parked "$parked" "$file" "$sum"
|
||||
echo " Free up space on the drive (it needs one unbroken $((size / 1048576)) MB area) and retry."
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
TMP_COPY="$(dirname "$file")/.e2b-contig.$$.tmp"
|
||||
info "Writing the copy into the freed run..."
|
||||
if ! dd if="$source" of="$TMP_COPY" bs=4M conv=fsync status=progress; then
|
||||
echo " Copy failed."
|
||||
rm -f -- "$TMP_COPY"; TMP_COPY=""; rm -rf -- "$FILL_DIR"; FILL_DIR=""; sync
|
||||
[[ -n "$parked" ]] && restore_parked "$parked" "$file" "$sum"
|
||||
return 1
|
||||
fi
|
||||
rm -rf -- "$FILL_DIR"; FILL_DIR=""
|
||||
sync
|
||||
|
||||
local newp
|
||||
newp=$(pieces "$TMP_COPY")
|
||||
info "Verifying..."
|
||||
if [[ "$(sha256sum "$TMP_COPY" | cut -d' ' -f1)" != "$sum" ]]; then
|
||||
echo " !! The copy doesn't match the original's checksum - discarding it."
|
||||
rm -f -- "$TMP_COPY"; TMP_COPY=""; sync
|
||||
[[ -n "$parked" ]] && restore_parked "$parked" "$file" "$sum"
|
||||
return 1
|
||||
fi
|
||||
if [[ "$newp" != 1 ]]; then
|
||||
echo " The copy still came out in $newp pieces (was $p)."
|
||||
fi
|
||||
# Same directory, so a rename: the data stays where it was written.
|
||||
mv -f -- "$TMP_COPY" "$file" && TMP_COPY=""
|
||||
[[ -n "$parked" ]] && rm -f -- "$parked"
|
||||
sync
|
||||
if [[ "$(pieces "$file")" == 1 ]]; then
|
||||
echo " ✓ Contiguous now (1 piece), checksum verified."
|
||||
return 0
|
||||
fi
|
||||
echo " Checksum verified, but still $(pieces "$file") pieces - see the notes above."
|
||||
return 1
|
||||
}
|
||||
|
||||
rc=0
|
||||
for f in "${FILES[@]}"; do
|
||||
make_contiguous "$f" || rc=1
|
||||
done
|
||||
exit "$rc"
|
||||
Reference in New Issue
Block a user