http://docs.khadas.com/_export/code/scripts/xze?codeblock=0
#!/bin/sh ## hyphop ## #= xze - advanced xz + suitable for mt decompression & padding & meta USAGE() { echo ' xze - advanced xz packer (suitable for mt decompression + padding + meta) USAGE xze [ARGS] FILE -3 # fast compression -9 # max slow compression -T#thread # -B#blocksize in M # -f # rewrite output file -c | -C # check is krescue ready -mn | --meta-remove # remove meta -MM | --meta # krescue meta tag=value # meta tags -p # add padding --help # full help META USAGE ./xze IMAGE -M INFO="any strings" var=VAR '; [ "$1" ] && echo '# META USAGE EXAMPLE (Krescue images) IN=FreeBSD-aarch64-13.0-Khadas-EDGE-V-361468M-20200528.img ./xze "$IN" \ --meta \ label=FreeBSD \ builder=Serega \ date="$(LANG=C TZ= date)" \ match=BOARD=Edge \ link=http://dev.kubsu.ru/images/ \ duration=60 \ desc="FreeBSD 13.0 for Khadas EDGE board aarch64...." \ # META minimal ./xze Manjaro-ARM-xfce-vim1-20.08.img.xz -p -MM LABEL=manjaro BOARD=VIM1 # CHECK META ./xze *.xz # UPDATE/REPLACE META all prev meta will be replaced by new values ./xze *.xz -M newvar=NEWVALUE # READ META alternative ways tail -c4096 FILE.xz | xz -dc # or curl -s -jkLf -r-4096 https://dl.khadas.com/Firmware/Krescue/images/Edge-FreeBSD-aarch64-13.0-CURRENT-20200620.img.xz | xz -dc # WHY NOT xz -l not work in this case xz need full file cat *.img.xz | xz -l xz: --list does not support reading from standard input ' exit } CLEAN(){ # echo "[i] clean $meta*">&2 [ -e "$meta" ] && rm $meta* } FAIL(){ echo "[e] $@">&2 CLEAN exit 1 } [ "$xz" ] || xz=$(which xz 2>/dev/null) [ $? = 0 ] || { echo "[e] plz install xz">&2 echo " sudo apt-get install xz-utils">&2 exit 1 } #echo "[i] $xz">&2 $xz --help | grep -q threads= || { echo "[i] plz install last xz with MT support">&2 exit 1 } IN= #OPTS="-F xz --arm --lzma2" OPTS="-F xz -9e" OPTS="-F xz --verbose" BS=4096 # block size [ $# = 0 ] && USAGE T=4 # threads #T=0 # threads T=$(nproc) #T=8 # threads B=90 # in MEGS META= FORCE= KRESCUE_CHK_END= KRESCUE_CHK_MT= SHOW_PROGRESS=1 for a in $@; do # echo "++ $a" shift case $a in -m) META=1 break ;; -M) META=2 break ;; -MM|--meta) META=2 META_LOW_CASE=1 META_PRE="##KRESCUE_META## type:xz " META_POST="##KRESCUE-META## ##KRESCUE##END" break ;; -mn|--meta--remove|--remove-meta) META_REMOVE=1 ;; -p) ADD_PAD=1 ;; -c) KRESCUE_CHK_END=1 ;; -C) KRESCUE_CHK_END=1 KRESCUE_CHK_MT=1 ;; -f) FORCE=1 OPTS="$OPTS $a" continue ;; -T*) T=${a#-T} continue ;; -B*) B=${a#-B} continue ;; -h|--help) USAGE FULL ;; *.xz) [ -f "$a" ] || FAIL not found "$a" CIN="$a" continue ;; *) [ -f "$a" ] && { [ "$IN" ] || IN="$a" continue } OPTS="$OPTS $a" ;; esac done [ "$T" = 0 ] && T=$(nproc) [ "$T" = "" ] && T=4 [ "$T" -gt 7 ] && T=8 # 8 cores can get too much memory meta="/tmp/xze.meta.$$" case $(uname -o) in *BSD*) fsize(){ for s_ in $(stat -L -f "%z" "$1");do break done echo $s_ } ;; ## linux gnu *) fsize(){ for s_ in $(stat -L -c%s "$1");do break done echo $s_ #echo "fsize $1 = $s_">&2 } ;; esac PAD(){ S=$(fsize "$1") S2=$((S/BS*BS)) [ $S2 = $S ] && { return } [ $S2 -lt $S ] && S2=$((S2+BS)) SD=$((S2-S)) [ "$2" ] && \ printf %${SD}s "$2" >> "$1" [ "$2" ] || \ dd if=/dev/zero bs=$SD count=1 2>/dev/null >> "$1" SC=$(fsize "$1") echo "[i] padded to $S2 from $S + $SD">&2 [ "$SC" = "$S2" ] || FAIL jopa #echo $S2 } REQ_label= REQ_match= ADD_image= ADD_builder= ADD_date= ADD_duration= ADD_desc= REQ=" label image builder date duration desc " # date="$(TZ= date)" \ # match=BOARD=Edge \ # link=http://dev.kubsu.ru/images/ \ # duration=60 \ # desc="FreeBSD 13.0 for Khadas EDGE board aarch64...." \ META(){ echo " [i] add meta block ">&2 touch $meta [ -s $meta ] || { ( [ "$S1" ] || { pixz=$(which pixz || which xz) echo "[i] calculating uncompressed size ...">&2 S1=$($pixz -dc < "$OUT" | wc --bytes) } echo "##META_FILE##" echo "FILE: $(basename "$OUT")" echo "UNPACKED_SIZE: $S1" echo "PACKED_SIZE: $S2" echo "FILE_SIZE: $((S2+BS))" echo "##META-FILE##" ) >> $meta } echo "$M" >> $meta for m in $META_PRE ; do echo "$m" >> $meta done for m in "$@" ; do M="$m" [ "$META" = "2" ] && { # echo "$m" | grep -q ^\# || { M=$(echo "$m" | sed s/=/:\ /) m1=$(echo "${M%%:*}" | tr '[:upper:]' '[:lower:]' ) m2=$(echo ${M#*:}) # echo "[=] '$m1' : '$m2'">&2 case $m1 in label) REQ_label="$m2" ;; date) ADD_date="$m2" ;; image) ADD_image="$m2" ;; builder) ADD_builder="$m2" ;; desc) ADD_desc="$m2" ;; duration) ADD_duration="$m2" ;; match) REQ_match="$m2" META_POST=" $META_POST" ;; board) m1=match m2="BOARD=$m2" REQ_match="$m2" esac M="$m1: $m2" echo "$M" >> $meta # } } # echo "[i] add meta $M">&2 done [ "$REQ_label" ] || FAIL meta label not defined [ "$REQ_match" ] || FAIL meta match not defined [ "$ADD_desc" ] || echo "desc: undescripted image" >> $meta [ "$ADD_duration" ] || echo "duration: 60" >> $meta [ "$ADD_builder" ] || echo "builder: $(uname -n)" >> $meta [ "$ADD_date" ] || echo "date: $(TZ= date)" >> $meta [ "$ADD_image" ] || echo "image: $(basename ${OUT%.*})" >> $meta for m in $META_POST ; do echo "$m" >> $meta done [ -s $meta ] && { cat $meta >&2 # PAD $meta " " } $xz $meta || exit 1 PAD $meta.xz || exit 2 cat $meta.xz >> "$OUT" || exit 3 rm $meta.xz CLEAN # echo "[i] DONE">&2 exit 0 } [ "$CIN" ] && { # check just header HEADER=$(head -c5 "$CIN" | tail -c4) [ "$HEADER" = "7zXZ" ] || FAIL broken or not xz input file # fast scan XZS=$($xz -l "$CIN" 2>/dev/null) || FAIL broken xz echo "$XZS" streams= blocks= compressed= uncompressed= cmp1= cmp2= unc1= unc2= i=0 for v in ${XZS#*Filename}; do case $i in 0) streams=$v;; 1) blocks=$v;; 2) cmp1=$v;; 3) cmp2=$v;; 4) unc1=$v;; 5) unc2=$v;; esac i=$((i+1)) done echo "[i] blocks: $blocks // $cmp1*$cmp2 // $unc1*$unc2">&2 [ "$blocks" -lt 2 ] && { echo "[w] its not mt xz">&2 #[ "$KRESCUE_CHK_END" ] || FAIL not ready for krescue mt usage } CS=$(fsize "$CIN") CSCHK=$((CS/BS*BS)) [ "$ADD_PADD" ] && [ "$CSCHK" = "$CS" ] || { PAD "$CIN" CS=$(fsize "$CIN") CSCHK=$((CS/BS*BS)) } [ "$CSCHK" = "$CS" ] || FAIL xz size $CS not padded by $BS tail -c4096 "$CIN" | $xz -dc 2>/dev/null | grep -A99 "^##META" | tee $meta [ -s "$meta" ] || { [ "$META" ] && { OUT="$CIN" #echo "[w] empty meta">&2 META "$@" #CLEAN exit } FAIL empty meta } [ "$META_REMOVE" ] && { echo "[i] remove meta">&2 truncate -s-4096 "$CIN" exit } grep -A99 "##META_FILE" $meta \ | grep -B99 "##META-FILE" > $meta.2 || FAIL xz meta broken [ "$KRESCUE_CHK_END" ] && { tail -c15 $meta | grep -q "##KRESCUE##END" || FAIL xz meta not finished } [ "$META" ] && { OUT="$CIN" mv $meta.2 $meta [ "$S1" ] || { # echo "[i] get size" for S1 in $(grep UNPACKED_SIZE $meta) ; do echo $S1 >/dev/null done } truncate -s-$BS "$OUT" META "$@" } CLEAN exit } [ "$IN" ] || FAIL input file not exist/defined [ "$OUT" ] || { OUT2="$(readlink /proc/$$/fd/1)" case $OUT2 in "") OUT="$IN.xz" ;; /dev/pts*) OUT="$IN.xz" ;; /dev/tty*) OUT="$IN.xz" ;; *) # *pipe:*) # OUT=/dev/stdout OUT="$OUT2" ;; esac } [ -s "$OUT" ] && { echo "[w] $OUT exist already">&2 [ "$FORCE" ] || FAIL plz use -f for rewrite it } S1=$(fsize "$IN") S14=$((S1/8)) [ "$T" -gt 0 ] && \ S14=$((S1/T)) [ $S14 -lt $B ] && B=$S14 B="${B}M" ARGS="$xz -T$T --block-size $B -k -c $OPTS" echo "[i] xze compress $IN ($S1 bytes) > $OUT">&2 echo "# $ARGS ">&2 $ARGS "$IN" > "$OUT" || FAIL xz failed S= S2= PAD "$OUT" XZS=$($xz -l "$OUT") echo "[i] XZ info $XZS">&2 [ "$META" ] && { META "$@" exit } exit 0