scripts:xze

xze

http://docs.khadas.com/_export/code/scripts/xze?codeblock=0
#!/bin/sh
 
## hyphop ##
 
#= xze - advanced xz + suitable for mt decompression & padding & meta
 
USAGE() { echo '
    xze - advanced xz packer (suitable for mt decompression + padding + meta)
 
USAGE
 
    xze [ARGS] FILE
 
    -3                   # fast compression
    -9                   # max slow compression
 
    -T#thread            #
    -B#blocksize in M    #
    -f                   # rewrite output file
    -c | -C              # check is krescue ready
 
    -mn | --meta-remove      # remove meta
    -MM | --meta             # krescue meta (replace all tags)
    --meta-update | -MU      # update/replace existing meta tags only
    tag=value                # meta tags
 
    -p                   # add padding
    --help               # full help
 
META USAGE
 
    ./xze IMAGE -M INFO="any strings" var=VAR
'; [ "$1" ] && echo '# META USAGE EXAMPLE (Krescue images)
 
    IN=FreeBSD-aarch64-13.0-Khadas-EDGE-V-361468M-20200528.img
    ./xze "$IN" \
--meta \
label=FreeBSD \
builder=Serega \
date="$(LANG=C TZ= date)" \
match=BOARD=Edge \
link=http://dev.kubsu.ru/images/ \
duration=60 \
desc="FreeBSD 13.0 for Khadas EDGE board aarch64...." \
 
# META minimal
 
    ./xze Manjaro-ARM-xfce-vim1-20.08.img.xz -p -MM LABEL=manjaro BOARD=VIM1
 
# CHECK META
 
    ./xze *.xz
 
# UPDATE/REPLACE META (replace all tags)
 
    ./xze *.xz -M newvar=NEWVALUE
 
# UPDATE ONLY SOME META TAGS (keep others intact)
 
    ./xze *.xz --meta-update label="New label" duration=120
 
# READ META alternative ways
 
    tail -c4096 FILE.xz | xz -dc
    # or
    curl -s -jkLf -r-4096 https://dl.khadas.com/Firmware/Krescue/images/Edge-FreeBSD-aarch64-13.0-CURRENT-20200620.img.xz | xz -dc
 
# WHY NOT xz -l
 
not work in this case xz need full file
 
    cat *.img.xz | xz -l
    xz: --list does not support reading from standard input
'
 
exit
}
 
CLEAN(){
#   echo "[i] clean $meta*">&2
    [ -e "$meta" ] && rm $meta*
}
 
FAIL(){
    echo "[e] $@">&2
    CLEAN
    exit 1
}
 
[ "$xz" ] || xz=$(which xz 2>/dev/null)
[ $? = 0 ] || {
    echo "[e] plz install xz">&2
    echo " sudo apt-get install xz-utils">&2
    exit 1
}
 
#echo "[i] $xz">&2
 
$xz --help | grep -q threads= || {
    echo "[i] plz install last xz with MT support">&2
    exit 1
}
 
 
IN=
#OPTS="-F xz --arm --lzma2"
OPTS="-F xz -9e"
OPTS="-F xz --verbose"
 
BS=4096 # block size
 
[ $# = 0 ] && USAGE
 
T=4        # threads
#T=0       # threads
T=$(nproc)
 
 
#T=8      # threads
B=90     # in MEGS
 
META=
FORCE=
KRESCUE_CHK_END=
KRESCUE_CHK_MT=
SHOW_PROGRESS=1
 
META_UPDATE=
META_LOW_CASE=
META_PRE=
META_POST=
 
for a in $@; do
#    echo "++ $a"
    shift
    case $a in
-m)
META=1
break
;;
-M)
META=2
break
;;
-MM|--meta)
META=2
META_LOW_CASE=1
META_PRE="##KRESCUE_META## type:xz "
META_POST="##KRESCUE-META## ##KRESCUE##END"
break
;;
--meta-update|-MU)
META=2
META_UPDATE=1
META_LOW_CASE=1
META_PRE="##KRESCUE_META## type:xz "
META_POST="##KRESCUE-META## ##KRESCUE##END"
break
;;
-mn|--meta--remove|--remove-meta)
META_REMOVE=1
;;
-p)
ADD_PAD=1
;;
-c)
KRESCUE_CHK_END=1
;;
-C)
KRESCUE_CHK_END=1
KRESCUE_CHK_MT=1
;;
-f)
FORCE=1
OPTS="$OPTS $a"
continue
;;
-T*)
T=${a#-T}
continue
;;
-B*)
B=${a#-B}
continue
;;
-h|--help)
USAGE FULL
;;
*.xz)
[ -f "$a" ] || FAIL not found "$a"
CIN="$a"
continue
;;
*)
[ -f "$a" ] && {
    [ "$IN" ] || IN="$a"
    continue
}
OPTS="$OPTS $a"
;;
    esac
done
 
 
[ "$T" = 0 ] && T=$(nproc)
[ "$T" = "" ] && T=4
[ "$T" -gt 7 ] && T=8 # 8 cores can get too much memory
 
meta="/tmp/xze.meta.$$"
 
case $(uname -o) in
    *BSD*)
fsize(){
for s_ in $(stat -L -f "%z" "$1");do
    break
done
echo $s_
}
    ;;
    ## linux gnu
    *)
fsize(){
for s_ in $(stat -L -c%s "$1");do
    break
done
echo $s_
#echo "fsize $1 = $s_">&2
}
    ;;
 
esac
 
 
PAD(){
S=$(fsize "$1")
S2=$((S/BS*BS))
[ $S2 = $S ] && {
    return
}
[ $S2 -lt $S ] && S2=$((S2+BS))
SD=$((S2-S))
[ "$2" ] && \
printf %${SD}s "$2" >> "$1"
[ "$2" ] || \
dd if=/dev/zero bs=$SD count=1 2>/dev/null >> "$1"
 
SC=$(fsize "$1")
 
echo "[i] padded to $S2 from $S + $SD">&2
 
[ "$SC" = "$S2" ] || FAIL jopa
 
#echo $S2
}
 
REQ_label=
REQ_match=
 
ADD_image=
ADD_builder=
ADD_date=
ADD_duration=
ADD_desc=
 
REQ="
    label
    image
    builder
    date
    duration
    desc
"
 
#    date="$(TZ= date)" \
#    match=BOARD=Edge \
#    link=http://dev.kubsu.ru/images/ \
#    duration=60 \
#    desc="FreeBSD 13.0 for Khadas EDGE board aarch64...." \
 
 
META(){
    echo "
[i] add meta block
">&2
 
    touch $meta
 
    [ -s $meta ] || {
    (
    [ "$S1" ] || {
pixz=$(which pixz || which xz)
echo "[i] calculating uncompressed size ...">&2
S1=$($pixz -dc < "$OUT" | wc --bytes)
    }
    echo "##META_FILE##"
    echo "FILE: $(basename "$OUT")"
    echo "UNPACKED_SIZE: $S1"
    echo "PACKED_SIZE: $S2"
    echo "FILE_SIZE: $((S2+BS))"
    echo "##META-FILE##"
    ) >> $meta
    }
    echo "$M" >> $meta
 
    for m in $META_PRE  ; do
echo "$m" >> $meta
    done
 
    for m in "$@"  ; do
    M="$m"
    [ "$META" = "2" ] && {
#echo "$m" | grep -q ^\# || {
M=$(echo "$m" | sed s/=/:\ /)
m1=$(echo "${M%%:*}" | tr '[:upper:]' '[:lower:]' )
m2=$(echo ${M#*:})
#echo "[=] '$m1' : '$m2'">&2
case $m1 in
    label)
    REQ_label="$m2"
    ;;
    date)
    ADD_date="$m2"
    ;;
    image)
    ADD_image="$m2"
    ;;
    builder)
    ADD_builder="$m2"
    ;;
    desc)
    ADD_desc="$m2"
    ;;
    duration)
    ADD_duration="$m2"
    ;;
    match)
    REQ_match="$m2"
    META_POST=" $META_POST"
    ;;
    board)
    m1=match
    m2="BOARD=$m2"
    REQ_match="$m2"
esac
 
M="$m1: $m2"
echo "$M" >> $meta
#}
    }
 
#   echo "[i] add meta $M">&2
    done
 
    [ "$REQ_label" ] || FAIL meta label not defined
    [ "$REQ_match" ] || FAIL meta match not defined
 
    [ "$ADD_desc" ] || echo "desc: undescripted image" >> $meta
    [ "$ADD_duration" ] || echo "duration: 60" >> $meta
    [ "$ADD_builder" ] || echo "builder: $(uname -n)" >> $meta
    [ "$ADD_date" ] || echo "date: $(LANG=C TZ= date)" >> $meta
    [ "$ADD_image" ] || echo "image: $(basename ${OUT%.*})" >> $meta
 
    for m in $META_POST  ; do
echo "$m" >> $meta
    done
 
    [ -s $meta ] && {
    cat $meta >&2
#    PAD $meta " "
    }
 
    $xz $meta || exit 1
    PAD $meta.xz || exit 2
    cat $meta.xz >> "$OUT" || exit 3
    rm $meta.xz
 
    CLEAN
#   echo "[i] DONE">&2
    exit 0
}
 
META_UPDATE_ONLY(){
    echo "
[i] update meta block
">&2
 
    [ -s "$meta" ] || FAIL meta empty
 
    tmp="$meta.upd"
    cp "$meta" "$tmp" || exit 1
 
    for m in "$@"; do
        [ "$m" ] || continue
        case "$m" in
        *=*)
            key=${m%%=*}
            val=${m#*=}
 
            # normalize key to lower-case
            key_lc=$(printf '%s\n' "$key" | tr '[:upper:]' '[:lower:]')
 
            # escape value for sed replacement
            esc_val=$(printf '%s\n' "$val" | sed 's/[&/\\]/\\&/g')
 
            if grep -q "^$key_lc:" "$tmp"; then
                # replace existing line
                sed "s/^$key_lc:[[:space:]].*/$key_lc: $esc_val/" "$tmp" > "$tmp.1"
            else
                # insert before krescue marker if exists, else append
                if grep -q "^##KRESCUE-META##" "$tmp"; then
                    sed "/^##KRESCUE-META##/i$key_lc: $esc_val" "$tmp" > "$tmp.1"
                else
                    sed '$a\'"$key_lc: $esc_val" "$tmp" > "$tmp.1"
                fi
            fi
 
            mv "$tmp.1" "$tmp"
            ;;
        *)
            echo "[w] skip wrong meta-update arg '$m' (need key=value)" >&2
            ;;
        esac
    done
 
    mv "$tmp" "$meta"
 
    # show new meta
    cat "$meta" >&2
 
    $xz "$meta" || exit 1
    PAD "$meta.xz" || exit 2
    cat "$meta.xz" >> "$OUT" || exit 3
    rm "$meta.xz"
 
    CLEAN
    exit 0
}
 
 
[ "$CIN" ] && {
 
    # check just header
    HEADER=$(head -c5 "$CIN" | tail -c4)
    [ "$HEADER" = "7zXZ" ] || FAIL broken or not xz input file
 
    # fast scan
    XZS=$($xz -l "$CIN" 2>/dev/null) || FAIL broken xz
    echo "$XZS"
 
    streams=
    blocks=
    compressed=
    uncompressed=
    cmp1=
    cmp2=
    unc1=
    unc2=
    i=0
 
    for v in ${XZS#*Filename}; do
case $i in
    0) streams=$v;;
    1) blocks=$v;;
    2) cmp1=$v;;
    3) cmp2=$v;;
    4) unc1=$v;;
    5) unc2=$v;;
esac
i=$((i+1))
    done
 
    echo "[i] blocks: $blocks // $cmp1*$cmp2 // $unc1*$unc2">&2
 
    [ "$blocks" -lt 2 ] && {
echo "[w] its not mt xz">&2
#[ "$KRESCUE_CHK_END" ] || FAIL not ready for krescue mt usage
    }
 
    CS=$(fsize "$CIN")
    CSCHK=$((CS/BS*BS))
 
    [ "$ADD_PADD" ] &&
    [ "$CSCHK" = "$CS" ] || {
PAD "$CIN"
CS=$(fsize "$CIN")
CSCHK=$((CS/BS*BS))
    }
    [ "$CSCHK" = "$CS" ] || FAIL xz size $CS not padded by $BS
    tail -c4096 "$CIN" | $xz -dc 2>/dev/null | grep -A99 "^##META" | tee $meta
 
 
    [ -s "$meta" ] || {
[ "$META" ] && {
OUT="$CIN"
#echo "[w] empty meta">&2
META "$@"
#CLEAN
exit
}
FAIL empty meta
    }
 
    [ "$META_REMOVE" ] && {
echo "[i] remove meta">&2
truncate -s-4096 "$CIN"
exit
    }
    grep -A99 "##META_FILE" $meta \
    | grep -B99 "##META-FILE" > $meta.2 || FAIL xz meta broken
 
    [ "$KRESCUE_CHK_END" ] && {
    tail -c15 $meta | grep -q "##KRESCUE##END" || FAIL xz meta not finished
    }
 
    [ "$META" ] && {
    OUT="$CIN"
 
    # for full replace, keep old behaviour (use only META_FILE block)
    if [ "$META_UPDATE" ]; then
        # keep full $meta (META_FILE + tags) for in-place update
        :
    else
        mv $meta.2 $meta
    fi
 
    [ "$S1" ] || {
#    echo "[i] get size"
    for S1 in $(grep UNPACKED_SIZE $meta.2); do
echo $S1 >/dev/null
    done
    }
 
    truncate -s-$BS "$OUT"
 
    if [ "$META_UPDATE" ]; then
        META_UPDATE_ONLY "$@"
    else
        META "$@"
    fi
    }
    CLEAN
    exit
}
 
[ "$IN" ] || FAIL input file not exist/defined
 
[ "$OUT" ] || {
    OUT2="$(readlink /proc/$$/fd/1)"
 
case $OUT2 in
    "")
    OUT="$IN.xz"
    ;;
    /dev/pts*)
    OUT="$IN.xz"
    ;;
    /dev/tty*)
    OUT="$IN.xz"
    ;;
    *)
#    *pipe:*)
#    OUT=/dev/stdout
    OUT="$OUT2"
    ;;
esac
 
}
 
[ -s "$OUT" ] && {
    echo "[w] $OUT exist already">&2
    [ "$FORCE" ] || FAIL plz use -f for rewrite it
}
 
S1=$(fsize "$IN")
 
 
S14=$((S1/8))
 
[ "$T" -gt 0 ] && \
S14=$((S1/T))
 
[ $S14 -lt $B ] && B=$S14
 
B="${B}M"
 
ARGS="$xz -T$T --block-size $B -k -c $OPTS"
 
echo "[i] xze compress $IN ($S1 bytes) > $OUT">&2
echo "# $ARGS ">&2
 
$ARGS "$IN" > "$OUT" || FAIL xz failed
 
 
S=
S2=
 
PAD "$OUT"
 
XZS=$($xz -l "$OUT")
echo "[i] XZ info
$XZS">&2
 
[ "$META" ] && {
META "$@"
exit
}
 
exit 0