Using save/ to keep downloads and partial progress

diff --git a/mkit.sh b/mkit.sh
index 8ebc8e7ece3108a9cef07b3083f66f745cb8df67..2fb62d0e5e20594bf7dff267ea2e4f2bbae05d12 100755 (executable)
--- a/mkit.sh
+++ b/mkit.sh
@@ -9,17 +9,28 @@
 
 set -e
 
-echo '### Step 1. Download original Packages file from the source'
-REPOS=( deb.devuan.org/{merged_daedalus_main,merged_ceres_main} )
+echo '### Step 1. Download original Packages files from sources'
+mkdir -p save
+[ -L save/save ] || ln -s . save/save
+MAPFILE=save/mapfile.txt
+DEPFILE=save/mapdepends.txt
+
+REPOS=(
+    deb.devuan.org/merged_daedalus_main
+    deb.devuan.org/merged_ceres_main
+)
 ARCH="amd64"
 MBR=dos # gpt
 
+: > $MAPFILE
+: > $DEPFILE
+
 for REPO in ${REPOS[@]} ; do
     W=( ${REPO//_/ } )
     SUITE=${W[1]}
     SECTION=${W[2]}
     
-    PKGFILE=${W[0]/\//_}_${SUITE}_${SECTION}_binary-${ARCH}_Packages
+    PKGFILE=save/${W[0]/\//_}_${SUITE}_${SECTION}_binary-${ARCH}_Packages
     if [ ! -r $PKGFILE ] ; then
        XZSRC="http://${W[0]}/dists/$SUITE/$SECTION/binary-$ARCH/Packages.xz"
        echo "$XZSRC"
@@ -27,16 +38,24 @@ for REPO in ${REPOS[@]} ; do
     fi
 
     echo '# Reduce Packages file into two maps for filename and depends'
-    echo "# ..creating mapdepends.txt and mapfile.txt"
-    awk '
-    BEGIN { print "###" >> "mapdepends.txt"; print "###" >> "mapfile.txt"; }
+    echo "# ..creating $MAPFILE and $DEPFILE"
+    awk -v MAP=$MAPFILE -v DEP=$DEPFILE '
+    BEGIN { print "###" >> MAP; print "###" >> DEP; }
 $1=="Package:" {P=$2; next}
-$1=="Pre-Depends:" {print P,$0 >> "mapdepends.txt";next }
-$1=="Depends:" {print P,$0 >> "mapdepends.txt";next }
-$1=="Filename:" {print P,$2 >> "mapfile.txt";next }
+$1=="Pre-Depends:" {print P,$0 >> DEP;next }
+$1=="Depends:" {print P,$0 >> DEP;next }
+$1=="Filename:" {print P,$2 >> MAP;next }
 ' ${PKGFILE}
 done
 
+# Append any additional saved deb files
+for DEB in save/*.deb ; do
+    P="${DEB%%_*}"
+    P="${P#save/}"
+    echo "$P save/$DEB" >> $MAPFILE
+    dpkg-deb -f "$DEB" Depends Pre-Depends | sed "s/^/$P/" >> $DEPFILE
+done
+
 # Function to find the filename for a given package in the given mapfile
 maplookup() {
     local FN=${3:-tail}
@@ -45,18 +64,48 @@ maplookup() {
     grep "^$1 " $2 | $FN -n 1 | sed 's/[^ ]* //'
 }
 
+# Function to clean up a "depends' line
+depclean() {
+    sed 's/^[^:]*: //;s/([^)]*)//g;s/|[^,]*/ /g;s/,/ /g;s/\s\+/ /g'
+}
+
+# Map all dependencies for a list of package
+#
+mapdeps() {
+    local D
+    local B=( )
+    local X=" $* "
+    for D in $* ; do
+       local A=(
+           $(grep "^$D Pre-Depends:" $DEPFILE | head -n 1 | depclean)
+           $(grep "^$D Depends:" $DEPFILE | head -n 1 | depclean)
+       )
+       for P in ${A[@]} ; do
+           [ -z "${X%%* $P *}" ] && continue
+           B=( $B $P )
+           X="$X$P "
+       done
+    done
+    if [ -z "$B" ] ; then
+       echo $X
+    else
+       mapdeps $X
+    fi
+}
+
 # Function to download a deb file and return its name
 debfile() {
     local FN=${2-tail}
-    local F="$(maplookup $1 mapfile.txt $FN)"
-    if [ ! -e "${F##*/}" ] ; then
+    local F="$(maplookup $1 $MAPFILE $FN)"
+    local P="${F##*/}"
+    if [ ! -e "save/$P" ] ; then
        for REPO in ${REPOS[@]} ; do
            echo "download http://${REPO%%_*}/$F" >&2
-           wget -q "http://${REPO%%_*}/$F" && break
-           ls -l $(basename $F) >&2
+           ( cd save && wget -q "http://${REPO%%_*}/$F" ) && break
+           ls -l save/$P >&2
        done || return 1
     fi
-    echo "${F##*/}"
+    echo "save/$P"
 }
 
 # Function to extract from a deb without executing and pre/post scripts
@@ -67,7 +116,7 @@ debextract() {
 
 # Deteremine which kernel to use; this is
 echo "# Determine kernel"
-KERNEL="$(maplookup linux-image-amd64 mapdepends.txt head | \
+KERNEL="$(maplookup linux-image-amd64 $DEPFILE head | \
                    sed 's/.*\(linux-image[^ ]*\).*/\1/')"
 echo "# ... $KERNEL"
 
@@ -78,11 +127,49 @@ echo '### Step 2. Create and populate the initrd, and packit up.'
 echo "# Create initrd filesystem"
 rm -fr initrd
 
-echo "# Extract busybox, and fluff it up"
-mkdir initrd
+# Helper function to copy a file or link, given full path
+# $1 = full pathname, $2 = (relative) root path
+copylinks() {
+    echo "copylinks $*" >&2
+    [ -e "initrd/$1" ] && return 0
+    mkdir -p "initrd${1%/*}"
+    if [ -L "$1" ] ; then
+       local L=$(readlink $2$1)
+       ln -s $L initrd$1
+       [ "${L:0:1}" = "/" ] || L="${1%/*}/$L"
+       copylinks $L $2
+    else
+       cp $2$1 initrd/$1
+    fi
+}
+
+# Helper function to copy dynamic binary and its libraries
+# $1 = full pathname, $2 = (relative) root path
+bincp() {
+    echo "bincp $*" >&2
+    [ -e "initrd/$1" ] && return 0
+    copylinks $1 $2
+    objdump -x $2$1 | while read A B ; do
+       [ "$A" == "NEEDED" ] || continue
+       local L="$(find ${2:-/} -name $B)"
+       [ -z "$L" ] && echo "MISSING $B" && exit 1
+       copylinks ${L#$2} $2
+    done
+}
+
+echo '# Include static vtoydump for Ventoy support'
+VTOYDUMP=../ventoy/vtoydump/vtoydump
+if [ -e "$VTOYDUMP" ] ; then
+    mkdir -p initrd/bin
+    cp "$VTOYDUMP" initrd/bin
+fi
+
+echo "# Extract static busybox, and fluff it up for more utilities"
+mkdir -p initrd
 debextract initrd $(debfile busybox-static head)
 for L in $(initrd/bin/busybox --listfull) ; do
     mkdir -p $(dirname initrd/$L)
+    [ -e "initrd/$L" ] && continue
     case "$L" in
        bin/busybox) : ;;
        usr/*) ln -s ../../bin/busybox initrd/$L ;;
@@ -97,6 +184,7 @@ echo "# .. and syslinux stuff if needed"
 if [ ! -d kernel ] ; then
     mkdir kernel
     debextract kernel $(debfile $KERNEL head)
+    debextract kernel $(debfile squashfs-tools head)
     debextract kernel $(debfile syslinux)
     debextract kernel $(debfile syslinux-common)
     debextract kernel $(debfile syslinux-efi)
@@ -114,10 +202,13 @@ MODULES=(
     crc32c_generic jbd2 mbcache crc16 ext4
     usb-storage usbcore usb-common xhci-pci xhci-hcd
     isofs exfat loop
+    efivarfs
     # input
     psmouse evdev
     # network
     e1000
+    # overlay
+    overlay squashfs
 )
 MOODLES=""
 B=$(pwd)
@@ -184,7 +275,6 @@ mcopy $IMG \
 mcopy $IMG syslinux-legacy.cfg ::/syslinux.cfg
 
 syslinux --install --offset=${IMG#*@@} bootimage.raw
-
 mmd $IMG ::/EFI
 mmd $IMG ::/EFI/BOOT
 mcopy $IMG kernel/usr/lib/SYSLINUX.EFI/efi64/syslinux.efi \

diff --git a/packiso.sh b/packiso.sh
index b6558f2793626ac9a8cfeb3e19762b75608f5b56..f3e7ee95e3fe2055a8f533ece736d4bcc838d0dc 100755 (executable)
--- a/packiso.sh
+++ b/packiso.sh
@@ -13,6 +13,8 @@ echo "# Prepare ISO content tree unless it's already set up"
 if [ ! -d $ISOTREE ] ; then
     mkdir $ISOTREE
 
+    [ -d extra ] && rsync -a extra/. $ISOTREE/.
+
     # Determine start and size of first parition
     X=( $(sfdisk -l -ostart,sectors bootimage.raw | tail -n 2) )
     dd if=bootimage.raw of=$P1 skip=${X[0]} count=${X[1]}
@@ -30,7 +32,6 @@ if [ ! -d $ISOTREE ] ; then
 
     ## put splash, vmlinux and initrd as siblings
     mcopy -i $P1 ::/splash.png ::/vmlinuz ::/initrd.gz $ISOTREE/boot/isolinux
-    
 fi
 
 [ -r $ISO ] && rm $ISO