From d63575971cf20f98ad57ba778ed2faa626042871 Mon Sep 17 00:00:00 2001 From: Ralph Ronnquist Date: Thu, 28 Dec 2023 17:46:03 +1100 Subject: [PATCH] Using save/ to keep downloads and partial progress --- mkit.sh | 126 +++++++++++++++++++++++++++++++++++++++++++++-------- packiso.sh | 3 +- 2 files changed, 110 insertions(+), 19 deletions(-) diff --git a/mkit.sh b/mkit.sh index 8ebc8e7..2fb62d0 100755 --- a/mkit.sh +++ b/mkit.sh @@ -9,17 +9,28 @@ set -e -echo '### Step 1. Download original Packages file from the source' -REPOS=( deb.devuan.org/{merged_daedalus_main,merged_ceres_main} ) +echo '### Step 1. Download original Packages files from sources' +mkdir -p save +[ -L save/save ] || ln -s . save/save +MAPFILE=save/mapfile.txt +DEPFILE=save/mapdepends.txt + +REPOS=( + deb.devuan.org/merged_daedalus_main + deb.devuan.org/merged_ceres_main +) ARCH="amd64" MBR=dos # gpt +: > $MAPFILE +: > $DEPFILE + for REPO in ${REPOS[@]} ; do W=( ${REPO//_/ } ) SUITE=${W[1]} SECTION=${W[2]} - PKGFILE=${W[0]/\//_}_${SUITE}_${SECTION}_binary-${ARCH}_Packages + PKGFILE=save/${W[0]/\//_}_${SUITE}_${SECTION}_binary-${ARCH}_Packages if [ ! -r $PKGFILE ] ; then XZSRC="http://${W[0]}/dists/$SUITE/$SECTION/binary-$ARCH/Packages.xz" echo "$XZSRC" @@ -27,16 +38,24 @@ for REPO in ${REPOS[@]} ; do fi echo '# Reduce Packages file into two maps for filename and depends' - echo "# ..creating mapdepends.txt and mapfile.txt" - awk ' - BEGIN { print "###" >> "mapdepends.txt"; print "###" >> "mapfile.txt"; } + echo "# ..creating $MAPFILE and $DEPFILE" + awk -v MAP=$MAPFILE -v DEP=$DEPFILE ' + BEGIN { print "###" >> MAP; print "###" >> DEP; } $1=="Package:" {P=$2; next} -$1=="Pre-Depends:" {print P,$0 >> "mapdepends.txt";next } -$1=="Depends:" {print P,$0 >> "mapdepends.txt";next } -$1=="Filename:" {print P,$2 >> "mapfile.txt";next } +$1=="Pre-Depends:" {print P,$0 >> DEP;next } +$1=="Depends:" {print P,$0 >> DEP;next } +$1=="Filename:" {print P,$2 >> MAP;next } ' ${PKGFILE} done +# Append any additional saved deb files +for DEB in save/*.deb ; do + P="${DEB%%_*}" + P="${P#save/}" + echo "$P save/$DEB" >> $MAPFILE + dpkg-deb -f "$DEB" Depends Pre-Depends | sed "s/^/$P/" >> $DEPFILE +done + # Function to find the filename for a given package in the given mapfile maplookup() { local FN=${3:-tail} @@ -45,18 +64,48 @@ maplookup() { grep "^$1 " $2 | $FN -n 1 | sed 's/[^ ]* //' } +# Function to clean up a "depends' line +depclean() { + sed 's/^[^:]*: //;s/([^)]*)//g;s/|[^,]*/ /g;s/,/ /g;s/\s\+/ /g' +} + +# Map all dependencies for a list of package +# +mapdeps() { + local D + local B=( ) + local X=" $* " + for D in $* ; do + local A=( + $(grep "^$D Pre-Depends:" $DEPFILE | head -n 1 | depclean) + $(grep "^$D Depends:" $DEPFILE | head -n 1 | depclean) + ) + for P in ${A[@]} ; do + [ -z "${X%%* $P *}" ] && continue + B=( $B $P ) + X="$X$P " + done + done + if [ -z "$B" ] ; then + echo $X + else + mapdeps $X + fi +} + # Function to download a deb file and return its name debfile() { local FN=${2-tail} - local F="$(maplookup $1 mapfile.txt $FN)" - if [ ! -e "${F##*/}" ] ; then + local F="$(maplookup $1 $MAPFILE $FN)" + local P="${F##*/}" + if [ ! -e "save/$P" ] ; then for REPO in ${REPOS[@]} ; do echo "download http://${REPO%%_*}/$F" >&2 - wget -q "http://${REPO%%_*}/$F" && break - ls -l $(basename $F) >&2 + ( cd save && wget -q "http://${REPO%%_*}/$F" ) && break + ls -l save/$P >&2 done || return 1 fi - echo "${F##*/}" + echo "save/$P" } # Function to extract from a deb without executing and pre/post scripts @@ -67,7 +116,7 @@ debextract() { # Deteremine which kernel to use; this is echo "# Determine kernel" -KERNEL="$(maplookup linux-image-amd64 mapdepends.txt head | \ +KERNEL="$(maplookup linux-image-amd64 $DEPFILE head | \ sed 's/.*\(linux-image[^ ]*\).*/\1/')" echo "# ... $KERNEL" @@ -78,11 +127,49 @@ echo '### Step 2. Create and populate the initrd, and packit up.' echo "# Create initrd filesystem" rm -fr initrd -echo "# Extract busybox, and fluff it up" -mkdir initrd +# Helper function to copy a file or link, given full path +# $1 = full pathname, $2 = (relative) root path +copylinks() { + echo "copylinks $*" >&2 + [ -e "initrd/$1" ] && return 0 + mkdir -p "initrd${1%/*}" + if [ -L "$1" ] ; then + local L=$(readlink $2$1) + ln -s $L initrd$1 + [ "${L:0:1}" = "/" ] || L="${1%/*}/$L" + copylinks $L $2 + else + cp $2$1 initrd/$1 + fi +} + +# Helper function to copy dynamic binary and its libraries +# $1 = full pathname, $2 = (relative) root path +bincp() { + echo "bincp $*" >&2 + [ -e "initrd/$1" ] && return 0 + copylinks $1 $2 + objdump -x $2$1 | while read A B ; do + [ "$A" == "NEEDED" ] || continue + local L="$(find ${2:-/} -name $B)" + [ -z "$L" ] && echo "MISSING $B" && exit 1 + copylinks ${L#$2} $2 + done +} + +echo '# Include static vtoydump for Ventoy support' +VTOYDUMP=../ventoy/vtoydump/vtoydump +if [ -e "$VTOYDUMP" ] ; then + mkdir -p initrd/bin + cp "$VTOYDUMP" initrd/bin +fi + +echo "# Extract static busybox, and fluff it up for more utilities" +mkdir -p initrd debextract initrd $(debfile busybox-static head) for L in $(initrd/bin/busybox --listfull) ; do mkdir -p $(dirname initrd/$L) + [ -e "initrd/$L" ] && continue case "$L" in bin/busybox) : ;; usr/*) ln -s ../../bin/busybox initrd/$L ;; @@ -97,6 +184,7 @@ echo "# .. and syslinux stuff if needed" if [ ! -d kernel ] ; then mkdir kernel debextract kernel $(debfile $KERNEL head) + debextract kernel $(debfile squashfs-tools head) debextract kernel $(debfile syslinux) debextract kernel $(debfile syslinux-common) debextract kernel $(debfile syslinux-efi) @@ -114,10 +202,13 @@ MODULES=( crc32c_generic jbd2 mbcache crc16 ext4 usb-storage usbcore usb-common xhci-pci xhci-hcd isofs exfat loop + efivarfs # input psmouse evdev # network e1000 + # overlay + overlay squashfs ) MOODLES="" B=$(pwd) @@ -184,7 +275,6 @@ mcopy $IMG \ mcopy $IMG syslinux-legacy.cfg ::/syslinux.cfg syslinux --install --offset=${IMG#*@@} bootimage.raw - mmd $IMG ::/EFI mmd $IMG ::/EFI/BOOT mcopy $IMG kernel/usr/lib/SYSLINUX.EFI/efi64/syslinux.efi \ diff --git a/packiso.sh b/packiso.sh index b6558f2..f3e7ee9 100755 --- a/packiso.sh +++ b/packiso.sh @@ -13,6 +13,8 @@ echo "# Prepare ISO content tree unless it's already set up" if [ ! -d $ISOTREE ] ; then mkdir $ISOTREE + [ -d extra ] && rsync -a extra/. $ISOTREE/. + # Determine start and size of first parition X=( $(sfdisk -l -ostart,sectors bootimage.raw | tail -n 2) ) dd if=bootimage.raw of=$P1 skip=${X[0]} count=${X[1]} @@ -30,7 +32,6 @@ if [ ! -d $ISOTREE ] ; then ## put splash, vmlinux and initrd as siblings mcopy -i $P1 ::/splash.png ::/vmlinuz ::/initrd.gz $ISOTREE/boot/isolinux - fi [ -r $ISO ] && rm $ISO -- 2.39.2