Improve gentoo-2024.8 bootstrap by a bunch

gentoo-2024.8/gentoo.txt

@@ -1,33 +1,28 @@
 # vim:set ft=sh:
 
-git clone --depth=1 --recursive https://github.com/fosslinux/live-bootstrap
+git clone -b 1.0 --depth=1 --recursive https://github.com/fosslinux/live-bootstrap
 cd live-bootstrap
 ./rootfs.py -c --external-sources --cores $(nproc)
 umount target/dev/shm  # double mounted?
 umount target/dev/shm target/sys target/proc target/tmp
 
 # Optional: back up the system
-env -i chroot target tar --sort=name -cf /target.tar /
+env -i chroot target tar --exclude='/external' --sort=name -cf /target.tar /
 env -i chroot target bzip2 -9v /target.tar
 
 # Goal:
 # install cross compiler in /cross
 # cross compiled system in /gentoo
 
-mount -t proc proc target/proc
+cd target
-mount -t sysfs sysfs target/sys
+mount -t proc proc proc
-mount -t devtmpfs devtmpfs target/dev
+mount -t sysfs sysfs sys
-mount -t devpts devpts target/dev/pts
+mount -t devtmpfs devtmpfs dev
-env -i chroot target
+mount -t devpts devpts dev/pts
+env -i TERM="$TERM" chroot .
 source /steps/env
 cd /tmp
 
-curl -LO https://github.com/ninja-build/ninja/archive/v1.11.1/ninja-1.11.1.tar.gz
-curl -LO https://github.com/mesonbuild/meson/releases/download/1.4.1/meson-1.4.1.tar.gz
-curl -LO http://gitweb.gentoo.org/proj/portage.git/snapshot/portage-3.0.65.tar.bz2
-curl -LO https://github.com/plougher/squashfs-tools/archive/refs/tags/4.6.1/squashfs-tools-4.6.1.tar.gz
-curl -LO http://distfiles.gentoo.org/snapshots/squashfs/gentoo-20240801.xz.sqfs
-
 cat > portage.patch << 'EOF'
 +++ b/lib/portage/util/compression_probe.py
 @@ -1,13 +1,13 @@
@@ -56,26 +51,9 @@ cat > portage.patch << 'EOF'
  }
 EOF
 
-tar xf ninja-1.11.1.tar.gz
+curl -LO http://gitweb.gentoo.org/proj/portage.git/snapshot/portage-3.0.65.tar.bz2
-cd ninja-1.11.1
+curl -LO https://github.com/plougher/squashfs-tools/archive/refs/tags/4.6.1/squashfs-tools-4.6.1.tar.gz
-./configure.py --bootstrap
+curl -LO http://distfiles.gentoo.org/snapshots/squashfs/gentoo-20240801.xz.sqfs
-install -m755 ninja /usr/bin
-cd ..
-rm -rf ninja-1.11.1
-
-tar xf meson-1.4.1.tar.gz
-cd meson-1.4.1
-python setup.py install
-cd ..
-rm -rf meson-1.4.1
-
-tar xf portage-3.0.65.tar.bz2
-cd portage-3.0.65
-patch -p1 -i ../portage.patch
-meson setup build --prefix /usr
-meson install -C build
-cd ..
-rm -rf portage-3.0.65
 
 tar xf squashfs-tools-4.6.1.tar.gz
 cd squashfs-tools-4.6.1
@@ -90,6 +68,9 @@ mkdir -p /var/db/repos
 rm -rf /var/db/repos/gentoo
 mv squashfs-root /var/db/repos/gentoo
 
+tar xf portage-3.0.65.tar.bz2
+ln -sf portage-3.0.65 portage
+
 # Configure portage
 echo 'portage:x:250:250:portage:/var/tmp/portage:/bin/false' >> /etc/passwd
 echo 'portage::250:portage' >> /etc/group
@@ -105,38 +86,71 @@ ACCEPT_KEYWORDS="$ARCH"
 CHOST="i386-unknown-linux-musl"
 LIBDIR_x86="lib/i386-unknown-linux-musl"
 PKG_CONFIG_PATH="/usr/lib/i386-unknown-linux-musl/pkgconfig"
-PYTHON_COMPAT_OVERRIDE="python3_11"
 IUSE_IMPLICIT="kernel_linux elibc_glibc elibc_musl prefix prefix-guest"
-USE="kernel_linux elibc_musl python_targets_python3_11"
+USE="kernel_linux elibc_musl python_targets_python3_12"
+EOF
+cat > /etc/portage/package.use << 'EOF'
+dev-lang/python -readline -ncurses
 EOF
 
-# Install extra dependencies
+# Install dependencies to make emerge work nicely
-FETCHCOMMAND="curl -o \"\${DISTDIR}/\${FILE}\" -L \"\${URI}\""
+FETCHCOMMAND='curl -o "${DISTDIR}/${FILE}" -L "${URI}"'
-FETCHCOMMAND="$FETCHCOMMAND" MAKEOPTS=-j1 emerge -O1 dev-build/make
+FETCHCOMMAND="$FETCHCOMMAND" MAKEOPTS=-j1 ./portage/bin/emerge -O1 app-arch/lzip
-FETCHCOMMAND="$FETCHCOMMAND" emerge -O1 app-arch/lzip
+FETCHCOMMAND="$FETCHCOMMAND" MAKEOPTS=-j1 ./portage/bin/emerge -O1 dev-build/make
-FETCHCOMMAND="$FETCHCOMMAND" emerge -O1 net-misc/wget
+FETCHCOMMAND="$FETCHCOMMAND" ./portage/bin/emerge -O1 net-misc/wget
+
+# Upgrade python so we can use it to cross-compile later on
+./portage/bin/emerge -O1 dev-build/autoconf
+./portage/bin/emerge -O1 dev-build/autoconf-wrapper
+./portage/bin/emerge -O1 dev-build/automake
+./portage/bin/emerge -O1 dev-build/automake-wrapper
+./portage/bin/emerge -O1 sys-apps/gentoo-functions
+./portage/bin/emerge -O1 app-portage/elt-patches
+./portage/bin/emerge -O1 dev-libs/mpdecimal
+./portage/bin/emerge -O1 dev-libs/expat
+mv /bin/bzip2 /bin/bzip2-reference
+ln -s bzip2-reference /bin/bzip2
+./portage/bin/emerge -O1 app-arch/bzip2
+./portage/bin/emerge -O1 dev-lang/python
+./portage/bin/emerge -O1 dev-lang/python-exec
+
+# Install the rest of the dependencies for meson
+./portage/bin/emerge -O1 dev-python/gpep517
+./portage/bin/emerge -O1 app-arch/unzip
+./portage/bin/emerge -O1 dev-python/installer
+./portage/bin/emerge -O1 dev-python/flit-core
+./portage/bin/emerge -O1 dev-python/packaging
+./portage/bin/emerge -O1 dev-python/more-itertools
+./portage/bin/emerge -O1 dev-python/ordered-set
+./portage/bin/emerge -O1 dev-python/jaraco-text
+./portage/bin/emerge -O1 dev-python/jaraco-functools
+./portage/bin/emerge -O1 dev-python/jaraco-context
+./portage/bin/emerge -O1 dev-python/wheel
+./portage/bin/emerge -O1 dev-python/setuptools
+./portage/bin/emerge -O1 dev-build/meson
+./portage/bin/emerge -O1 dev-build/ninja
+
+# Finally install portage itself
+./portage/bin/emerge -O1 sys-apps/portage
+
+# Install pax-utils to allow stripping binaries
+emerge -O1 dev-build/meson-format-array
+emerge -O1 app-misc/pax-utils
 
 # Fix "find" warnings in emerge
 emerge -O1 sys-apps/findutils
 
-# Set up python for dev-build/meson-format-array
+# Install additional BDEPENDs
-py=python3.11
+emerge -O1 sys-devel/binutils-config  # sys-devel/binutils
-mkdir -p /usr/lib/python-exec/$py
+emerge -O1 sys-devel/gcc-config  # sys-devel/gcc
-ln -sv python3 /usr/lib/python-exec/$py/python
+emerge -O1 net-misc/rsync  # sys-kernel/linux-headers
-ln -svr /usr/bin/$py /usr/lib/python-exec/$py/python3
+emerge -O1 dev-util/pkgconf  # dev-lang/python requires --keep-system-libs option when cross compiling
-emerge -O1 dev-lang/python-exec
 
-# Allow stripping binaries by installing "scanelf"
+# Add cross compiler to PATH
-emerge -O1 dev-build/meson-format-array
+cat > /etc/env.d/50baselayout << 'EOF'
-emerge -O1 app-misc/pax-utils
+PATH=/cross/usr/bin:/usr/bin
-
+EOF
-# Install tooling around autoconf/automake for sys-devel/binutils and sys-devel/gcc
+env-update
-emerge -O1 sys-devel/gnuconfig sys-apps/gentoo-functions app-portage/elt-patches sys-devel/binutils-config sys-devel/gcc-config
-
-# Install rsync for sys-kernel/linux-headers
-emerge -O1 dev-build/autoconf dev-build/autoconf-wrapper
-emerge -O1 dev-build/automake dev-build/automake-wrapper
-emerge -O1 net-misc/rsync
 
 # Set up cross compiler
 mkdir -p /cross/etc/portage
@@ -149,61 +163,23 @@ LIBDIR_amd64="lib64"
 DEFAULT_ABI="amd64"
 MULTILIB_ABIS="amd64 x86"
 EOF
+cat > /cross/etc/portage/package.use << 'EOF'
+sys-devel/gcc -sanitize -fortran
+EOF
 mkdir -p /cross/etc/portage/env/sys-devel
 cat > /cross/etc/portage/env/sys-devel/gcc << 'EOF'
 EXTRA_ECONF='--with-sysroot=$EPREFIX/usr/$CTARGET'
 EOF
-cat > /cross/etc/portage/package.use << 'EOF'
-sys-devel/gcc -sanitize
-EOF
-#EXTRA_EMAKE='MAKE=make MAKE+=libsuffix=../lib64'
-
-# Set path for emerge builds
-cat > /etc/env.d/50baselayout << 'EOF'
-PATH=/cross/usr/bin:/usr/bin
-ROOTPATH=/cross/usr/bin:/usr/bin
-EOF
-env-update
 
 # TODO: Build sys-libs/glibc in /gentoo instead, to avoid extra rebuilding
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross emerge -O1 sys-devel/binutils
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross USE='headers-only' emerge -O1 sys-kernel/linux-headers
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross USE='headers-only -multilib' emerge -O1 sys-libs/glibc 
-PORTAGE_CONFIGROOT=/cross EPREFIX=/cross USE='-cxx -fortran' emerge -O1 sys-devel/gcc
+PORTAGE_CONFIGROOT=/cross EPREFIX=/cross USE='-cxx -openmp' emerge -O1 sys-devel/gcc
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross emerge -O1 sys-kernel/linux-headers
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross emerge -O1 sys-libs/glibc
 PORTAGE_CONFIGROOT=/cross EPREFIX=/cross emerge -O1 sys-devel/gcc
 
-# Get extra BDEPENDs for the final system
-emerge -O1 dev-util/pkgconf  # dev-lang/python requires --keep-system-libs
-emerge -O1 dev-libs/mpdecimal  # dev-lang/python (host)
-emerge -O1 dev-libs/expat  # dev-lang/python (host)
-# TODO: verify bzip2 dep
-mv /bin/bzip2 /bin/bzip2-reference
-ln -s bzip2-reference /bin/bzip2
-emerge -O1 app-arch/bzip2  # dev-lang/python (build)
-# TODO: move python build so we don't need python-exec hack
-#       ... or switch to only using python3.11
-USE='-*' emerge -O1 dev-lang/python  # cross compiling python programs needs same version
-emerge -O1 dev-lang/python-exec-conf
-USE='python_targets_python3_12' emerge -O1 dev-lang/python-exec
-
-# Allow meson to still be ran despite python upgrade
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/gpep517
-emerge -O1 app-arch/unzip
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/installer
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/flit-core
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/packaging
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/more-itertools
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/ordered-set
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/jaraco-text
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/jaraco-functools
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/jaraco-context
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/wheel
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-python/setuptools
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 dev-build/meson
-PYTHON_COMPAT_OVERRIDE=python3_12 emerge -O1 sys-apps/portage
-
 # Reconfigure cross toolchain for final system
 cat > /cross/usr/lib/gcc/x86_64-bootstrap-linux-gnu/specs << 'EOF'
 *link:
@@ -217,7 +193,6 @@ exec /cross/usr/i386-unknown-linux-musl/x86_64-bootstrap-linux-gnu/gcc-bin/*/x86
 EOF
 chmod +x /cross/usr/bin/x86_64-bootstrap-linux-gnu-$tool
 done
-# TODO: Do I need this?
 cat > /cross/usr/bin/x86_64-bootstrap-linux-gnu-pkg-config << 'EOF'
 #!/bin/sh
 export PKG_CONFIG_SYSROOT_DIR=/gentoo
@@ -230,19 +205,18 @@ EOF
 chmod +x /cross/usr/bin/x86_64-bootstrap-linux-gnu-pkg-config
 
 # Compile final system (hopefully)
-# TODO: Try to get rid of CFLAGS_x86 override
+mkdir -p /gentoo.cfg/etc/portage
-mkdir -p /gentoo/etc/portage
+ln -sf ../../../var/db/repos/gentoo/profiles/default/linux/amd64/23.0 /gentoo.cfg/etc/portage/make.profile
-ln -sf /var/db/repos/gentoo/profiles/default/linux/amd64/23.0 /gentoo/etc/portage/make.profile
+cat > /gentoo.cfg/etc/portage/make.conf << 'EOF'
-cat > /gentoo/etc/portage/make.conf << 'EOF'
 FEATURES="-news -sandbox -usersandbox -pid-sandbox -parallel-fetch"
 BINPKG_COMPRESS="bzip2"
 CBUILD="i386-unknown-linux-musl"
 CHOST="x86_64-bootstrap-linux-gnu"
-CFLAGS_x86=-m32
+CFLAGS_x86="$CFLAGS_x86 -msse"  # bug 937637
-USE="-* build python_targets_$PYTHON_SINGLE_TARGET"
+USE="-* build openmp python_targets_python3_12"
 CONFIG_SITE="$PORTAGE_CONFIGROOT/etc/portage/config.site"
 EOF
-cat > /gentoo/etc/portage/config.site << 'EOF'
+cat > /gentoo.cfg/etc/portage/config.site << 'EOF'
 if [ "${CBUILD:-${CHOST}}" != "${CHOST}" ]; then
 # Settings grabbed from crossdev
 ac_cv_file__dev_ptmx=yes
@@ -250,12 +224,11 @@ ac_cv_file__dev_ptc=no
 fi
 EOF
 
-# TODO: Build gcc with openmp (required by zstd)
+PORTAGE_CONFIGROOT=/gentoo.cfg ROOT=/gentoo SYSROOT=/gentoo emerge -O1n \
-
-PORTAGE_CONFIGROOT=/gentoo ROOT=/gentoo SYSROOT=/gentoo emerge -O1n \
     sys-apps/baselayout \
     sys-kernel/linux-headers \
     sys-libs/glibc \
+    sys-libs/zlib \
     sys-devel/binutils \
     dev-libs/gmp \
     dev-libs/mpfr \
@@ -269,7 +242,6 @@ PORTAGE_CONFIGROOT=/gentoo ROOT=/gentoo SYSROOT=/gentoo emerge -O1n \
     dev-libs/mpdecimal \
     sys-apps/util-linux \
     sys-libs/libxcrypt \
-    sys-libs/zlib \
     dev-lang/python:3.12 \
     \
     dev-lang/python-exec \
@@ -281,25 +253,59 @@ PORTAGE_CONFIGROOT=/gentoo ROOT=/gentoo SYSROOT=/gentoo emerge -O1n \
     \
     sys-apps/coreutils \
     sys-apps/findutils \
-    app-arch/tar \
-    app-arch/gzip \
     sys-apps/sed \
     sys-apps/grep \
-    sys-devel/patch \
     sys-apps/gawk \
+    sys-devel/patch \
+    app-arch/tar \
+    app-arch/gzip \
     dev-build/make
 
 # TODO: Un-hardcode python 3.12
-# TODO: Avoid gzip, sed and grep?
+# TODO: Avoid gzip?
+
+# Set up final system
+mkdir -p /gentoo/etc/portage
+ln -sf ../../var/db/repos/gentoo/profiles/default/linux/amd64/23.0 /gentoo/etc/portage/make.profile
+rsync -aP /var/db/repos/ /gentoo/var/db/repos
+echo 'nameserver 1.1.1.1' > /gentoo/etc/resolv.conf
+echo 'C.UTF8 UTF-8' > /gentoo/etc/locale.gen
 
 # Fetch some extra source code
 DISTDIR=/gentoo/var/cache/distfiles \
 PORTAGE_CONFIGROOT=/gentoo ROOT=/gentoo SYSROOT=/gentoo emerge -O1f \
-    app-misc/ca-certificates \
+    dev-util/pkgconf \
     dev-lang/perl \
     dev-libs/openssl \
-    dev-util/pkgconf \
+    net-misc/wget \
-    net-misc/wget
+    app-misc/ca-certificates
 
 # From here you just chroot into /gentoo, clean /etc/portage (leaving only make.profile), install wget, and run "emerge -DN @world", untangling dependencies until it works
-# TODO: Draw the rest of the fucking owl
+# This is an example:
+# TODO: Avoid weird gcc rebuild...
+#       This part can be improved a lot by cross-compiling more stuff
+emerge -O1 dev-util/pkgconf
+USE=-gdbm emerge -O1 dev-lang/perl
+emerge -O1 dev-libs/openssl
+MAKEINFO=true emerge -O1 net-misc/wget
+emerge -O1 app-misc/ca-certificates
+emerge -O1 dev-build/ninja
+emerge -O1 app-alternatives/ninja
+emerge -O1 sys-apps/gentoo-functions
+emerge -O1 app-portage/elt-patches
+emerge -O1 sys-devel/m4
+MAKEINFO=true emerge -O1 sys-devel/flex
+emerge -O1 sys-apps/diffutils
+emerge -O1 sys-devel/gcc-config
+USE='-* openmp' EXTRA_ECONF=--disable-bootstrap emerge -O1 sys-devel/gcc
+emerge -O1 dev-build/ninja  # avoid segfault
+emerge -O1 sys-devel/bison
+USE='-* cet' emerge -O1 sys-libs/glibc
+emerge -O1 app-alternatives/yacc
+emerge -O1 app-alternatives/lex
+rm -f /var/run/bootstrap-progress && USE=-nls /var/db/repos/gentoo/scripts/bootstrap.sh
+USE='-* openmp' EXTRA_ECONF=--disable-bootstrap emerge -O1 sys-devel/gcc  # enable openmp again...
+USE='-nls -pam -su -http2' emerge -1 net-misc/rsync
+USE='-nls -pam -su -http2' emerge -e @system
+emerge -DN @system
+emerge -c