Bootstrapping gentoo in 2024

gentoo-2024/gentoo.txt

@@ -0,0 +1,217 @@
+# vim:set textwidth=0:
+# This guide uses live-bootstrap instead of the old chain
+
+# live-bootstrap version: 34cb6758d241b92c3c59e61f0a1d62f86f0c36c1 (Tue Feb 13 10:08:37 2024 +0000)
+# LFS version: 12.0
+# Gentoo version: 20240201
+
+# Do the live-bootstrap
+# Every step of this process is checksummed, so you can be relatively sure it's good when done
+cd live-bootstrap
+./download-distfiles.sh
+sudo ./rootfs.py --external-sources -q --cores 8
+cd ..
+
+# Prepare lfs sources
+mkdir lfs-sources
+wget -O lfs-sources/LFS-BOOK-12.0-NOCHUNKS.html https://linuxfromscratch.org/lfs/downloads/12.0/LFS-BOOK-12.0-NOCHUNKS.html
+wget -O lfs-sources/wget-list-sysv https://linuxfromscratch.org/lfs/downloads/12.0/wget-list-sysv
+wget -O lfs-sources/md5sums https://linuxfromscratch.org/lfs/downloads/12.0/md5sums
+wget --input-file=lfs-sources/wget-list-sysv --continue --directory-prefix=lfs-sources
+( cd lfs-sources; md5sum -c md5sums )
+
+# Prepare gentoo sources
+mkdir gentoo-sources
+wget --directory-prefix=gentoo-distfiles http://distfiles.gentoo.org/snapshots/squashfs/gentoo-20240201.xz.sqfs
+wget --input-file=wget-list-gentoo --continue --directory-prefix=gentoo-distfiles
+# md5sum: 35da573f5c622dd813adf462e8d046b7 *gentoo-20240201.xz.sqfs
+# The rest of the distfiles integrity is verified using manifests in the repo.
+
+# Install the live-bootstrap system into a new root
+sudo mkdir -p sys/mnt sys/dev sys/proc sys/sys sys/tmp sys/root
+sudo mount live-bootstrap/target/external.img -o offset=1048576 sys/mnt
+sudo rsync -avP sys/mnt/usr sys/mnt/etc sys/
+sudo umount sys/mnt
+
+sudo ln -sf usr/bin sys/bin
+sudo ln -sf usr/sbin sys/sbin
+sudo ln -sf usr/lib sys/lib
+
+sudo mkdir -p sys/mnt/lfs
+sudo rsync -rvP lfs-sources/ sys/mnt/lfs/sources
+sudo rsync -rvP gentoo-distfiles/ sys/mnt/lfs/distfiles
+
+sudo mount -t devtmpfs devtmpfs sys/dev
+sudo mount -t devpts devpts sys/dev/pts
+sudo mount -t proc proc sys/proc
+sudo mount -t sysfs sysfs sys/sys
+sudo chroot sys /usr/bin/env -i HOME=/root TERM="$TERM" LFS=/mnt/lfs /bin/bash -l
+
+# Follow LFS chapter 4,5,6,7
+# Skip chapter 4.3
+
+# Chroot into the system as detailed in chapter 7
+
+# Install the following additional software from the chapter 8 (don't run tests):
+# Chapter 8.6: zlib-1.2.13
+# Chapter 8.7: bzip2-1.0.8
+# Chapter 8.14: flex-2.6.4 (for gcc)
+# Chapter 8.28: pkgconf-2.0.1
+# Chapter 8.39: expat-2.5.0
+# Chapter 8.47: openssl-3.1.2 (for wget)
+# Chapter 8.50: libffi-3.4.4
+# Chapter 8.51: python-3.11.4 (required for pip)
+# Chapter 8.52: flit-core-3.9.0
+# Chapter 8.53: wheel-0.41.1
+# Chapter 8.54: ninja-1.11.1
+# Chapter 8.55: meson-1.2.1 (for portage)
+
+# This is a good place save the current system using the instructions in
+# chapter 7.13. From here on out, use chapters 7.3 and 7.4 to enter the chroot.
+
+# Symlink python (needed for portage)
+ln -sv python3 /usr/bin/python
+
+# Install portage
+meson setup --prefix /usr build
+meson install -C build
+
+# LFS is a merged-usr system, where gentoo isn't, so unlink the directories
+# and create a minimum amount of symlinks
+rm /bin /sbin /lib
+/usr/lib/ld-linux-x86-64.so.2 /usr/bin/mkdir /bin /sbin /lib
+/usr/lib/ld-linux-x86-64.so.2 /usr/bin/ln -svr /usr/lib/ld-linux-x86-64.so.2 /lib
+ln -svr /usr/bin/sh /bin
+ln -svr /usr/bin/bash /bin
+ln -svr /usr/bin/cpp /lib
+ln -svr /usr/sbin/ldconfig /sbin
+
+# Configure portage
+mkdir -p /var/db/repos/gentoo
+mount /distfiles/gentoo-*.sqfs /mnt
+cp -avT /mnt /var/db/repos/gentoo
+umount /mnt
+
+echo 'portage:x:250:250:portage:/var/tmp/portage:/bin/false' >> /etc/passwd
+echo 'portage::250:portage' >> /etc/group
+echo 'nameserver 1.1.1.1' > /etc/resolv.conf
+echo 'C.UTF-8 UTF-8' > /etc/locale.gen
+
+mkdir -p /etc/portage/profile
+ln -svr /var/db/repos/gentoo/profiles/default/linux/amd64/17.1 /etc/portage/make.profile
+cat > /etc/portage/make.conf << 'EOF'
+FEATURES='-news -pid-sandbox'
+CONFIG_PROTECT='-* /etc/locale.gen'
+USE='-nls'
+EOF
+cat > /etc/portage/profile/package.use.force << 'EOF'
+sys-devel/gcc -cxx
+EOF
+
+# Install a few more tools using portage to avoid spelling out instructions
+GENTOO_MIRRORS=/distfiles emerge -O1 net-misc/wget
+GENTOO_MIRRORS=/distfiles PYTHON_COMPAT_OVERRIDE=python3_11 emerge -O1 app-misc/ca-certificates
+USE=-acl emerge -O1 net-misc/rsync
+
+# Pretend none of that ever happened
+rm -rf /var/lib/portage /var/db/pkg /var/cache/edb /var/log/emerge.log /var/log/portage
+
+# Install baselayout
+# This ensures that the library paths and other env vars are set correctly
+# and portage picks them up (it sources /etc/profile for every build).
+emerge -O1 sys-apps/baselayout
+source /etc/profile
+
+# We need to bootstrap a multilib system
+# For that we're going to abuse portage's semi-support for cross-compilation,
+# and do something akin to what both LFS and crossdev do
+# Even when not bootstrapping multilib, this isn't a bad way to install
+# the gentoo toolchain while avoiding other differences, like multilib paths
+
+# TODO: Try building gcc without the glibc headers (toolchain.eclass supports this)
+#       and bootstrapping glibc straight into /
+
+# Install core gentoo toolchain support
+emerge -O1 sys-apps/gentoo-functions app-portage/elt-patches sys-devel/gnuconfig
+
+# Bootstrap glibc headers
+CTARGET=x86_64-bootstrap-linux-gnu USE=headers-only emerge -O1 sys-kernel/linux-headers
+CTARGET=x86_64-bootstrap-linux-gnu USE=headers-only PYTHON_COMPAT_OVERRIDE=python3_11 emerge -O1 sys-libs/glibc
+
+# Install the gentoo cross compiler
+emerge -O1 dev-libs/gmp dev-libs/mpfr dev-libs/mpc
+emerge -O1 sys-devel/binutils-config sys-devel/gcc-config
+CTARGET=x86_64-bootstrap-linux-gnu emerge -O1 sys-devel/binutils
+CTARGET=x86_64-bootstrap-linux-gnu EXTRA_ECONF=--with-sysroot=/usr/$CTARGET EXTRA_EMAKE='MAKE=make MAKE+=libsuffix=../lib64' USE='-sanitize -openmp -fortran -cxx' emerge -O1 sys-devel/gcc
+CTARGET=x86_64-bootstrap-linux-gnu CFLAGS_x86=-m32 PYTHON_COMPAT_OVERRIDE=python3_11 emerge -O1 sys-libs/glibc
+CTARGET=x86_64-bootstrap-linux-gnu EXTRA_ECONF='--with-sysroot=/usr/$CTARGET --enable-shared' EXTRA_EMAKE='MAKE=make MAKE+=libsuffix=../lib64' USE='-sanitize -openmp -fortran' emerge -O1 sys-devel/gcc
+# --with-sysroot and --enable-shared needed here because toolchain.eclass
+# figures out the bootstrap stage based on ${CATEGORY}/glibc, with no overrides
+
+# Install final glibc
+/usr/x86_64-bootstrap-linux-gnu/lib64/ld-linux-x86-64.so.2 /usr/x86_64-bootstrap-linux-gnu/sbin/ldconfig
+rm /usr/x86_64-bootstrap-linux-gnu/usr/lib/crti.o  # HACK to avoid ABI test failing in glibc ebuild. sue me.
+CC=x86_64-bootstrap-linux-gnu-gcc CXX=x86_64-bootstrap-linux-gnu-g++ CFLAGS_x86=-m32 PYTHON_COMPAT_OVERRIDE=python3_11 emerge -O1 sys-libs/glibc
+# This will throw a *lot* of segmentation faults in prerm/postinst stages
+# because collision-protect removes the LFS glibc before it should (I think)
+# It doesn't really matter in the grand scheme, however.
+# glibc will be rebuilt again in bootstrap.sh
+
+# Install final compiler
+CC='x86_64-bootstrap-linux-gnu-gcc --sysroot=/' CXX='x86_64-bootstrap-linux-gnu-g++ --sysroot=/' emerge -O1 sys-kernel/linux-headers
+CC='x86_64-bootstrap-linux-gnu-gcc --sysroot=/' CXX='x86_64-bootstrap-linux-gnu-g++ --sysroot=/' EXTRA_ECONF=--disable-bootstrap USE=-sanitize emerge -O1 sys-devel/gcc
+emerge -O1 sys-devel/binutils
+
+# Set up python-exec
+# Bootstrap python version doesn't have to match with python-exec dir...
+mkdir -p /usr/lib/python-exec/python3.11
+ln -sv python3 /usr/lib/python-exec/python3.11/python
+ln -svr /usr/bin/python3.11 /usr/lib/python-exec/python3.11/python3
+emerge -O1 dev-lang/python-exec
+
+# Break dependency cycles
+emerge -O1 app-alternatives/ninja app-alternatives/yacc app-alternatives/lex app-alternatives/bzip2 app-alternatives/gzip app-alternatives/tar app-alternatives/awk
+emerge -O1 sys-libs/libxcrypt
+
+# Install implicit build dependencies
+emerge -O1 dev-build/meson-format-array app-misc/pax-utils
+
+# Run bootstrap.sh
+rm -rf /etc/portage/profile  # don't build gcc with -cxx
+/var/db/repos/gentoo/scripts/bootstrap.sh
+# Might have to be ran a couple of times as some things may error out
+# for the same reason as glibc before.
+
+# Install the rest of @system
+emerge -1N sys-devel/gcc  # Install with USE="openmp"
+USE=-pam emerge -1 sys-libs/libcap
+USE=-http2 emerge -1 net-misc/curl
+emerge -1 sys-apps/shadow  # required by everything in acct-user and acct-group
+emerge -DN @system
+
+# Rebuild and install everything into a new root, completely cleaning out LFS
+USE=build emerge --root /mnt/gentoo sys-apps/baselayout
+emerge --root /mnt/gentoo @system
+
+# Pack it up
+tar cf /gentoo-bootstrap-20240201.tar -C /mnt/gentoo .
+xz -9v /gentoo-bootstrap-20240201.tar
+
+# Distfiles you might want to back up are in /var/cache/distfiles
+
+# Possible improvements for the future:
+# - Use portage to cross-compile a bunch of tools into a temporary EPREFIX
+#   instead of replacing tools and libraries in /usr. This would avoid a bunch
+#   of errors and weird cross-contamination, as well as allow for an eventual
+#   "rm -rf /usr". Afterwards, use the EPREFIX tools to build the final system.
+# - Don't use LFS as an intermediate step. Right now it's convenient as a
+#   stable base to jump to from any distribution if live-bootstrap fails or
+#   changes out under us, but the ideal end goal is definitely live-bootstrap.
+# - For the purposes of bootstrapping multilib, a lot less steps should be
+#   required. If the above idea of cross-compilation isn't followed, this
+#   should at least be considered.
+# - Allow bootstrapping other architectures from x86. In my eyes, I see a
+#   future where it's way easier to make a qemu-like user-mode emulator than
+#   it is to implement a new architecture in every compiler and assembler used
+#   in the bootstrap chain. The LFS instructions are already workable towards
+#   this goal, but it'd be better to automate it with portage.

gentoo-2024/gentoo_wgetlist.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+set -e
+
+# Run this script against the copy of the gentoo tree that's going to be used
+
+export PORTDIR="${1:-/}"
+export ACCEPT_KEYWORDS="-* amd64"
+
+pkgs="
+app-misc/ca-certificates
+net-misc/wget
+sys-apps/portage
+"
+
+for x in $pkgs; do
+    ver="$(portageq best_visible / ebuild "$x")"
+    portageq metadata / ebuild "$ver" SRC_URI
+    echo
+done

gentoo-2024/wget-list-gentoo

@@ -0,0 +1,6 @@
+https://deb.debian.org/debian/pool/main/c/ca-certificates/ca-certificates_20230311.tar.xz
+https://archive.mozilla.org/pub/security/nss/releases/NSS_3_95_RTM/src/nss-3.95.tar.gz
+
+https://ftp.gnu.org/gnu/wget/wget-1.21.4.tar.gz
+
+https://gitweb.gentoo.org/proj/portage.git/snapshot/portage-3.0.61.tar.bz2