gdb with python support integration

This commits enables gdb's python support. In order to make it work, we
had to create a python fork with some patches to the buildsystem, and
also had to patch gdb as well.

.github/workflows/pipeline.yaml

@@ -1,19 +1,19 @@
-name: gdb-static-release-pipeline
+name: gdb-static-pipeline
 
 on:
+  pull_request:
+    branches:
+      - '*'
   push:
     tags:
       - 'v*'
 
-# Use a non-parallel single job pipeline because artifacts weigh too much. Instead,
-# simply build the files in the same job they are released.
 jobs:
-  build_and_publish:
+  build:
     runs-on: ubuntu-latest
+
     steps:
     - uses: actions/checkout@v4
-      with:
-        submodules: recursive
 
     - name: Install dependencies
       run: sudo apt-get install -y wget
@@ -24,7 +24,14 @@ jobs:
     - name: Pack
       run: make pack
 
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: gdb-static
+        path: build/artifacts/gdb-static*.tar.gz
+
     - name: Publish release
+      if: github.event_name == 'push'
       uses: softprops/action-gh-release@v2
       with:
-        files: build/artifacts/gdb-static*.tar.gz
+        files: build/artifacts/gdb-static*.tar.gz

.github/workflows/pr-pipeline.yaml

@@ -1,25 +0,0 @@
-name: gdb-static-pr-pipeline
-
-on:
-  pull_request:
-    branches:
-      - '*'
-
-jobs:
-  build:
-    strategy:
-      matrix:
-        build_type: ["build", "build-with-python"]
-        architecture: ["x86_64", "arm", "aarch64", "powerpc", "mips", "mipsel"]
-
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: recursive
-
-    - name: Install dependencies
-      run: sudo apt-get install -y wget
-
-    - name: Build
-      run: make ${{ matrix.build_type }}-${{ matrix.architecture }} -j$((`nproc`+1))

.gitmodules

@@ -1,14 +0,0 @@
-[submodule "cpython-static"]
-	path = src/submodule_packages/cpython-static
-	url = git@github.com:guyush1/cpython-static.git
-	branch = python3.12-static
-[submodule "binutils-gdb-static"]
-	path = src/submodule_packages/binutils-gdb
-	url = git@github.com:guyush1/binutils-gdb.git
-	branch = gdb-static
-[submodule "src/submodule_packages/pygments"]
-	path = src/submodule_packages/pygments
-	url = git@github.com:pygments/pygments.git
-[submodule "src/submodule_packages/libexpat"]
-	path = src/submodule_packages/libexpat
-	url = git@github.com:guyush1/libexpat.git

Dockerfile

@@ -19,7 +19,6 @@ RUN apt update && apt install -y \
     gcc-powerpc-linux-gnu \
     git \
     libncurses-dev \
-    libtool \
     m4  \
     make \
     patch \

Makefile

@@ -1,24 +1,15 @@
 ARCHS := x86_64 arm aarch64 powerpc mips mipsel
-
 TARGETS := $(addprefix build-, $(ARCHS))
-PYTHON_TARGETS := $(addprefix build-with-python-, $(ARCHS))
-ALL_TARGETS := $(TARGETS) $(PYTHON_TARGETS)
-
 PACK_TARGETS := $(addprefix pack-, $(ARCHS))
-PYTHON_PACK_TARGETS := $(addprefix pack-with-python-, $(ARCHS))
-ALL_PACK_TARGETS := $(PACK_TARGETS) $(PYTHON_PACK_TARGETS)
 
-SUBMODULE_PACKAGES := $(wildcard src/submodule_packages/*)
+.PHONY: clean help download_packages build build-docker-image $(TARGETS) $(PACK_TARGETS)
-BUILD_PACKAGES_DIR := "build/packages"
-
-.PHONY: clean help download_packages build build-docker-image $(ALL_TARGETS) $(ALL_PACK_TARGETS)
 
 help:
 	@echo "Usage:"
 	@echo "  make build"
 	@echo ""
 
-	@for target in $(ALL_TARGETS); do \
+	@for target in $(TARGETS); do \
 		echo "  $$target"; \
 	done
 
@@ -27,57 +18,36 @@ help:
 
 build/build-docker-image.stamp: Dockerfile
 	mkdir -p build
-	docker buildx build --tag gdb-static .
+	docker build -t gdb-static .
 	touch build/build-docker-image.stamp
 
 build-docker-image: build/build-docker-image.stamp
 
-build/download-packages.stamp: build/build-docker-image.stamp src/compilation/download_packages.sh
+build/download-packages.stamp: build/build-docker-image.stamp src/download_packages.sh
-	mkdir -p $(BUILD_PACKAGES_DIR)
+	mkdir -p build/packages
 	docker run --user $(shell id -u):$(shell id -g) \
 		--rm --volume .:/app/gdb gdb-static env TERM=xterm-256color \
-		/app/gdb/src/compilation/download_packages.sh /app/gdb/$(BUILD_PACKAGES_DIR)/
+		/app/gdb/src/download_packages.sh /app/gdb/build/packages
 	touch build/download-packages.stamp
 
-build/symlink-git-packages.stamp: $(SUBMODULE_PACKAGES)
-	mkdir -p $(BUILD_PACKAGES_DIR)
-	ln -sf $(addprefix /app/gdb/, $(SUBMODULE_PACKAGES)) $(BUILD_PACKAGES_DIR)/
-
-symlink-git-packages: build/symlink-git-packages.stamp
-
 download-packages: build/download-packages.stamp
 
-build: $(ALL_TARGETS)
+build: $(TARGETS)
 
-$(TARGETS): build-%:
+$(TARGETS): build-%: download-packages build-docker-image
-	@$(MAKE) _build-$*
-
-$(PYTHON_TARGETS): build-with-python-%:
-	@WITH_PYTHON="--with-python" $(MAKE) _build-$*
-
-_build-%: symlink-git-packages download-packages build-docker-image
 	mkdir -p build
 	docker run --user $(shell id -u):$(shell id -g) \
 		--rm --volume .:/app/gdb gdb-static env TERM=xterm-256color \
-		/app/gdb/src/compilation/build.sh $* /app/gdb/build/ /app/gdb/src $(WITH_PYTHON)
+		/app/gdb/src/build.sh $* /app/gdb/build/ /app/gdb/src
 
-pack: $(ALL_PACK_TARGETS)
+pack: $(PACK_TARGETS)
 
-$(PACK_TARGETS): pack-%:
+$(PACK_TARGETS): pack-%: build-%
-	@$(MAKE) _pack-$*
+	if [ ! -f "build/artifacts/gdb-static-$*.tar.gz" ]; then \
-
+		tar -czf "build/artifacts/gdb-static-$*.tar.gz" -C "build/artifacts/$*" .; \
-$(PYTHON_PACK_TARGETS): pack-with-python-%:
-	@TAR_EXT="with-python-" ARTIFACT_EXT="_with_python" $(MAKE) _pack-$*
-
-_pack-%: build-%
-	if [ ! -f "build/artifacts/gdb-static-$(TAR_EXT)$*.tar.gz" ]; then \
-		tar -czf "build/artifacts/gdb-static-$(TAR_EXT)$*.tar.gz" -C "build/artifacts/$*$(ARTIFACT_EXT)" .; \
 	fi
 
-clean-git-packages:
+clean:
-	git submodule foreach '[[ ! "$$sm_path" == src/submodule_packages/* ]] || git clean -xffd'
-
-clean: clean-git-packages
 	rm -rf build
 # Kill and remove all containers of image gdb-static
 	docker ps -a | grep -P "^[a-f0-9]+\s+gdb-static\s+" | awk '{print $$1}' | xargs docker rm -f 2>/dev/null || true

src/build.sh

@@ -4,9 +4,6 @@
 script_dir=$(dirname "$0")
 source "$script_dir/utils.sh"
 
-# Don't want random unknown things to fail in the build procecss!
-set -e
-
 function set_compliation_variables() {
     # Set compilation variables such as which compiler to use.
     #
@@ -43,7 +40,7 @@ function set_compliation_variables() {
         CROSS=mipsel-linux-gnu-
         export HOST=mipsel-linux-gnu
     elif [[ "$target_arch" == "x86_64" ]]; then
-        CROSS=x86_64-linux-gnu-
+        CROSS=""
         export HOST=x86_64-linux-gnu
     fi
 
@@ -52,23 +49,17 @@ function set_compliation_variables() {
 
     export CFLAGS="-O2"
     export CXXFLAGS="-O2"
-
-    # Strip the binary to reduce it's size.
-    export LDFLAGS="-s"
 }
 
-function set_up_lib_search_paths() {
+function set_ncurses_link_variables() {
-    # Set up library-related linker search paths.
+    # Set up ncurses library link variables
     #
     # Parameters:
     # $1: ncursesw build dir
-    # $2: libexpat build dir
     local ncursesw_build_dir="$1"
-    local libexpat_build_dir="$2"
 
-    # I) Allow tui mode by adding our custom built static ncursesw library to the linker search path.
+    # Allow tui mode by adding our custom built static ncursesw library to the linker search path.
-    # II) Allow parsing xml files by adding libexpat library to the linker search path.
+    export LDFLAGS="-L$ncursesw_build_dir/lib $LDFLAGS"
-    export LDFLAGS="-L$ncursesw_build_dir/lib -L$libexpat_build_dir/lib/.libs $LDFLAGS"
 }
 
 function build_iconv() {
@@ -217,64 +208,12 @@ function build_ncurses() {
     popd > /dev/null
 }
 
-function build_libexpat() {
-    # Build libexpat.
-    #
-    # Parameters:
-    # $1: libexpat package directory
-    # $2: target architecture
-    #
-    # Echoes:
-    # The libexpat build directory
-    #
-    # Returns:
-    # 0: success
-    # 1: failure
-    local libexpat_dir="$1"
-    local target_arch="$2"
-    local libexpat_build_dir="$(realpath "$libexpat_dir/build-$target_arch")"
-
-    echo "$libexpat_build_dir"
-    mkdir -p "$libexpat_build_dir"
-
-    if [[ -f "$libexpat_build_dir/lib/.libs/libexpat.a" ]]; then
-        >&2 echo "Skipping build: libexpat already built for $target_arch"
-        return 0
-    fi
-
-    pushd "$libexpat_build_dir" > /dev/null
-
-    >&2 fancy_title "Building libexpat for $target_arch"
-
-    # Generate configure if it doesnt exist.
-    if [[ ! -f "$libexpat_build_dir/../expat/configure" ]]; then
-        >&2 ../expat/buildconf.sh ../expat/
-    fi
-
-    ../expat/configure --enable-static "CC=$CC" "CXX=$CXX" "--host=$HOST" \
-        "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2
-    if [[ $? -ne 0 ]]; then
-        return 1
-    fi
-
-    make -j$(nproc) 1>&2
-    if [[ $? -ne 0 ]]; then
-        return 1
-    fi
-
-    >&2 fancy_title "Finished building libexpat for $target_arch"
-
-    popd > /dev/null
-}
-
 function build_python() {
     # Build python.
     #
     # Parameters:
     # $1: python package directory
     # $2: target architecture
-    # $3: gdb's python module directory parent
-    # $4: pygment's toplevel source dir.
     #
     # Echoes:
     # The python build directory
@@ -284,8 +223,6 @@ function build_python() {
     # 1: failure
     local python_dir="$1"
     local target_arch="$2"
-    local gdb_python_parent="$3"
-    local pygments_source_dir="$4"
     local python_lib_dir="$(realpath "$python_dir/build-$target_arch")"
 
     echo "$python_lib_dir"
@@ -314,17 +251,6 @@ function build_python() {
         --disable-ipv6 \
         --disable-shared
 
-    # Extract the regular standard library modules that are to be frozen and include the gdb and pygments custom libraries.
-    export EXTRA_FROZEN_MODULES="$(printf "%s" "$(< ${script_dir}/frozen_python_modules.txt)" | tr $'\n' ";")"
-    export EXTRA_FROZEN_MODULES="${EXTRA_FROZEN_MODULES};<gdb.**.*>: gdb = ${gdb_python_parent};<pygments.**.*>: pygments = ${pygments_source_dir}"
-    >&2 echo "Frozen Modules: ${EXTRA_FROZEN_MODULES}"
-
-    # Regenerate frozen modules with gdb env varaible. Do it after the configure because we need
-    # the `regen-frozen` makefile.
-    >&2 python3.12 ../Tools/build/freeze_modules.py
-    >&2 make regen-frozen
-
-    # Build python after configuring the project and regnerating frozen files.
     >&2 make -j $(nproc)
     if [[ $? -ne 0 ]]; then
         return 1
@@ -403,7 +329,6 @@ function build_gdb() {
     # $3: libiconv prefix
     # $4: libgmp prefix
     # $5: libmpfr prefix
-    # $6: whether to build with python or not
     #
     # Echoes:
     # The gdb build directory
@@ -417,15 +342,7 @@ function build_gdb() {
     local libiconv_prefix="$3"
     local libgmp_prefix="$4"
     local libmpfr_prefix="$5"
-    local with_python="$6"
+    local gdb_build_dir="$(realpath "$gdb_dir/build-$target_arch")"
-
-    if [[ "$with_python" == "yes" ]]; then
-        local python_flag="--with-python=/app/gdb/build/packages/cpython-static/build-$target_arch/bin/python3-config"
-        local gdb_build_dir="$(realpath "$gdb_dir/build-${target_arch}_with_python")"
-    else
-        local python_flag="--without-python"
-        local gdb_build_dir="$(realpath "$gdb_dir/build-${target_arch}")"
-    fi
 
     echo "$gdb_build_dir"
     mkdir -p "$gdb_build_dir"
@@ -440,12 +357,11 @@ function build_gdb() {
     >&2 fancy_title "Building gdb for $target_arch"
 
     ../configure -C --enable-static --with-static-standard-libraries --disable-inprocess-agent \
-                 --enable-tui "$python_flag" \
+                 --enable-tui --with-python=/app/gdb/build/packages/cpython-static/build-$target_arch/bin/python3-config \
-                 --with-expat --with-libexpat-type="static" \
                  "--with-libiconv-prefix=$libiconv_prefix" --with-libiconv-type=static \
                  "--with-gmp=$libgmp_prefix" \
                  "--with-mpfr=$libmpfr_prefix" \
-                 "CC=$CC" "CXX=$CXX" "LDFLAGS=$LDFLAGS" "--host=$HOST" \
+                 "CC=$CC" "CXX=$CXX" "--host=$HOST" \
                  "CFLAGS=$CFLAGS" "CXXFLAGS=$CXXFLAGS" 1>&2
     if [[ $? -ne 0 ]]; then
         return 1
@@ -468,7 +384,6 @@ function install_gdb() {
     # $1: gdb build directory
     # $2: artifacts directory
     # $3: target architecture
-    # $4: whether gdb was built with or without python
     #
     # Returns:
     # 0: success
@@ -477,22 +392,15 @@ function install_gdb() {
     local gdb_build_dir="$1"
     local artifacts_dir="$2"
     local target_arch="$3"
-    local with_python="$4"
 
-    if [[ "$with_python" == "yes" ]]; then
+    if [[ -d "$artifacts_dir/$target_arch" && -n "$(ls -A "$artifacts_dir/$target_arch")" ]]; then
-        local artifacts_location="$artifacts_dir/${target_arch}_with_python"
-    else
-        local artifacts_location="$artifacts_dir/${target_arch}"
-    fi
-
-    if [[ -d "$artifacts_location" && -n "$(ls -A "$artifacts_location")" ]]; then
         >&2 echo "Skipping install: gdb already installed for $target_arch"
         return 0
     fi
 
     temp_artifacts_dir="$(mktemp -d)"
 
-    mkdir -p "$artifacts_location"
+    mkdir -p "$artifacts_dir/$target_arch"
 
     make -C "$gdb_build_dir" install "DESTDIR=$temp_artifacts_dir" 1>&2
     if [[ $? -ne 0 ]]; then
@@ -501,7 +409,7 @@ function install_gdb() {
     fi
 
     while read file; do
-        cp "$file" "$artifacts_location/"
+        cp "$file" "$artifacts_dir/$target_arch/"
     done < <(find "$temp_artifacts_dir/usr/local/bin" -type f -executable)
 
     rm -rf "$temp_artifacts_dir"
@@ -515,9 +423,8 @@ function build_and_install_gdb() {
     # $2: libiconv prefix
     # $3: libgmp prefix
     # $4: libmpfr prefix
-    # $5: whether to build with python or not
+    # $5: install directory
-    # $6: install directory
+    # $6: target architecture
-    # $7: target architecture
     #
     # Returns:
     # 0: success
@@ -527,16 +434,15 @@ function build_and_install_gdb() {
     local libiconv_prefix="$2"
     local libgmp_prefix="$3"
     local libmpfr_prefix="$4"
-    local with_python="$5"
+    local artifacts_dir="$5"
-    local artifacts_dir="$6"
+    local target_arch="$6"
-    local target_arch="$7"
 
-    gdb_build_dir="$(build_gdb "$gdb_dir" "$target_arch" "$libiconv_prefix" "$libgmp_prefix" "$libmpfr_prefix" "$with_python")"
+    gdb_build_dir="$(build_gdb "$gdb_dir" "$target_arch" "$libiconv_prefix" "$libgmp_prefix" "$libmpfr_prefix")"
     if [[ $? -ne 0 ]]; then
         return 1
     fi
 
-    install_gdb "$gdb_build_dir" "$artifacts_dir" "$target_arch" "$with_python"
+    install_gdb "$gdb_build_dir" "$artifacts_dir" "$target_arch"
     if [[ $? -ne 0 ]]; then
         return 1
     fi
@@ -549,12 +455,10 @@ function build_gdb_with_dependencies() {
     # $1: target architecture
     # $2: build directory
     # $3: src directory
-    # $4: whether to build gdb with python or not
 
     local target_arch="$1"
     local build_dir="$2"
     local source_dir="$3"
-    local with_python="$4"
     local packages_dir="$build_dir/packages"
     local artifacts_dir="$build_dir/artifacts"
 
@@ -584,28 +488,17 @@ function build_gdb_with_dependencies() {
     if [[ $? -ne 0 ]]; then
         return 1
     fi
+    set_ncurses_link_variables "$ncursesw_build_dir"
 
-    libexpat_build_dir="$(build_libexpat "$packages_dir/libexpat" "$target_arch")"
+    python_build_dir="$(build_python "$packages_dir/cpython-static" "$target_arch")"
     if [[ $? -ne 0 ]]; then
         return 1
     fi
 
-    set_up_lib_search_paths "$ncursesw_build_dir" "$libexpat_build_dir"
-
-    if [[ "$with_python" == "yes" ]]; then
-        local gdb_python_dir="$packages_dir/binutils-gdb/gdb/python/lib/"
-        local pygments_source_dir="$packages_dir/pygments/"
-        local python_build_dir="$(build_python "$packages_dir/cpython-static" "$target_arch" "$gdb_python_dir" "$pygments_source_dir")"
-        if [[ $? -ne 0 ]]; then
-            return 1
-        fi
-    fi
-
     build_and_install_gdb "$packages_dir/binutils-gdb" \
                           "$iconv_build_dir/lib/.libs/" \
                           "$gmp_build_dir/.libs/" \
                           "$mpfr_build_dir/src/.libs/" \
-                          "$with_python" \
                           "$artifacts_dir" \
                           "$target_arch"
     if [[ $? -ne 0 ]]; then
@@ -614,17 +507,12 @@ function build_gdb_with_dependencies() {
 }
 
 function main() {
-    if [[ $# -lt 3 ]]; then
+    if [[ $# -ne 3 ]]; then
-        >&2 echo "Usage: $0 <target_arch> <build_dir> <src_dir> [--with-python]"
+        >&2 echo "Usage: $0 <target_arch> <build_dir> <src_dir>"
         exit 1
     fi
 
-    local with_python="no"
+    build_gdb_with_dependencies "$1" "$2" "$3"
-    if [[ "$4" == "--with-python" ]]; then
-        with_python="yes"
-    fi
-
-    build_gdb_with_dependencies "$1" "$2" "$3" "$with_python"
     if [[ $? -ne 0 ]]; then
         >&2 echo "Error: failed to build gdb with dependencies"
         exit 1

src/compilation/frozen_python_modules.txt

@@ -1,150 +0,0 @@
-abc
-_aix_support
-antigravity
-argparse
-ast
-base64
-bdb
-bisect
-calendar
-cmd
-codecs
-codeop
-code
-<collections.**.*>
-_collections_abc
-colorsys
-_compat_pickle
-compileall
-_compression
-<concurrent.**.*>
-configparser
-contextlib
-contextvars
-copy
-copyreg
-cProfile
-csv
-dataclasses
-datetime
-<dbm.**.*>
-decimal
-difflib
-dis
-<encodings.**.*>
-<ensurepip.**.*>
-enum
-filecmp
-fileinput
-fnmatch
-fractions
-ftplib
-functools
-__future__
-genericpath
-getopt
-getpass
-gettext
-glob
-graphlib
-gzip
-hashlib
-heapq
-hmac
-imaplib
-<importlib.**.*>
-inspect
-io
-ipaddress
-<json.**.*>
-keyword
-linecache
-locale
-<logging.**.*>
-lzma
-_markupbase
-mimetypes
-modulefinder
-<multiprocessing.**.*>
-netrc
-ntpath
-nturl2path
-numbers
-opcode
-operator
-optparse
-os
-_osx_support
-pathlib
-pdb
-<__phello__.**.*>
-pickle
-pickletools
-pkgutil
-platform
-plistlib
-poplib
-posixpath
-pprint
-profile
-pstats
-pty
-_py_abc
-pyclbr
-py_compile
-_pydatetime
-_pydecimal
-_pyio
-_pylong
-queue
-quopri
-random
-<re.**.*>
-reprlib
-rlcompleter
-sched
-selectors
-shelve
-shlex
-shutil
-signal
-smtplib
-socket
-socketserver
-statistics
-stat
-stringprep
-string
-_strptime
-struct
-subprocess
-symtable
-sysconfig
-tabnanny
-tempfile
-textwrap
-this
-_threading_local
-threading
-timeit
-tokenize
-token
-<tomllib.**.*>
-traceback
-tracemalloc
-trace
-tty
-types
-typing
-uuid
-warnings
-wave
-weakref
-_weakrefset
-webbrowser
-<wsgiref.**.*>
-zipapp
-<zipfile.**.*>
-<zoneinfo.**.*>
-<email.**.*>
-<urllib.**.*>

src/download_packages.sh

@@ -2,7 +2,7 @@
 
 # Include utils library
 script_dir=$(dirname "$0")
-source "$script_dir/utils.sh"
+. "$script_dir/utils.sh"
 
 # List of package URLs to download
 SOURCE_URLS=(
@@ -201,6 +201,14 @@ function download_gdb_packages() {
         fi
     done
 
+    if [[ ! -d gdb-static ]]; then
+        git clone https://github.com/guyush1/binutils-gdb.git --single-branch --branch gdb-static
+    fi
+
+    if [[ ! -d python3.12-static ]]; then
+        git clone https://github.com/guyush1/cpython-static.git --single-branch --branch python3.12-static
+    fi
+
     fancy_title "Finished downloading GDB packages"
     popd
 }