Verify package artifacts before skipping build

Reviewed-on: #3

.woodpecker.yml

@@ -1,6 +1,21 @@
+matrix:
+  include:
+    - DISTRO_NAME: ubuntu-22-04
+      IMAGE: ubuntu:22.04
+    - DISTRO_NAME: ubuntu-24-04
+      IMAGE: ubuntu:24.04
+    - DISTRO_NAME: ubuntu-26-04
+      IMAGE: ubuntu:26.04
+    - DISTRO_NAME: debian-12
+      IMAGE: debian:12
+    - DISTRO_NAME: debian-13
+      IMAGE: debian:13
+    - DISTRO_NAME: debian-sid
+      IMAGE: debian:sid
+
 steps:
-  - name: build-and-upload-debian-package
+  - name: build-and-upload-debian-package-${DISTRO_NAME}
-    image: ubuntu:24.04
+    image: ${IMAGE}
     pull: true
     environment:
       GITEA_PACKAGE_TOKEN:

scripts/ci/bootstrap-apt.sh

@@ -2,24 +2,67 @@
 
 set -euo pipefail
 
+source "$(cd "$(dirname "$0")/.." && pwd)/lib/common.sh"
+
 export DEBIAN_FRONTEND=noninteractive
 
-printf 'Acquire::http::Proxy "http://cbrapt01.lan.cyber.gent:3142";";\n' >/etc/apt/apt.conf.d/80proxy
+distribution_id="$(detect_debian_distribution_id)"
+distribution_codename="$(detect_debian_distribution)"
+metadata_file="$(ci_metadata_file)"
+refresh_package_indexes=false
+
+log "Bootstrapping APT packages for $distribution_id:$distribution_codename"
+
+printf 'Acquire::http::Proxy "http://cbrapt01.lan.cyber.gent:3142";\n' >/etc/apt/apt.conf.d/80proxy
 apt-get update
+apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git
+update-ca-certificates
+
+bash ./scripts/ci/preflight-package.sh
+
+if [[ -f "$metadata_file" ]]; then
+  set -a
+    # shellcheck disable=SC1090
+    . "$metadata_file"
+  set +a
+
+  if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+    log "Skipping package bootstrap because the package already exists in Gitea"
+    exit 0
+  fi
+fi
+
 apt-get upgrade -y
 apt-get install -y --no-install-recommends build-essential
-apt-get install -y --no-install-recommends software-properties-common
+
-add-apt-repository -y universe
+case "$distribution_id" in
-apt-get update
+  ubuntu)
+    apt-get install -y --no-install-recommends software-properties-common
+    if ! grep -RqsE '(^|[[:space:]])universe([[:space:]]|$)' /etc/apt/sources.list /etc/apt/sources.list.d 2>/dev/null; then
+      add-apt-repository -y universe
+      refresh_package_indexes=true
+    fi
+    ;;
+  debian)
+    ;;
+  *)
+    die "Unsupported distribution ID: $distribution_id"
+    ;;
+esac
+
+if [[ "$refresh_package_indexes" == true ]]; then
+  apt-get update
+fi
+
 apt-get install -y --no-install-recommends \
   autoconf \
   automake \
-  ca-certificates \
   cmake \
-  curl \
   flex \
   gawk \
-  git \
   gnupg \
   libbz2-dev \
   libcap2-bin \
@@ -27,6 +70,7 @@ apt-get install -y --no-install-recommends \
   libtool-bin \
   m4 \
   ncat \
+  regina-rexx \
   reprepro \
   sudo \
   time \

scripts/ci/build-package.sh

@@ -13,11 +13,20 @@ load_build_config
 work_root="$(ci_work_root)"
 helper_dir="$work_root/hercules-helper"
 artifacts_dir="$work_root/artifacts"
-helper_entrypoint="$helper_dir/create-package-hyperion.sh"
-compat_template_dir="$helper_dir/packagers/debian/hyperion-4.4"
 build_path="/home/bill/hyperion-build-package"
-raw_package_path="$build_path/hyperion-4.4.deb"
+metadata_file="$(ci_metadata_file)"
-metadata_file="$artifacts_dir/package.env"
+
+if [[ -f "$metadata_file" ]]; then
+  set -a
+    # shellcheck disable=SC1090
+    . "$metadata_file"
+  set +a
+
+  if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+    log "Skipping package build because the package already exists in Gitea"
+    exit 0
+  fi
+fi
 
 log "Preparing CI work root at $work_root"
 rm -rf "$work_root"
@@ -26,11 +35,17 @@ mkdir -p "$artifacts_dir"
 log "Cloning helper repo $REPO (branch: $BRANCH)"
 git clone --depth 1 --branch "$BRANCH" "$REPO" "$helper_dir"
 
-if [[ ! -e "$helper_entrypoint" ]]; then
+helper_entrypoint=""
-  ln -s "packagers/debian/create-package-hyperion.sh" "$helper_entrypoint"
+if [[ -f "$helper_dir/packagers/debian/create-package-hyperion.sh" ]]; then
+  helper_entrypoint="$helper_dir/packagers/debian/create-package-hyperion.sh"
+elif [[ -f "$helper_dir/create-package-hyperion.sh" ]]; then
+  helper_entrypoint="$helper_dir/create-package-hyperion.sh"
+else
+  die "No Hyperion package entrypoint found in $helper_dir"
 fi
 
-if [[ ! -e "$compat_template_dir" ]]; then
+compat_template_dir="$helper_dir/packagers/debian/hyperion-4.4"
+if [[ -d "$helper_dir/packagers/debian/hercules-hyperion" ]] && [[ ! -e "$compat_template_dir" ]]; then
   ln -s "hercules-hyperion" "$compat_template_dir"
 fi
 
@@ -46,24 +61,49 @@ log "Invoking helper repo Debian package script"
     opt_beeps=false \
     opt_verbose=true \
     TRACE=false \
+    build_path="$build_path" \
+    OUTPUT_DIR="$artifacts_dir" \
     bash "$helper_entrypoint"
 )
 
-[[ -f "$raw_package_path" ]] || die "Expected package was not created: $raw_package_path"
+artifact_candidates=()
+while IFS= read -r candidate; do
+  artifact_candidates+=("$candidate")
+done < <(
+  find "$artifacts_dir" -maxdepth 1 -type f -name '*.deb' 2>/dev/null | sort
+)
+
+if (( ${#artifact_candidates[@]} == 0 )); then
+  while IFS= read -r candidate; do
+    artifact_candidates+=("$candidate")
+  done < <(
+    find "$build_path" -maxdepth 1 -type f -name '*.deb' 2>/dev/null | sort
+  )
+fi
+
+(( ${#artifact_candidates[@]} > 0 )) || die "Expected package was not created under $artifacts_dir or $build_path"
+
+raw_package_path="${artifact_candidates[0]}"
+if (( ${#artifact_candidates[@]} > 1 )); then
+  raw_package_path="${artifact_candidates[-1]}"
+fi
 
 package_name="$(dpkg-deb -f "$raw_package_path" Package)"
 package_version="$(dpkg-deb -f "$raw_package_path" Version)"
 package_arch="$(dpkg-deb -f "$raw_package_path" Architecture)"
 artifact_path="$artifacts_dir/${package_name}_${package_version}_${package_arch}.deb"
+package_distribution="${DEBIAN_DISTRIBUTION:-$(detect_debian_distribution)}"
 
-cp "$raw_package_path" "$artifact_path"
+if [[ "$raw_package_path" != "$artifact_path" ]]; then
+  cp "$raw_package_path" "$artifact_path"
+fi
 
 cat >"$metadata_file" <<EOF
 PACKAGE_FILE=$artifact_path
 PACKAGE_NAME=$package_name
 PACKAGE_VERSION=$package_version
 PACKAGE_ARCHITECTURE=$package_arch
-PACKAGE_DISTRIBUTION=${DEBIAN_DISTRIBUTION:-$DEFAULT_DEBIAN_DISTRIBUTION}
+PACKAGE_DISTRIBUTION=$package_distribution
 PACKAGE_COMPONENT=${DEBIAN_COMPONENT:-$DEFAULT_DEBIAN_COMPONENT}
 EOF
 

scripts/ci/preflight-package.sh

@@ -0,0 +1,286 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(cd "$(dirname "$0")/.." && pwd)/lib/common.sh"
+
+require_cmd awk
+require_cmd curl
+require_cmd dpkg
+require_cmd git
+require_cmd gzip
+
+load_build_config
+
+work_root="$(ci_work_root)"
+artifacts_dir="$(ci_artifacts_dir)"
+metadata_file="$(ci_metadata_file)"
+helper_dir="$work_root/hercules-helper-preflight"
+hyperion_dir="$work_root/hyperion-preflight"
+
+owner="${GITEA_PACKAGE_OWNER:-${CI_REPO_OWNER:-}}"
+username="${GITEA_PACKAGE_USERNAME:-${CI_REPO_OWNER:-}}"
+base_url="${GITEA_BASE_URL:-$DEFAULT_GITEA_BASE_URL}"
+package_distribution="${DEBIAN_DISTRIBUTION:-$(detect_debian_distribution)}"
+package_component="${DEBIAN_COMPONENT:-$DEFAULT_DEBIAN_COMPONENT}"
+package_architecture="$(dpkg --print-architecture)"
+
+resolve_helper_template_dir() {
+  local template_dir="$1/packagers/debian/hercules-hyperion"
+
+  if [[ -d "$template_dir" ]]; then
+    printf '%s\n' "$template_dir"
+    return 0
+  fi
+
+  template_dir="$(
+    find "$1/packagers/debian" -mindepth 1 -maxdepth 1 -type d -name 'hyperion-*' \
+      | sort -V \
+      | tail -n 1
+  )"
+
+  [[ -n "$template_dir" ]] || die "No Hyperion Debian package template directory found under $1/packagers/debian"
+  printf '%s\n' "$template_dir"
+}
+
+resolve_hyperion_repo_url() {
+  printf '%s\n' "${GIT_REPO_HYPERION:-${git_repo_hyperion:-https://github.com/SDL-Hercules-390/hyperion.git}}"
+}
+
+resolve_hyperion_repo_branch() {
+  printf '%s\n' "${GIT_BRANCH_HYPERION:-${git_branch_hyperion:-}}"
+}
+
+resolve_hyperion_repo_commit() {
+  printf '%s\n' "${GIT_COMMIT_HYPERION:-${git_commit_hyperion:-}}"
+}
+
+clone_repo() {
+  local repo_url="$1"
+  local repo_dir="$2"
+  local repo_branch="${3:-}"
+
+  rm -rf "$repo_dir"
+
+  if [[ -n "$repo_branch" ]]; then
+    git clone --branch "$repo_branch" "$repo_url" "$repo_dir"
+  else
+    git clone "$repo_url" "$repo_dir"
+  fi
+}
+
+determine_hyperion_version() {
+  local repo_dir="$1"
+
+  if [[ -x "$repo_dir/_dynamic_version" ]]; then
+    (
+      cd "$repo_dir"
+      ./_dynamic_version . VERSION \
+        | awk '{sub("-modified","", $0); print}' \
+        | tr -d '"'
+    )
+    return 0
+  fi
+
+  git -C "$repo_dir" describe --tags --dirty --always 2>/dev/null || echo unknown
+}
+
+fetch_package_index() {
+  local response_file
+  local status_code
+  local url
+  local suffix
+
+  response_file="$(mktemp)"
+
+  for suffix in Packages Packages.gz; do
+    url="$base_url/api/packages/$owner/debian/dists/$package_distribution/$package_component/binary-$package_architecture/$suffix"
+    status_code="$(
+      curl \
+        --silent \
+        --show-error \
+        --location \
+        --output "$response_file" \
+        --write-out '%{http_code}' \
+        --user "$username:$GITEA_PACKAGE_TOKEN" \
+        "$url"
+    )"
+
+    case "$status_code" in
+      200)
+        if [[ "$suffix" == "Packages.gz" ]]; then
+          gzip -dc "$response_file"
+        else
+          cat "$response_file"
+        fi
+        rm -f "$response_file"
+        return 0
+        ;;
+      404)
+        ;;
+      *)
+        cat "$response_file" >&2
+        rm -f "$response_file"
+        die "Package index query failed with HTTP $status_code for $url"
+        ;;
+    esac
+  done
+
+  rm -f "$response_file"
+  return 1
+}
+
+package_index_match_filenames() {
+  local package_name="$1"
+  local package_version="$2"
+  local package_architecture="$3"
+
+  awk \
+    -v target_package="$package_name" \
+    -v target_version="$package_version" \
+    -v target_architecture="$package_architecture" '
+    BEGIN {
+      RS = ""
+      FS = "\n"
+    }
+    {
+      found_package = 0
+      found_version = 0
+      found_architecture = 0
+      filename = ""
+
+      for (i = 1; i <= NF; i++) {
+        if ($i == "Package: " target_package) {
+          found_package = 1
+        }
+        if ($i == "Version: " target_version) {
+          found_version = 1
+        }
+        if ($i == "Architecture: " target_architecture) {
+          found_architecture = 1
+        }
+        if ($i ~ /^Filename: /) {
+          filename = substr($i, 11)
+        }
+      }
+
+      if (found_package && found_version && found_architecture && filename != "") {
+        print filename
+        matched = 1
+      }
+    }
+    END {
+      if (matched) {
+        exit 0
+      }
+      exit 1
+    }
+  '
+}
+
+package_file_exists() {
+  local package_filename="${1#/}"
+  local url="$base_url/api/packages/$owner/debian/$package_filename"
+  local status_code
+
+  status_code="$(
+    curl \
+      --silent \
+      --show-error \
+      --location \
+      --output /dev/null \
+      --write-out '%{http_code}' \
+      --user "$username:$GITEA_PACKAGE_TOKEN" \
+      --range 0-0 \
+      "$url"
+  )"
+
+  case "$status_code" in
+    200|206)
+      return 0
+      ;;
+    404)
+      return 1
+      ;;
+    *)
+      die "Package file probe failed with HTTP $status_code for $url"
+      ;;
+  esac
+}
+
+write_metadata() {
+  local skip_build="$1"
+  local package_name="$2"
+  local package_version="$3"
+
+  mkdir -p "$artifacts_dir"
+
+  cat >"$metadata_file" <<EOF
+SKIP_PACKAGE_BUILD=$skip_build
+PACKAGE_NAME=$package_name
+PACKAGE_VERSION=$package_version
+PACKAGE_ARCHITECTURE=$package_architecture
+PACKAGE_DISTRIBUTION=$package_distribution
+PACKAGE_COMPONENT=$package_component
+EOF
+}
+
+require_env GITEA_PACKAGE_TOKEN
+[[ -n "$owner" ]] || die "Unable to determine Gitea package owner. Set GITEA_PACKAGE_OWNER or CI_REPO_OWNER."
+[[ -n "$username" ]] || die "Unable to determine Gitea package username. Set GITEA_PACKAGE_USERNAME or CI_REPO_OWNER."
+
+mkdir -p "$work_root"
+
+log "Cloning helper repo $REPO (branch: $BRANCH) for package preflight"
+clone_repo "$REPO" "$helper_dir" "$BRANCH"
+
+template_dir="$(resolve_helper_template_dir "$helper_dir")"
+control_file="$template_dir/DEBIAN/control"
+[[ -f "$control_file" ]] || die "Package control file not found: $control_file"
+package_name="$(awk '/^Package:/ {print $2; exit}' "$control_file")"
+[[ -n "$package_name" ]] || die "Unable to determine package name from $control_file"
+
+hyperion_repo_url="$(resolve_hyperion_repo_url)"
+hyperion_repo_branch="$(resolve_hyperion_repo_branch)"
+hyperion_repo_commit="$(resolve_hyperion_repo_commit)"
+
+log "Cloning Hyperion source repo for version preflight"
+clone_repo "$hyperion_repo_url" "$hyperion_dir" "$hyperion_repo_branch"
+if [[ -n "$hyperion_repo_commit" ]]; then
+  git -C "$hyperion_dir" checkout "$hyperion_repo_commit"
+fi
+
+package_version="$(determine_hyperion_version "$hyperion_dir")"
+[[ -n "$package_version" ]] || die "Unable to determine package version"
+
+log "Checking Gitea for $package_name/$package_version ($package_distribution/$package_component/$package_architecture)"
+package_index_file="$(mktemp)"
+have_package_index=false
+matched_package_filenames=()
+
+if fetch_package_index >"$package_index_file"; then
+  have_package_index=true
+fi
+
+if [[ "$have_package_index" == "true" ]]; then
+  while IFS= read -r package_filename; do
+    matched_package_filenames+=("$package_filename")
+  done < <(
+    package_index_match_filenames "$package_name" "$package_version" "$package_architecture" <"$package_index_file"
+  )
+fi
+
+for package_filename in "${matched_package_filenames[@]}"; do
+  if package_file_exists "$package_filename"; then
+    rm -f "$package_index_file"
+    write_metadata true "$package_name" "$package_version"
+    log "Package already exists in registry; build will be skipped"
+    exit 0
+  fi
+
+  log "Package index entry found for $package_name/$package_version, but artifact $package_filename is missing; build will continue"
+done
+
+rm -f "$package_index_file"
+write_metadata false "$package_name" "$package_version"
+log "Package does not exist in registry; build will continue"

scripts/ci/upload-package.sh

@@ -5,10 +5,9 @@ set -euo pipefail
 source "$(cd "$(dirname "$0")/.." && pwd)/lib/common.sh"
 
 require_cmd curl
-require_env GITEA_PACKAGE_TOKEN
 
 work_root="$(ci_work_root)"
-metadata_file="$work_root/artifacts/package.env"
+metadata_file="$(ci_metadata_file)"
 
 [[ -f "$metadata_file" ]] || die "Package metadata file not found: $metadata_file"
 
@@ -17,6 +16,13 @@ set -a
   . "$metadata_file"
 set +a
 
+if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+  log "Skipping package upload because the package already exists in Gitea"
+  exit 0
+fi
+
+require_env GITEA_PACKAGE_TOKEN
+
 owner="${GITEA_PACKAGE_OWNER:-${CI_REPO_OWNER:-}}"
 username="${GITEA_PACKAGE_USERNAME:-${CI_REPO_OWNER:-}}"
 base_url="${GITEA_BASE_URL:-$DEFAULT_GITEA_BASE_URL}"

scripts/lib/common.sh

@@ -2,13 +2,6 @@
 
 set -euo pipefail
 
-readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-readonly BUILD_CONFIG_FILE="${BUILD_CONFIG_FILE:-$REPO_ROOT/build.config}"
-readonly DEFAULT_GITEA_BASE_URL="https://git.cyber.gent"
-readonly DEFAULT_DEBIAN_DISTRIBUTION="noble"
-readonly DEFAULT_DEBIAN_COMPONENT="main"
-readonly DEFAULT_CI_WORK_ROOT="$REPO_ROOT/.ci-work"
-
 log() {
   printf '[%s] %s\n' "$(basename "$0")" "$*"
 }
@@ -18,6 +11,88 @@ die() {
   exit 1
 }
 
+detect_debian_distribution_id() {
+  local os_release="/etc/os-release"
+  local distro_id=""
+  local lsb_id=""
+
+  if [[ -r "$os_release" ]]; then
+    # shellcheck disable=SC1090
+    . "$os_release"
+    distro_id="${ID:-}"
+  fi
+
+  if [[ -n "$distro_id" ]]; then
+    case "$distro_id" in
+      ubuntu|debian)
+        printf '%s\n' "$distro_id"
+        return 0
+        ;;
+    esac
+  fi
+
+  if command -v lsb_release >/dev/null 2>&1; then
+    lsb_id="$(lsb_release -is | tr '[:upper:]' '[:lower:]')"
+    case "$lsb_id" in
+      ubuntu|debian)
+        printf '%s\n' "$lsb_id"
+        return 0
+        ;;
+    esac
+  fi
+
+  die "Unable to determine Debian/Ubuntu distribution ID from $os_release"
+}
+
+detect_debian_distribution() {
+  local os_release="/etc/os-release"
+  local distro_id=""
+  local version_codename=""
+  local ubuntu_codename=""
+
+  if [[ -r "$os_release" ]]; then
+    # shellcheck disable=SC1090
+    . "$os_release"
+    distro_id="${ID:-}"
+    version_codename="${VERSION_CODENAME:-}"
+    ubuntu_codename="${UBUNTU_CODENAME:-}"
+  fi
+
+  case "$distro_id" in
+    ubuntu)
+      if [[ -n "$ubuntu_codename" ]]; then
+        printf '%s\n' "$ubuntu_codename"
+        return 0
+      fi
+      ;;
+    debian)
+      if [[ -n "$version_codename" ]]; then
+        printf '%s\n' "$version_codename"
+        return 0
+      fi
+      ;;
+  esac
+
+  if [[ -n "$version_codename" ]]; then
+    printf '%s\n' "$version_codename"
+    return 0
+  fi
+
+  if command -v lsb_release >/dev/null 2>&1; then
+    lsb_release -cs
+    return 0
+  fi
+
+  die "Unable to determine Debian/Ubuntu distribution codename from $os_release"
+}
+
+readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+readonly BUILD_CONFIG_FILE="${BUILD_CONFIG_FILE:-$REPO_ROOT/build.config}"
+readonly DEFAULT_GITEA_BASE_URL="https://git.cyber.gent"
+readonly DEFAULT_DEBIAN_DISTRIBUTION="$(detect_debian_distribution)"
+readonly DEFAULT_DEBIAN_COMPONENT="main"
+readonly DEFAULT_CI_WORK_ROOT="$REPO_ROOT/.ci-work"
+
 require_cmd() {
   command -v "$1" >/dev/null 2>&1 || die "Required command not found: $1"
 }
@@ -42,3 +117,11 @@ load_build_config() {
 ci_work_root() {
   printf '%s\n' "${CI_WORK_ROOT:-$DEFAULT_CI_WORK_ROOT}"
 }
+
+ci_artifacts_dir() {
+  printf '%s/artifacts\n' "$(ci_work_root)"
+}
+
+ci_metadata_file() {
+  printf '%s/package.env\n' "$(ci_artifacts_dir)"
+}