Verify package artifacts before skipping build

Reviewed-on: #3

.woodpecker.yml

@@ -10,8 +10,8 @@ matrix:
       IMAGE: debian:12
     - DISTRO_NAME: debian-13
       IMAGE: debian:13
-    - DISTRO_NAME: debian-11
+    - DISTRO_NAME: debian-sid
-      IMAGE: debian:11
+      IMAGE: debian:sid
 
 steps:
   - name: build-and-upload-debian-package-${DISTRO_NAME}

scripts/ci/bootstrap-apt.sh

@@ -8,12 +8,33 @@ export DEBIAN_FRONTEND=noninteractive
 
 distribution_id="$(detect_debian_distribution_id)"
 distribution_codename="$(detect_debian_distribution)"
+metadata_file="$(ci_metadata_file)"
 refresh_package_indexes=false
 
 log "Bootstrapping APT packages for $distribution_id:$distribution_codename"
 
 printf 'Acquire::http::Proxy "http://cbrapt01.lan.cyber.gent:3142";\n' >/etc/apt/apt.conf.d/80proxy
 apt-get update
+apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git
+update-ca-certificates
+
+bash ./scripts/ci/preflight-package.sh
+
+if [[ -f "$metadata_file" ]]; then
+  set -a
+    # shellcheck disable=SC1090
+    . "$metadata_file"
+  set +a
+
+  if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+    log "Skipping package bootstrap because the package already exists in Gitea"
+    exit 0
+  fi
+fi
+
 apt-get upgrade -y
 apt-get install -y --no-install-recommends build-essential
 
@@ -39,12 +60,9 @@ fi
 apt-get install -y --no-install-recommends \
   autoconf \
   automake \
-  ca-certificates \
   cmake \
-  curl \
   flex \
   gawk \
-  git \
   gnupg \
   libbz2-dev \
   libcap2-bin \

scripts/ci/build-package.sh

@@ -14,7 +14,19 @@ work_root="$(ci_work_root)"
 helper_dir="$work_root/hercules-helper"
 artifacts_dir="$work_root/artifacts"
 build_path="/home/bill/hyperion-build-package"
-metadata_file="$artifacts_dir/package.env"
+metadata_file="$(ci_metadata_file)"
+
+if [[ -f "$metadata_file" ]]; then
+  set -a
+    # shellcheck disable=SC1090
+    . "$metadata_file"
+  set +a
+
+  if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+    log "Skipping package build because the package already exists in Gitea"
+    exit 0
+  fi
+fi
 
 log "Preparing CI work root at $work_root"
 rm -rf "$work_root"

scripts/ci/preflight-package.sh

@@ -0,0 +1,286 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source "$(cd "$(dirname "$0")/.." && pwd)/lib/common.sh"
+
+require_cmd awk
+require_cmd curl
+require_cmd dpkg
+require_cmd git
+require_cmd gzip
+
+load_build_config
+
+work_root="$(ci_work_root)"
+artifacts_dir="$(ci_artifacts_dir)"
+metadata_file="$(ci_metadata_file)"
+helper_dir="$work_root/hercules-helper-preflight"
+hyperion_dir="$work_root/hyperion-preflight"
+
+owner="${GITEA_PACKAGE_OWNER:-${CI_REPO_OWNER:-}}"
+username="${GITEA_PACKAGE_USERNAME:-${CI_REPO_OWNER:-}}"
+base_url="${GITEA_BASE_URL:-$DEFAULT_GITEA_BASE_URL}"
+package_distribution="${DEBIAN_DISTRIBUTION:-$(detect_debian_distribution)}"
+package_component="${DEBIAN_COMPONENT:-$DEFAULT_DEBIAN_COMPONENT}"
+package_architecture="$(dpkg --print-architecture)"
+
+resolve_helper_template_dir() {
+  local template_dir="$1/packagers/debian/hercules-hyperion"
+
+  if [[ -d "$template_dir" ]]; then
+    printf '%s\n' "$template_dir"
+    return 0
+  fi
+
+  template_dir="$(
+    find "$1/packagers/debian" -mindepth 1 -maxdepth 1 -type d -name 'hyperion-*' \
+      | sort -V \
+      | tail -n 1
+  )"
+
+  [[ -n "$template_dir" ]] || die "No Hyperion Debian package template directory found under $1/packagers/debian"
+  printf '%s\n' "$template_dir"
+}
+
+resolve_hyperion_repo_url() {
+  printf '%s\n' "${GIT_REPO_HYPERION:-${git_repo_hyperion:-https://github.com/SDL-Hercules-390/hyperion.git}}"
+}
+
+resolve_hyperion_repo_branch() {
+  printf '%s\n' "${GIT_BRANCH_HYPERION:-${git_branch_hyperion:-}}"
+}
+
+resolve_hyperion_repo_commit() {
+  printf '%s\n' "${GIT_COMMIT_HYPERION:-${git_commit_hyperion:-}}"
+}
+
+clone_repo() {
+  local repo_url="$1"
+  local repo_dir="$2"
+  local repo_branch="${3:-}"
+
+  rm -rf "$repo_dir"
+
+  if [[ -n "$repo_branch" ]]; then
+    git clone --branch "$repo_branch" "$repo_url" "$repo_dir"
+  else
+    git clone "$repo_url" "$repo_dir"
+  fi
+}
+
+determine_hyperion_version() {
+  local repo_dir="$1"
+
+  if [[ -x "$repo_dir/_dynamic_version" ]]; then
+    (
+      cd "$repo_dir"
+      ./_dynamic_version . VERSION \
+        | awk '{sub("-modified","", $0); print}' \
+        | tr -d '"'
+    )
+    return 0
+  fi
+
+  git -C "$repo_dir" describe --tags --dirty --always 2>/dev/null || echo unknown
+}
+
+fetch_package_index() {
+  local response_file
+  local status_code
+  local url
+  local suffix
+
+  response_file="$(mktemp)"
+
+  for suffix in Packages Packages.gz; do
+    url="$base_url/api/packages/$owner/debian/dists/$package_distribution/$package_component/binary-$package_architecture/$suffix"
+    status_code="$(
+      curl \
+        --silent \
+        --show-error \
+        --location \
+        --output "$response_file" \
+        --write-out '%{http_code}' \
+        --user "$username:$GITEA_PACKAGE_TOKEN" \
+        "$url"
+    )"
+
+    case "$status_code" in
+      200)
+        if [[ "$suffix" == "Packages.gz" ]]; then
+          gzip -dc "$response_file"
+        else
+          cat "$response_file"
+        fi
+        rm -f "$response_file"
+        return 0
+        ;;
+      404)
+        ;;
+      *)
+        cat "$response_file" >&2
+        rm -f "$response_file"
+        die "Package index query failed with HTTP $status_code for $url"
+        ;;
+    esac
+  done
+
+  rm -f "$response_file"
+  return 1
+}
+
+package_index_match_filenames() {
+  local package_name="$1"
+  local package_version="$2"
+  local package_architecture="$3"
+
+  awk \
+    -v target_package="$package_name" \
+    -v target_version="$package_version" \
+    -v target_architecture="$package_architecture" '
+    BEGIN {
+      RS = ""
+      FS = "\n"
+    }
+    {
+      found_package = 0
+      found_version = 0
+      found_architecture = 0
+      filename = ""
+
+      for (i = 1; i <= NF; i++) {
+        if ($i == "Package: " target_package) {
+          found_package = 1
+        }
+        if ($i == "Version: " target_version) {
+          found_version = 1
+        }
+        if ($i == "Architecture: " target_architecture) {
+          found_architecture = 1
+        }
+        if ($i ~ /^Filename: /) {
+          filename = substr($i, 11)
+        }
+      }
+
+      if (found_package && found_version && found_architecture && filename != "") {
+        print filename
+        matched = 1
+      }
+    }
+    END {
+      if (matched) {
+        exit 0
+      }
+      exit 1
+    }
+  '
+}
+
+package_file_exists() {
+  local package_filename="${1#/}"
+  local url="$base_url/api/packages/$owner/debian/$package_filename"
+  local status_code
+
+  status_code="$(
+    curl \
+      --silent \
+      --show-error \
+      --location \
+      --output /dev/null \
+      --write-out '%{http_code}' \
+      --user "$username:$GITEA_PACKAGE_TOKEN" \
+      --range 0-0 \
+      "$url"
+  )"
+
+  case "$status_code" in
+    200|206)
+      return 0
+      ;;
+    404)
+      return 1
+      ;;
+    *)
+      die "Package file probe failed with HTTP $status_code for $url"
+      ;;
+  esac
+}
+
+write_metadata() {
+  local skip_build="$1"
+  local package_name="$2"
+  local package_version="$3"
+
+  mkdir -p "$artifacts_dir"
+
+  cat >"$metadata_file" <<EOF
+SKIP_PACKAGE_BUILD=$skip_build
+PACKAGE_NAME=$package_name
+PACKAGE_VERSION=$package_version
+PACKAGE_ARCHITECTURE=$package_architecture
+PACKAGE_DISTRIBUTION=$package_distribution
+PACKAGE_COMPONENT=$package_component
+EOF
+}
+
+require_env GITEA_PACKAGE_TOKEN
+[[ -n "$owner" ]] || die "Unable to determine Gitea package owner. Set GITEA_PACKAGE_OWNER or CI_REPO_OWNER."
+[[ -n "$username" ]] || die "Unable to determine Gitea package username. Set GITEA_PACKAGE_USERNAME or CI_REPO_OWNER."
+
+mkdir -p "$work_root"
+
+log "Cloning helper repo $REPO (branch: $BRANCH) for package preflight"
+clone_repo "$REPO" "$helper_dir" "$BRANCH"
+
+template_dir="$(resolve_helper_template_dir "$helper_dir")"
+control_file="$template_dir/DEBIAN/control"
+[[ -f "$control_file" ]] || die "Package control file not found: $control_file"
+package_name="$(awk '/^Package:/ {print $2; exit}' "$control_file")"
+[[ -n "$package_name" ]] || die "Unable to determine package name from $control_file"
+
+hyperion_repo_url="$(resolve_hyperion_repo_url)"
+hyperion_repo_branch="$(resolve_hyperion_repo_branch)"
+hyperion_repo_commit="$(resolve_hyperion_repo_commit)"
+
+log "Cloning Hyperion source repo for version preflight"
+clone_repo "$hyperion_repo_url" "$hyperion_dir" "$hyperion_repo_branch"
+if [[ -n "$hyperion_repo_commit" ]]; then
+  git -C "$hyperion_dir" checkout "$hyperion_repo_commit"
+fi
+
+package_version="$(determine_hyperion_version "$hyperion_dir")"
+[[ -n "$package_version" ]] || die "Unable to determine package version"
+
+log "Checking Gitea for $package_name/$package_version ($package_distribution/$package_component/$package_architecture)"
+package_index_file="$(mktemp)"
+have_package_index=false
+matched_package_filenames=()
+
+if fetch_package_index >"$package_index_file"; then
+  have_package_index=true
+fi
+
+if [[ "$have_package_index" == "true" ]]; then
+  while IFS= read -r package_filename; do
+    matched_package_filenames+=("$package_filename")
+  done < <(
+    package_index_match_filenames "$package_name" "$package_version" "$package_architecture" <"$package_index_file"
+  )
+fi
+
+for package_filename in "${matched_package_filenames[@]}"; do
+  if package_file_exists "$package_filename"; then
+    rm -f "$package_index_file"
+    write_metadata true "$package_name" "$package_version"
+    log "Package already exists in registry; build will be skipped"
+    exit 0
+  fi
+
+  log "Package index entry found for $package_name/$package_version, but artifact $package_filename is missing; build will continue"
+done
+
+rm -f "$package_index_file"
+write_metadata false "$package_name" "$package_version"
+log "Package does not exist in registry; build will continue"

scripts/ci/upload-package.sh

@@ -5,10 +5,9 @@ set -euo pipefail
 source "$(cd "$(dirname "$0")/.." && pwd)/lib/common.sh"
 
 require_cmd curl
-require_env GITEA_PACKAGE_TOKEN
 
 work_root="$(ci_work_root)"
-metadata_file="$work_root/artifacts/package.env"
+metadata_file="$(ci_metadata_file)"
 
 [[ -f "$metadata_file" ]] || die "Package metadata file not found: $metadata_file"
 
@@ -17,6 +16,13 @@ set -a
   . "$metadata_file"
 set +a
 
+if [[ "${SKIP_PACKAGE_BUILD:-false}" == "true" ]]; then
+  log "Skipping package upload because the package already exists in Gitea"
+  exit 0
+fi
+
+require_env GITEA_PACKAGE_TOKEN
+
 owner="${GITEA_PACKAGE_OWNER:-${CI_REPO_OWNER:-}}"
 username="${GITEA_PACKAGE_USERNAME:-${CI_REPO_OWNER:-}}"
 base_url="${GITEA_BASE_URL:-$DEFAULT_GITEA_BASE_URL}"

scripts/lib/common.sh

@@ -117,3 +117,11 @@ load_build_config() {
 ci_work_root() {
   printf '%s\n' "${CI_WORK_ROOT:-$DEFAULT_CI_WORK_ROOT}"
 }
+
+ci_artifacts_dir() {
+  printf '%s/artifacts\n' "$(ci_work_root)"
+}
+
+ci_metadata_file() {
+  printf '%s/package.env\n' "$(ci_artifacts_dir)"
+}