#!/usr/bin/env bash
# Builds the Cold Spare backup-compression corpus.
# Every file comes from a stable public URL, so the corpus is reproducible.
# Usage: ./build-corpus.sh [target-dir]
set -euo pipefail
# Wikimedia asks API and file clients to identify themselves; anonymous
# default-agent bulk fetches get throttled to a stall.
UA="ColdSpare-corpus-builder/1.0 (+https://iotlinefair.com/bench/; hello@iotlinefair.com)"
GET=(curl -sSL --retry 3 --retry-connrefused --retry-delay 2 --max-time 120 -C - -A "$UA")
DEST="${1:-corpus}"
mkdir -p "$DEST"/{photos,video,code,docs,db}
here="$(cd "$(dirname "$0")" && pwd)"

echo "== photos: 25 JPEG photographs, 2-6 MB each, from Wikimedia Commons"
head -25 "$here/photo-urls.txt" | while read -r u; do
  f="$DEST/photos/$(basename "$u")"
  [ -s "$f" ] || "${GET[@]}" -o "$f" "$u"
done

echo "== video: one 1080p H.264 clip (Big Buck Bunny, CC-BY, Blender Foundation)"
[ -s "$DEST/video/bbb-1080p-10s.mp4" ] || "${GET[@]}" -o "$DEST/video/bbb-1080p-10s.mp4" \
  "https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/1080/Big_Buck_Bunny_1080_10s_30MB.mp4"

echo "== code: SQLite 3.46.0 source tree (C, headers, test scripts, makefiles)"
if [ ! -s "$DEST/code/sqlite-src.tar" ]; then
  "${GET[@]}" -o "$DEST/code/src.tar.gz" \
    "https://codeload.github.com/sqlite/sqlite/tar.gz/refs/tags/version-3.46.0"
  gzip -dc "$DEST/code/src.tar.gz" > "$DEST/code/sqlite-src.tar"
  rm -f "$DEST/code/src.tar.gz"
fi

echo "== docs: 10 plain-text books from Project Gutenberg"
for id in 2600 1342 84 11 1661 98 2701 5200 74 345; do
  f="$DEST/docs/pg$id.txt"
  [ -s "$f" ] || "${GET[@]}" -o "$f" "https://www.gutenberg.org/cache/epub/$id/pg$id.txt"
done

echo "== db: SQLite database built from the OurAirports public dataset"
if [ ! -s "$DEST/db/airports.sqlite" ]; then
  "${GET[@]}" -o "$DEST/db/airports.csv" "https://ourairports.com/data/airports.csv"
  "${GET[@]}" -o "$DEST/db/runways.csv"  "https://ourairports.com/data/runways.csv"
  "${GET[@]}" -o "$DEST/db/navaids.csv"  "https://ourairports.com/data/navaids.csv"
  sqlite3 "$DEST/db/airports.sqlite" <<SQL
.mode csv
.import --csv "$DEST/db/airports.csv" airports
.import --csv "$DEST/db/runways.csv" runways
.import --csv "$DEST/db/navaids.csv" navaids
CREATE INDEX ix_ap_ident ON airports(ident);
CREATE INDEX ix_rw_ref ON runways(airport_ident);
VACUUM;
SQL
  rm -f "$DEST/db"/*.csv
fi

echo "== packing one tar per category"
for c in photos video code docs db; do
  [ -s "$DEST/$c.tar" ] || tar -cf "$DEST/$c.tar" -C "$DEST/$c" .
done
ls -l "$DEST"/*.tar