|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +# Reliable download wrapper with retry logic and timeout handling |
| 4 | +# Prefers curl over wget for better error handling and features |
| 5 | + |
| 6 | +set -euo pipefail |
| 7 | + |
| 8 | +# Source common utilities |
| 9 | +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" |
| 10 | +# shellcheck source=.ci/common.sh |
| 11 | +source "${SCRIPT_DIR}/common.sh" |
| 12 | + |
| 13 | +# Configuration |
| 14 | +RETRIES=3 |
| 15 | +TIMEOUT=30 |
| 16 | +READ_TIMEOUT=60 |
| 17 | +WAIT_RETRY=1 |
| 18 | + |
| 19 | +usage() |
| 20 | +{ |
| 21 | + local exit_code="${1:-0}" |
| 22 | + cat << EOF |
| 23 | +Usage: $0 [OPTIONS] URL [OUTPUT_FILE] |
| 24 | +
|
| 25 | +Reliable download wrapper with automatic retry logic. |
| 26 | +
|
| 27 | +OPTIONS: |
| 28 | + -h, --help Show this help message |
| 29 | + -r, --retries N Number of retry attempts (default: $RETRIES) |
| 30 | + -t, --timeout N Timeout in seconds (default: $TIMEOUT) |
| 31 | + -o, --output FILE Output file path |
| 32 | + -q, --quiet Quiet mode (minimal output) |
| 33 | +
|
| 34 | +EXAMPLES: |
| 35 | + $0 https://example.com/file.tar.gz |
| 36 | + $0 -o output.zip https://example.com/archive.zip |
| 37 | + $0 --retries 5 --timeout 30 https://example.com/large-file.bin |
| 38 | +
|
| 39 | +ENVIRONMENT: |
| 40 | + PREFER_WGET=1 Force using wget instead of curl |
| 41 | +EOF |
| 42 | + exit "$exit_code" |
| 43 | +} |
| 44 | + |
| 45 | +log() |
| 46 | +{ |
| 47 | + echo -e "${GREEN}[reliable-download]${NC} $*" >&2 |
| 48 | +} |
| 49 | + |
| 50 | +# Parse arguments |
| 51 | +QUIET=0 |
| 52 | +OUTPUT="" |
| 53 | +URL="" |
| 54 | + |
| 55 | +while [[ $# -gt 0 ]]; do |
| 56 | + case $1 in |
| 57 | + -h | --help) |
| 58 | + usage |
| 59 | + ;; |
| 60 | + -r | --retries) |
| 61 | + RETRIES="$2" |
| 62 | + shift 2 |
| 63 | + ;; |
| 64 | + -t | --timeout) |
| 65 | + TIMEOUT="$2" |
| 66 | + shift 2 |
| 67 | + ;; |
| 68 | + -o | --output) |
| 69 | + OUTPUT="$2" |
| 70 | + shift 2 |
| 71 | + ;; |
| 72 | + -q | --quiet) |
| 73 | + QUIET=1 |
| 74 | + shift |
| 75 | + ;; |
| 76 | + -*) |
| 77 | + print_error "Unknown option: $1" |
| 78 | + usage 1 |
| 79 | + ;; |
| 80 | + *) |
| 81 | + if [[ -z "$URL" ]]; then |
| 82 | + URL="$1" |
| 83 | + elif [[ -z "$OUTPUT" ]]; then |
| 84 | + OUTPUT="$1" |
| 85 | + else |
| 86 | + print_error "Too many arguments" |
| 87 | + usage 1 |
| 88 | + fi |
| 89 | + shift |
| 90 | + ;; |
| 91 | + esac |
| 92 | +done |
| 93 | + |
| 94 | +# Validate URL |
| 95 | +if [[ -z "$URL" ]]; then |
| 96 | + print_error "URL is required" |
| 97 | + usage 1 |
| 98 | +fi |
| 99 | + |
| 100 | +# Determine output file |
| 101 | +if [[ -z "$OUTPUT" ]]; then |
| 102 | + OUTPUT=$(basename "$URL") |
| 103 | +fi |
| 104 | + |
| 105 | +# Check available tools |
| 106 | +HAS_CURL=0 |
| 107 | +HAS_WGET=0 |
| 108 | + |
| 109 | +if command -v curl &> /dev/null; then |
| 110 | + HAS_CURL=1 |
| 111 | +fi |
| 112 | + |
| 113 | +if command -v wget &> /dev/null; then |
| 114 | + HAS_WGET=1 |
| 115 | +fi |
| 116 | + |
| 117 | +if [[ $HAS_CURL -eq 0 && $HAS_WGET -eq 0 ]]; then |
| 118 | + print_error "Neither curl nor wget is available" |
| 119 | + exit 1 |
| 120 | +fi |
| 121 | + |
| 122 | +# Prefer curl unless PREFER_WGET is set |
| 123 | +USE_CURL=0 |
| 124 | +if [[ $HAS_CURL -eq 1 && -z "${PREFER_WGET:-}" ]]; then |
| 125 | + USE_CURL=1 |
| 126 | +elif [[ $HAS_WGET -eq 0 ]]; then |
| 127 | + print_error "wget not available and PREFER_WGET is set" |
| 128 | + exit 1 |
| 129 | +fi |
| 130 | + |
| 131 | +# Download function |
| 132 | +download() |
| 133 | +{ |
| 134 | + local attempt=$1 |
| 135 | + local url=$2 |
| 136 | + local output=$3 |
| 137 | + |
| 138 | + [[ $QUIET -eq 0 ]] && log "Attempt $attempt/$RETRIES: Downloading $url" |
| 139 | + |
| 140 | + if [[ $USE_CURL -eq 1 ]]; then |
| 141 | + # Verify curl is available |
| 142 | + if ! command -v curl > /dev/null 2>&1; then |
| 143 | + print_error "curl binary not found in PATH" |
| 144 | + return 1 |
| 145 | + fi |
| 146 | + # curl options: |
| 147 | + # -f: Fail silently on HTTP errors |
| 148 | + # -L: Follow redirects |
| 149 | + # -S: Show error even with -s |
| 150 | + # -s: Silent mode (if quiet) |
| 151 | + # --retry: Number of retries |
| 152 | + # --retry-delay: Wait time between retries |
| 153 | + # --connect-timeout: Connection timeout |
| 154 | + # --max-time: Total operation timeout |
| 155 | + # -o: Output file |
| 156 | + local curl_opts=( |
| 157 | + -f |
| 158 | + -L |
| 159 | + --retry "$RETRIES" |
| 160 | + --retry-delay "$WAIT_RETRY" |
| 161 | + --retry-connrefused |
| 162 | + --connect-timeout "$TIMEOUT" |
| 163 | + --max-time "$READ_TIMEOUT" |
| 164 | + -o "$output" |
| 165 | + ) |
| 166 | + |
| 167 | + if [[ $QUIET -eq 1 ]]; then |
| 168 | + curl_opts+=(-s -S) |
| 169 | + else |
| 170 | + curl_opts+=(--progress-bar) |
| 171 | + fi |
| 172 | + |
| 173 | + curl "${curl_opts[@]}" "$url" |
| 174 | + else |
| 175 | + # Verify wget is available |
| 176 | + if ! command -v wget > /dev/null 2>&1; then |
| 177 | + print_error "wget binary not found in PATH" |
| 178 | + return 1 |
| 179 | + fi |
| 180 | + # wget options: |
| 181 | + # --retry-connrefused: Retry on connection refused |
| 182 | + # --waitretry: Wait between retries |
| 183 | + # --read-timeout: Read timeout |
| 184 | + # --timeout: Connection timeout |
| 185 | + # --tries: Number of attempts |
| 186 | + # -O: Output file |
| 187 | + local wget_opts=( |
| 188 | + --retry-connrefused |
| 189 | + --waitretry="$WAIT_RETRY" |
| 190 | + --read-timeout="$READ_TIMEOUT" |
| 191 | + --timeout="$TIMEOUT" |
| 192 | + --tries="$RETRIES" |
| 193 | + -O "$output" |
| 194 | + ) |
| 195 | + |
| 196 | + if [[ $QUIET -eq 1 ]]; then |
| 197 | + wget_opts+=(-q) |
| 198 | + fi |
| 199 | + |
| 200 | + wget "${wget_opts[@]}" "$url" |
| 201 | + fi |
| 202 | +} |
| 203 | + |
| 204 | +# Main download logic with retry using atomic operations |
| 205 | +SUCCESS=0 |
| 206 | +TEMP_OUTPUT="${OUTPUT}.downloading.$$" |
| 207 | + |
| 208 | +# Cleanup trap for partial downloads |
| 209 | +trap 'rm -f "$TEMP_OUTPUT"' EXIT INT TERM |
| 210 | + |
| 211 | +for i in $(seq 1 "$RETRIES"); do |
| 212 | + if download "$i" "$URL" "$TEMP_OUTPUT"; then |
| 213 | + # Atomic move to final location |
| 214 | + if mv -f "$TEMP_OUTPUT" "$OUTPUT"; then |
| 215 | + SUCCESS=1 |
| 216 | + [[ $QUIET -eq 0 ]] && log "Download successful: $OUTPUT" |
| 217 | + break |
| 218 | + else |
| 219 | + print_error "Failed to move temporary file to $OUTPUT" |
| 220 | + fi |
| 221 | + else |
| 222 | + EXIT_CODE=$? |
| 223 | + print_warning "Download failed (attempt $i/$RETRIES, exit code: $EXIT_CODE)" |
| 224 | + |
| 225 | + if [[ $i -lt $RETRIES ]]; then |
| 226 | + [[ $QUIET -eq 0 ]] && log "Retrying in ${WAIT_RETRY}s..." |
| 227 | + sleep "$WAIT_RETRY" |
| 228 | + fi |
| 229 | + fi |
| 230 | +done |
| 231 | + |
| 232 | +if [[ $SUCCESS -eq 0 ]]; then |
| 233 | + print_error "Download failed after $RETRIES attempts" |
| 234 | + rm -f "$TEMP_OUTPUT" # Clean up partial download |
| 235 | + exit 1 |
| 236 | +fi |
| 237 | + |
| 238 | +# Verify file was created and is not empty |
| 239 | +if [[ ! -f "$OUTPUT" ]]; then |
| 240 | + print_error "Output file not created: $OUTPUT" |
| 241 | + exit 1 |
| 242 | +fi |
| 243 | + |
| 244 | +if [[ ! -s "$OUTPUT" ]]; then |
| 245 | + print_error "Output file is empty: $OUTPUT" |
| 246 | + rm -f "$OUTPUT" |
| 247 | + exit 1 |
| 248 | +fi |
| 249 | + |
| 250 | +exit 0 |
0 commit comments