coprmirror.sh


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63

#!/bin/sh
# File: coprmirror.sh
# Location: https://gitlab.com/bgstack15/coprmirror
# Author: bgstack15
# SPDX-License-Identifier: GPL-3.0
# Startdate: 2021-08-16 14:34
# Title: COPR Mirror Script
# Project: coprmirror
# Purpose: Mirror all available architectures of a COPR to local disk
# History:
#    2022-02-14 fix wget of pubkey to --output-document
# Usage:
#    In a cronjob:
#    COPRMIRROR_CONF=coprmirror.conf ./coprmirror.sh 1>/dev/null 2>&1
# Reference:
#    yummirror.sh
# Improve:
# Dependencies:
#    yummirror.sh from this project

# Flow: given a copr name, find all release-releasever-basearch entries, and pass each one to coprmirror-single.sh
test -n "${COPRMIRROR_CONF}" && . "${COPRMIRROR_CONF}"
copr_url="${copr_url%%/}"
copr="${copr%%/}"
test -z "${logfile}" && logfile=./coprmirror.all.$( date "+%FT%H%M%S" ).log
export DEBUG DRYRUN VERBOSE

# fetch listing from site
raw_listing="$( curl -s -L "${copr_url}/${copr}/" )"
# split html, list only the directory entries, show the href, split to just text that is displayed which is same as path for copr, and then exclude any the admin requests, then sort
listing="$( echo "${raw_listing}" | sed -n -r -e '/<table /p' | sed -r -e 's:(<\/tr>):\1\n:g;' | grep Directory | grep -oE '<a href=.{6,45}>.*<\/a>' | awk -F'[<>]' '$3 ~ /.*-.*-.*/{print $3}' | grep -viE "${excludes}" | sort )"
echo "Please get each of these:"
echo "${listing}" 

# The gpg key of a copr is for the whole thing, not per-repo.
test -z "${DRYRUN}" && {
   mkdir -p "${workdir}"
   # the -N prevents pubkey.gpg.1
   wget -N --output-document="${workdir}/pubkey.gpg" "${copr_url}/${copr}/pubkey.gpg"
}

for word in ${listing} ;
do
   echo "###########################################"
   test -n "${DEBUG}" && echo env DEBUG=$DEBUG VERBOSE=$VERBOSE DRYRUN=$DRYRUN logfile="${logfile}.${word}" inurl="${copr_url}/${copr}/${word}/" this_user=${this_user} workdir="${workdir}/${word}" COPRMIRROR_CONF= "${YUMMIRROR_SCRIPT}"
   test -z "${DRYRUN}" && {
      mkdir -p "${workdir}/${word}"
   }
   # exclude gpgkey because the big script, this one, handles it, not the single script
   # leave YUMMIRROR_CONF undefined; we have set all variables for the inner script here.
   env DEBUG=$DEBUG \
      VERBOSE=$VERBOSE \
      DRYRUN="${DRYRUN}" \
      logfile="${logfile}.${word}" \
      inurl="${copr_url}/${copr}/${word}/" \
      this_user="${this_user}" \
      workdir="${workdir}/${word}" \
      YUMMIRROR_CONF= \
      include_sources="${include_sources}" \
      resign_repo="${resign_repo}" \
      gpgkey= \
      "${YUMMIRROR_SCRIPT}"
done