initial commit

1 files changed, 145 insertions, 0 deletions

diff --git a/yummirror.sh b/yummirror.sh
new file mode 100755
index 0000000..d423515
--- /dev/null
+++ b/yummirror.sh
@@ -0,0 +1,145 @@
+#!/bin/sh
+# File: yummirror.sh
+# Location: https://gitlab.com/bgstack15/coprmirror
+# Author: bgstack15
+# Startdate: 2021-08-16 09:57
+# SPDX-License-Identifier: GPL-3.0
+# Title: Script that copies a single copr yum repo
+# Project: coprmirror
+# Purpose: mirror copr locally, given baseurl; part of a larger set
+# History:
+# Usage:
+#    Called by a larger script that catches the entire copr. This file only does a single yum repo for a single architecture and OS release. See coprmirror.sh
+# References:
+#    https://unix.stackexchange.com/questions/19701/yum-how-can-i-view-variables-like-releasever-basearch-yum0
+#    https://gitlab.com/bgstack15/former-gists/-/blob/master/obsmirror.sh/obsmirror.sh
+# Improve:
+#    resign_repo is not implemented yet.
+# Dependencies:
+#    jq, yum-utils, awk, sed, grep
+umask 0002
+
+test -n "${YUMMIRROR_CONF}" && . "${YUMMIRROR_CONF}"
+test -z "${logfile}" && logfile=./yummirror.$( date "+%FT%H%M%S" ).log
+test -z "${inurl}" && inurl='https://copr-be.cloud.fedoraproject.org/results/bgstack15/stackrpms/epel-7-$basearch/'
+test -z "${workdir}" && workdir=/tmp/copr
+test -z "${thisuser}" && thisuser=${USER}
+# also use include_sources resign_repo DEBUG VERBOSE
+
+exec 3>&1
+show() {
+    printf "%s" "${*}" 1>&3
+}
+
+reset_show() {
+    printf "\r%s" "${*}" 1>&3
+}
+
+## Functions
+get_file() {
+   # call: get_file "${tu}" "${md5sum}" "absolute"
+   ___tu="${1}"
+   ___sum="${2}"
+   ___abs="${3}"
+   if test "${___abs}" = "absolute" ; then
+      tn="$( basename "${___tu}" )"
+   else
+      tn="${___tu##${inurl}}"
+   fi
+   tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"
+   td="$( dirname "${tf}" )"
+   test -d "${td}" || mkdir -p "${td}"
+   gotten="skipped   "
+   #printf '\n%s\n' "inside get_file ${@}, DRYRUN=${DRYRUN}"
+   if test -z "${DRYRUN}" || test "${DRYRUN}" = "metadata";
+   then
+      if test -z "${___sum}" || test "$( sha256sum "${tf}" 2>/dev/null | awk '{print $1}' )" != "${___sum}" ;
+      then
+         test -n "${VERBOSE}" && show "retrieving ${___tu}" 2>/dev/null || :
+         wget --content-disposition --no-verbose --quiet -O "${tf}" "${___tu}" && gotten=DOWNLOADED
+      fi
+   fi
+   test -n "${VERBOSE}" && reset_show 2>/dev/null || :
+   echo "${gotten} ${___tu} -> ${tf}"
+}
+
+## MAIN
+
+# Interpret any yum vars in the inurl
+if echo "${inurl}" | grep -qE '\$' ;
+then
+   echo "" | jq 1>/dev/null 2>&1 || { echo "Need jq to interpret yum vars in baseurl ${inurl}. Aborted." ; exit 1 ; }
+   raw="$( python -c 'import yum, json; yb = yum.YumBase(); print json.dumps(yb.conf.yumvar, indent=2)' )"
+   # validated on centos 7
+   basearch="$( echo "${raw}" | sed -n '1!p' | jq '.basearch' | tr -d '"' )"
+   releasever="$( echo "${raw}" | sed -n '1!p' | jq '.releasever' | tr -d '"' )"
+   #echo "${inurl}" | awk -v "basearch=${basearch}" -v "releasever=${releasever}" '{gsub("\$basearch",basearch,$0);gsub("\$releasever",releasever,$0);print}'
+   newurl="$( echo "${inurl}" | sed -r -e "s/\\\$basearch/${basearch}/g;" -e "s/\\\$releasever/${releasever}/g;" )"
+   test -n "${DEBUG}" && {
+      echo "Interpreting ${inurl} -> ${newurl}"
+   } 1>&2
+   inurl="${newurl}"
+fi
+
+# sync to workdir
+wget_verbose=--quiet
+test -n "${VERBOSE}" && unset wget_verbose
+{
+   test "${DEBUG:-NONE}" = "FULL" && set -x
+   echo "logfile=${logfile}"
+   mkdir -p "${workdir}" ; cd "${workdir}"
+
+   # This file lists the files that define a yum repo
+   for word in repodata/repomd.xml ;
+   do
+      # coprmirror.sh will pass DRYRUN=metadata if it has anything set in dryrun, and we interpret "metadata" here, because we absolutely need the repomd.xml and associated files no matter what, even for a dry run.
+      DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}"
+   done
+
+   # loop through the important files listed in that file.
+   # read sha256sum, so we don't have to download these if we already have them.
+   metadata_list="$( grep -iE '<checksum |href' repodata/repomd.xml | awk -F'"' '/checksum/{print $3} /href/{print $2}' | awk "/^>/{gsub(\"^>\",\"\",\$0);gsub(\"<.*$\",\"\",\$0);a=\$0;} /repodata/{print a,\$0}" )"
+   echo "${metadata_list}" | while read sum word ;
+   do
+      DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}" "${sum}" 3>/dev/null
+   done
+
+   # go ahead and fetch the gpgkey if resign_repo=no
+   if echo "${resign_repo}" | grep -qiE 'yes|\<y\>|1|true' ;
+   then
+      # do something
+      echo "Resigning functionality not built yet. Skipping..."
+   else
+      if test -n "${gpgkey}" ;
+      then
+         # fetch the gpgkey
+         # note: yum variable parsing not yet implemented here. I would need to turn the variable parsing into a function and call it here.
+         # cannot use get_file because it uses a relative path evaluation
+         #wget --output-file="${workdir}/pubkey.gpg" "${gpgkey}"
+         get_file "${gpgkey}" "" "absolute"
+      fi
+   fi
+
+   # COPR always provides a primary.xml.gz file, which lists the assets to download
+   full_list="$( zgrep -iE 'checksum|href' $( grep -oiE 'href=.*primary.xml.gz.*$' repodata/repomd.xml 2>/dev/null | awk -F'"' '{print $2}' ; echo "none.XXXXXXX" ) 2>/dev/null | awk "/checksum/{gsub(\".*\\\">\",\"\",\$0);gsub(\"</.*\",\"\",\$0);a=\$0;} /href/{gsub(\".*=\\\"\",\"\",\$0);gsub(\"\\\"/.*\",\"\",\$0);print a,\$0;}" )"
+   #echo "FULL_LIST=${full_list}"
+
+   # protect against the "metadata only" run
+   test "${DRYRUN}" = "metadata" && DRYRUN=1
+   echo "${full_list}" | while read sum word ;
+   do
+      if echo "${word}" | grep -qiE "\.src\.rpm" ;
+      then
+         # if a srpm, only get it if user has set include_sources.
+         if echo "${include_sources}" | grep -qiE 'yes|\<y\>|1|true' ;
+         then
+            get_file "${inurl}/${word}" "${sum}"
+         fi
+      else
+         # always get all regular rpms and other non-.src.rpm files if any
+         get_file "${inurl}/${word}" "${sum}"
+      fi
+   done
+
+   chown -R "${thisuser}:$( id -G "${thisuser}" | awk '{print $1}' )" "${workdir}"
+} 2>&1 | tee -a "${logfile}"