Knowledge Base

Preserving for the future: Shell scripts, AoC, and more

Mirror Fedora cisco-openh264 repository

So for some reason Fedora 39 VLC depends on something from the fedora-cisco-openh264 codecs repository for things, for no good reason. Considering the signing key is the same as the rest of Fedora, it sounds awfully official to me. Anyway, so I had to add this repo to my kickstart (coming up soon) for Fedora 39.

And since I like to use all-local resources for installing my systems, I had to mirror this repo. It took 7 seconds to run, and 30 minutes to research and learn the URL, find my utility for doing this, and preparing the config file.

If you wish to use yummirror.sh (also on gitlab) from my coprmirror project, you just need this config file.

files/2023/12/listings/yummirror-fedora-cisco-openh264-39-x86_64.conf (Source)

# vim: syntax=sh
# startdate: 2023-12-13-4 22:28 for fc39x-ks.cfg kickstart to be able to access this locally
# usage:
#    time sudo YUMMIRROR_CONF=/etc/installed/coprmirror/yummirror-fedora-cisco-openh264-39-x86_64.conf /etc/installed/coprmirror/yummirror.sh
# improve:
#    this can probably be replaced with a shell script that loops through maybe a manual list of fedora release numbers, and then generates the inurl,workdir and then runs yummirror with env vars instead of various .conf files.
logfile=~/log/coprmirror.$( date "+%FT%H%M%S" ).log
# if we leave yum vars in inurl, it will be interpreted by whatever system is running the yummirror task, and not by a pretend version of the desired OS.
#inurl='https://codecs.fedoraproject.org/openh264/$releasever/$basearch/os/'
inurl='https://codecs.fedoraproject.org/openh264/39/x86_64/os/'
workdir=/mnt/mirror/fedora-cisco-openh264/39/x86_64/os/
thisuser=copruser
VERBOSE=1
DEBUG=1
# skip .src.rpms
include_sources=no
# resign is not implemented anyways
resign_repo=no
# because gpgkey is just a file: for a basic fedora signing key that is already here, just leave it undefined so yummirror will skip it.
#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch

And in case gitlab/cgit goes away, here's the listing for yummirror script itself.

files/2023/12/listings/yummirror.sh (Source)

#!/bin/sh
# File: yummirror.sh
# Location: https://gitlab.com/bgstack15/coprmirror
# Author: bgstack15
# Startdate: 2021-08-16 09:57
# SPDX-License-Identifier: GPL-3.0
# Title: Script that copies a single copr yum repo
# Project: coprmirror
# Purpose: mirror copr locally, given baseurl; part of a larger set
# History:
# Usage:
#    Called by a larger script that catches the entire copr. This file only does a single yum repo for a single architecture and OS release. See coprmirror.sh
# References:
#    https://unix.stackexchange.com/questions/19701/yum-how-can-i-view-variables-like-releasever-basearch-yum0
#    https://gitlab.com/bgstack15/former-gists/-/blob/master/obsmirror.sh/obsmirror.sh
# Improve:
#    resign_repo is not implemented yet.
# Dependencies:
#    jq, yum-utils, awk, sed, grep
umask 0002
test -n "${YUMMIRROR_CONF}" && . "${YUMMIRROR_CONF}"
test -z "${logfile}" && logfile=./yummirror.$( date "+%FT%H%M%S" ).log
test -z "${inurl}" && inurl='https://copr-be.cloud.fedoraproject.org/results/bgstack15/stackrpms/epel-7-$basearch/'
test -z "${workdir}" && workdir=/tmp/copr
test -z "${thisuser}" && thisuser=${USER}
# also use include_sources resign_repo DEBUG VERBOSE
exec 3>&1
show() {
    printf "%s" "${*}" 1>&3
}
reset_show() {
    printf "\r%s" "${*}" 1>&3
}
## Functions
get_file() {
   # call: get_file "${tu}" "${md5sum}" "absolute"
   ___tu="${1}"
   ___sum="${2}"
   ___abs="${3}"
   if test "${___abs}" = "absolute" ; then
      tn="$( basename "${___tu}" )"
   else
      tn="${___tu##${inurl}}"
   fi
   tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"
   td="$( dirname "${tf}" )"
   test -d "${td}" || mkdir -p "${td}"
   gotten="skipped   "
   #printf '\n%s\n' "inside get_file ${@}, DRYRUN=${DRYRUN}"
   if test -z "${DRYRUN}" || test "${DRYRUN}" = "metadata";
   then
      if test -z "${___sum}" || test "$( sha256sum "${tf}" 2>/dev/null | awk '{print $1}' )" != "${___sum}" ;
      then
         test -n "${VERBOSE}" && show "retrieving ${___tu}" 2>/dev/null || :
         wget --content-disposition --no-verbose --quiet -O "${tf}" "${___tu}" && gotten=DOWNLOADED
      fi
   fi
   test -n "${VERBOSE}" && reset_show 2>/dev/null || :
   echo "${gotten} ${___tu} -> ${tf}"
}
## MAIN
# Interpret any yum vars in the inurl
if echo "${inurl}" | grep -qE '\$' ;
then
   echo "" | jq 1>/dev/null 2>&1 || { echo "Need jq to interpret yum vars in baseurl ${inurl}. Aborted." ; exit 1 ; }
   raw="$( python -c 'import yum, json; yb = yum.YumBase(); print json.dumps(yb.conf.yumvar, indent=2)' )"
   # validated on centos 7
   basearch="$( echo "${raw}" | sed -n '1!p' | jq '.basearch' | tr -d '"' )"
   releasever="$( echo "${raw}" | sed -n '1!p' | jq '.releasever' | tr -d '"' )"
   #echo "${inurl}" | awk -v "basearch=${basearch}" -v "releasever=${releasever}" '{gsub("\$basearch",basearch,$0);gsub("\$releasever",releasever,$0);print}'
   newurl="$( echo "${inurl}" | sed -r -e "s/\\\$basearch/${basearch}/g;" -e "s/\\\$releasever/${releasever}/g;" )"
   test -n "${DEBUG}" && {
      echo "Interpreting ${inurl} -> ${newurl}"
   } 1>&2
   inurl="${newurl}"
fi
# clean up trailing slashes
inurl="${inurl%%/}"
# sync to workdir
wget_verbose=--quiet
test -n "${VERBOSE}" && unset wget_verbose
{
   test "${DEBUG:-NONE}" = "FULL" && set -x
   echo "logfile=${logfile}"
   mkdir -p "${workdir}" ; cd "${workdir}"
   # This file lists the files that define a yum repo
   for word in repodata/repomd.xml ;
   do
      # coprmirror.sh will pass DRYRUN=metadata if it has anything set in dryrun, and we interpret "metadata" here, because we absolutely need the repomd.xml and associated files no matter what, even for a dry run.
      DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}"
   done
   # loop through the important files listed in that file.
   # read sha256sum, so we don't have to download these if we already have them.
   metadata_list="$( grep -iE '<checksum |href' repodata/repomd.xml | awk -F'"' '/checksum/{print $3} /href/{print $2}' | awk "/^>/{gsub(\"^>\",\"\",\$0);gsub(\"<.*$\",\"\",\$0);a=\$0;} /repodata/{print a,\$0}" )"
   echo "${metadata_list}" | while read sum word ;
   do
      DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}" "${sum}" 3>/dev/null
   done
   # go ahead and fetch the gpgkey if resign_repo=no
   if echo "${resign_repo}" | grep -qiE 'yes|\<y\>|1|true' ;
   then
      # do something
      echo "Resigning functionality not built yet. Skipping..."
   else
      if test -n "${gpgkey}" ;
      then
         # fetch the gpgkey
         # note: yum variable parsing not yet implemented here. I would need to turn the variable parsing into a function and call it here.
         # cannot use get_file because it uses a relative path evaluation
         #wget --output-file="${workdir}/pubkey.gpg" "${gpgkey}"
         get_file "${gpgkey}" "" "absolute"
      fi
   fi
   # COPR always provides a primary.xml.gz file, which lists the assets to download
   full_list="$( zgrep -iE 'checksum|href' $( grep -oiE 'href=.*primary.xml.gz.*$' repodata/repomd.xml 2>/dev/null | awk -F'"' '{print $2}' ; echo "none.XXXXXXX" ) 2>/dev/null | awk "/checksum/{gsub(\".*\\\">\",\"\",\$0);gsub(\"</.*\",\"\",\$0);a=\$0;} /href/{gsub(\".*=\\\"\",\"\",\$0);gsub(\"\\\"/.*\",\"\",\$0);print a,\$0;}" )"
   #echo "FULL_LIST=${full_list}"
   # protect against the "metadata only" run
   test "${DRYRUN}" = "metadata" && DRYRUN=1
   echo "${full_list}" | while read sum word ;
   do
      if echo "${word}" | grep -qiE "\.src\.rpm" ;
      then
         # if a srpm, only get it if user has set include_sources.
         if echo "${include_sources}" | grep -qiE 'yes|\<y\>|1|true' ;
         then
            get_file "${inurl}/${word}" "${sum}"
         fi
      else
         # always get all regular rpms and other non-.src.rpm files if any
         get_file "${inurl}/${word}" "${sum}"
      fi
   done
   chown -R "${thisuser}:$( id -G "${thisuser}" | awk '{print $1}' )" "${workdir}"
} 2>&1 | tee -a "${logfile}"

And now you have the entire (3-package) yum repository for local use.

Comments