Knowledge Base (Posts about mirror)

Sword module repository

bgstack15 — Tue, 18 Jun 2024 12:53:37 GMT

This project isn't a mirror, per se, but it is a small repository. I host for myself a small subset of the available SWORD Project modules.

I wanted a private repository for my mobile devices (only places where I use the modules), in case the upstream sources ever disappear. At least one module has, see the fascinating thread.

F-droid repo web frontend

bgstack15 — Fri, 14 Jun 2024 13:06:06 GMT

In my previous post, I showed how I set up my fdroid mirror suitable for adding as an f-droid repository. I wrote a little web page generator, that parses the metadata of the packages.

files/2024/listings/fdroid_generate_web.py (Source)

	`#!/usr/bin/env python3`
	`# vim: set ts=3 sw=3 sts=3 et:`
	`# File: /etc/installed/fdroid/fdroid_generate_web.py`
	`# Location: server3`
	`# Author: bgstack15`
	`# SPDX-License-Identifier: GPL-3.0-only`
	`# Startdate: 2024-06-03-2 13:31`
	`# Title: Generate simple web view of F-droid repo`
	`# Purpose: Generate a small front-end web page of packages here`
	`# History:`
	`# Usage:`
	`# called by fdroid-sync.sh, or also just usable by itself`
	`# Reference:`
	`# https://stackoverflow.com/questions/15940280/how-to-get-utc-time-in-python`
	`# Improve:`
	`# Documentation: design idea is to parse index-v1.json and build a front page, and also a per-app page that resembles f-droid.org/en/packages/org.jellyfin.mobile/`
	`from configobj import ConfigObj`
	`import os, json, datetime`
	`CONF_FILE = os.environ.get("CONF_FILE",os.path.join("/etc","sysconfig","fdroid-generate-web"))`
	`if os.path.exists(CONF_FILE):`
	`cfg = ConfigObj(CONF_FILE)`
	`else:`
	`cfg = ConfigObj()`
	`print(f"Info: could not find CONF_FILE {CONF_FILE}. Using all defaults!")`
	`# defaults`
	`if "TOP_DIR" not in cfg:`
	`cfg["TOP_DIR"] = "/mnt/mirror/fdroid"`
	`if "MIRROR_DIR" not in cfg:`
	`cfg["MIRROR_DIR"] = "/mnt/mirror/fdroid/repo"`
	`if "INDEX_JSON" not in cfg:`
	`cfg["INDEX_JSON"] = "/mnt/mirror/fdroid/repo/index-v1.json"`
	`print(cfg)`

	`def get_app_attribute(app, attribute, localizations = ["en-US","en-GB"]):`
	`"""`
	`Given app dictionary, get attribute, using the order of localizations if necessary.`
	`Returns that output, and locale of used value if any.`
	`"""`
	`output = ""`
	`try:`
	`output = app[attribute]`
	`return output, ""`
	`except:`
	`if "localized" in app:`
	`some_l = app["localized"]`
	`for lchoice in localizations:`
	`if lchoice in app["localized"]:`
	`try:`
	`output = app["localized"][lchoice][attribute]`
	`return output, lchoice`
	`except:`
	`pass`
	`some_l.pop(lchoice)`
	`for l in some_l:`
	`try:`
	`output = app["localized"][l][attribute]`
	`return output, l`
	`except:`
	`pass`
	`else:`
	`pass`
	`return output, ""`

	`def get_app_icon(app, packageName, mirror_dir, localizations = ["en-US","en-GB"]):`
	`icon, localized = get_app_attribute(app, "icon", localizations)`
	`if localized:`
	`icon = f"{packageName}/{localized}/{icon}"`
	`else:`
	`icon = f"icons/{icon}"`
	`if not os.path.exists(os.path.join(mirror_dir,icon)):`
	`if os.path.exists(os.path.join(mirror_dir,packageName,"icon.png")):`
	`return f"{packageName}/icon.png"`
	`else:`
	`for l in localizations:`
	`if os.path.exists(os.path.join(mirror_dir,packageName,l,"icon.png")):`
	`return os.path.join(packageName,l,"icon.png")`
	`return icon`

	`def generate_index_html(json_file, top_dir, mirror_dir):`
	`"""`
	`Given the index-v1.json file, generate a top-level page for the front page, stored in top_dir.`
	`"""`
	`with open(json_file,"r") as o:`
	`j = json.load(o)`
	`#print(len(json_contents))`
	`with open(os.path.join(top_dir,"index.html"),"w") as w:`
	`w.write("<html>\n")`
	`w.write("<head>\n")`
	`w.write("<link rel='stylesheet' href='fdroid.css'>\n")`
	`w.write("""<meta name="viewport" content="width=device-width, initial-scale=1">\n""")`
	`w.write(f"<title>{j['repo']['name']}</title>\n")`
	`w.write("</head>\n")`
	`w.write("<body>\n")`
	`w.write(f"<h1><img src='repo/icons/{j['repo']['icon']}' class='headimg'>{j['repo']['name']}</h1>\n")`
	`w.write(f"<p><a href='repo/'>Instructions to install this repository</a></p>")`
	`#apps = sorted(j["apps"],key=lambda i: i["name"])`
	`apps = j["apps"]`
	`if len(apps):`
	`w.write(f"<h2>Packages ({len(apps)})</h2>")`
	`for app in apps:`
	`name, _ = get_app_attribute(app,"name")`
	`packageName, _ = get_app_attribute(app,"packageName")`
	`icon = get_app_icon(app, packageName, mirror_dir)`
	`desc, _ = get_app_attribute(app,"summary")`
	`#w.write(f"<a href='packages/{packageName}.html'><img src='repo/{icon}'>{name}</a> {desc}<br/>\n")`
	`w.write(f"<img src='repo/{icon}' class='img'><b>{name}</b> {desc}<br/>\n")`
	`w.write("</body>\n")`
	`w.write("<footer>\n")`
	`w.write(f"Last generated: " + str(datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")))`
	`w.write("</footer>\n")`
	`w.write("</html>")`

	`def generate_app_pages(json_file, top_dir):`
	`with open(json_file,"r") as o:`
	`j = json.load(o)`
	`apps = j["apps"]`
	`for app in apps:`
	`name, _ = get_app_attribute(app,"name")`
	`packageName, _ = get_app_attribute(app,"packageName")`
	`icon, localized = get_app_attribute(app,"icon")`
	`if localized:`
	`icon = f"../repo/{packageName}/{localized}/{icon}"`
	`else:`
	`icon = f"../repo/icons/{icon}"`
	`with open(os.path.join(top_dir,"packages",packageName+".html"),"w") as w:`
	`w.write("<html>\n")`
	`w.write("<head>\n")`
	`w.write("<link rel='stylesheet' href='../fdroid-package.css'>\n")`
	`w.write("""<meta name="viewport" content="width=device-width, initial-scale=1">\n""")`
	`w.write(f"<title>{name}</title>\n")`
	`w.write("</head>\n")`
	`w.write("<body>\n")`
	`w.write(f"<img src='{icon}' class='icon'/><h1>{name}</h1>")`
	`w.write("</body>\n")`
	`w.write("</html>\n")`

	`if "__main__" == __name__:`
	`generate_index_html(cfg["INDEX_JSON"],cfg["TOP_DIR"],cfg["MIRROR_DIR"])`
	`# Not used at this time:`
	`#generate_app_pages(cfg["INDEX_JSON"],cfg["TOP_DIR"])`

I wanted to get an amazing web frontend like f-droid.org has but I think they have more metadata available to them because they actually compile the apps and so have the source and that complex metadata available inside the various possible places. I didn't want to go through all that effort, so I gave up on the idea of an individual page per app.

F-droid partial mirror

bgstack15 — Mon, 10 Jun 2024 13:12:09 GMT

I want to ensure the longevity of the tools I use on my mobile devices. Of course I use the amazing F-droid app store. I wanted to make sure the packages I use from this repository are available even if F-droid ever disappears. (A secondary benefit is that if I ever reinstall, or update, apps from my local repo, the downloads are way faster!)

So I spent a bunch of time reading the docs and learning how to set up my own, partial binary mirror of F-droid. A future step might include building packages, but for now I'm sticking to just the binaries.

Preparing the main F-droid custom mirror

All work is done on server3.

sudo useradd fdroid # set pw in keepass
sudo mkdir -p /mnt/mirror/fdroid /opt/android-sdk
sudo dnf install java-1.8.0-openjdk-headless
sudo dnf install --setopt=install_weak_deps=False java-1.8.0-openjdk-devel

The bare command would be this:

sudo -u fdroid -E /usr/bin/rsync -n -aHS --delete --delete-delay --info=progress2 plug-mirror.rcac.purdue.edu::fdroid/repo/ /mnt/mirror/fdroid/

But I of course wrote a script for myself, which is shown farther down in this post.

Mirror Fedora cisco-openh264 repository

bgstack15 — Sun, 17 Dec 2023 14:13:38 GMT

So for some reason Fedora 39 VLC depends on something from the fedora-cisco-openh264 codecs repository for things, for no good reason. Considering the signing key is the same as the rest of Fedora, it sounds awfully official to me. Anyway, so I had to add this repo to my kickstart (coming up soon) for Fedora 39.

And since I like to use all-local resources for installing my systems, I had to mirror this repo. It took 7 seconds to run, and 30 minutes to research and learn the URL, find my utility for doing this, and preparing the config file.

If you wish to use yummirror.sh (also on gitlab) from my coprmirror project, you just need this config file.

files/2023/12/listings/yummirror-fedora-cisco-openh264-39-x86_64.conf (Source)

	`# vim: syntax=sh`
	`# startdate: 2023-12-13-4 22:28 for fc39x-ks.cfg kickstart to be able to access this locally`
	`# usage:`
	`# time sudo YUMMIRROR_CONF=/etc/installed/coprmirror/yummirror-fedora-cisco-openh264-39-x86_64.conf /etc/installed/coprmirror/yummirror.sh`
	`# improve:`
	`# this can probably be replaced with a shell script that loops through maybe a manual list of fedora release numbers, and then generates the inurl,workdir and then runs yummirror with env vars instead of various .conf files.`
	`logfile=~/log/coprmirror.$( date "+%FT%H%M%S" ).log`
	`# if we leave yum vars in inurl, it will be interpreted by whatever system is running the yummirror task, and not by a pretend version of the desired OS.`
	`#inurl='https://codecs.fedoraproject.org/openh264/$releasever/$basearch/os/'`
	`inurl='https://codecs.fedoraproject.org/openh264/39/x86_64/os/'`
	`workdir=/mnt/mirror/fedora-cisco-openh264/39/x86_64/os/`
	`thisuser=copruser`
	`VERBOSE=1`
	`DEBUG=1`
	`# skip .src.rpms`
	`include_sources=no`
	`# resign is not implemented anyways`
	`resign_repo=no`
	`# because gpgkey is just a file: for a basic fedora signing key that is already here, just leave it undefined so yummirror will skip it.`
	`#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-fedora-$releasever-$basearch`

And in case gitlab/cgit goes away, here's the listing for yummirror script itself.

files/2023/12/listings/yummirror.sh (Source)

	`#!/bin/sh`
	`# File: yummirror.sh`
	`# Location: https://gitlab.com/bgstack15/coprmirror`
	`# Author: bgstack15`
	`# Startdate: 2021-08-16 09:57`
	`# SPDX-License-Identifier: GPL-3.0`
	`# Title: Script that copies a single copr yum repo`
	`# Project: coprmirror`
	`# Purpose: mirror copr locally, given baseurl; part of a larger set`
	`# History:`
	`# Usage:`
	`# Called by a larger script that catches the entire copr. This file only does a single yum repo for a single architecture and OS release. See coprmirror.sh`
	`# References:`
	`# https://unix.stackexchange.com/questions/19701/yum-how-can-i-view-variables-like-releasever-basearch-yum0`
	`# https://gitlab.com/bgstack15/former-gists/-/blob/master/obsmirror.sh/obsmirror.sh`
	`# Improve:`
	`# resign_repo is not implemented yet.`
	`# Dependencies:`
	`# jq, yum-utils, awk, sed, grep`
	`umask 0002`

	`test -n "${YUMMIRROR_CONF}" && . "${YUMMIRROR_CONF}"`
	`test -z "${logfile}" && logfile=./yummirror.$( date "+%FT%H%M%S" ).log`
	`test -z "${inurl}" && inurl='https://copr-be.cloud.fedoraproject.org/results/bgstack15/stackrpms/epel-7-$basearch/'`
	`test -z "${workdir}" && workdir=/tmp/copr`
	`test -z "${thisuser}" && thisuser=${USER}`
	`# also use include_sources resign_repo DEBUG VERBOSE`

	`exec 3>&1`
	`show() {`
	`printf "%s" "${*}" 1>&3`
	`}`

	`reset_show() {`
	`printf "\r%s" "${*}" 1>&3`
	`}`

	`## Functions`
	`get_file() {`
	`# call: get_file "${tu}" "${md5sum}" "absolute"`
	`___tu="${1}"`
	`___sum="${2}"`
	`___abs="${3}"`
	`if test "${___abs}" = "absolute" ; then`
	`tn="$( basename "${___tu}" )"`
	`else`
	`tn="${___tu##${inurl}}"`
	`fi`
	`tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"`
	`td="$( dirname "${tf}" )"`
	`test -d "${td}" \|\| mkdir -p "${td}"`
	`gotten="skipped "`
	`#printf '\n%s\n' "inside get_file ${@}, DRYRUN=${DRYRUN}"`
	`if test -z "${DRYRUN}" \|\| test "${DRYRUN}" = "metadata";`
	`then`
	`if test -z "${___sum}" \|\| test "$( sha256sum "${tf}" 2>/dev/null \| awk '{print $1}' )" != "${___sum}" ;`
	`then`
	`test -n "${VERBOSE}" && show "retrieving ${___tu}" 2>/dev/null \|\| :`
	`wget --content-disposition --no-verbose --quiet -O "${tf}" "${___tu}" && gotten=DOWNLOADED`
	`fi`
	`fi`
	`test -n "${VERBOSE}" && reset_show 2>/dev/null \|\| :`
	`echo "${gotten} ${___tu} -> ${tf}"`
	`}`

	`## MAIN`

	`# Interpret any yum vars in the inurl`
	`if echo "${inurl}" \| grep -qE '\$' ;`
	`then`
	`echo "" \| jq 1>/dev/null 2>&1 \|\| { echo "Need jq to interpret yum vars in baseurl ${inurl}. Aborted." ; exit 1 ; }`
	`raw="$( python -c 'import yum, json; yb = yum.YumBase(); print json.dumps(yb.conf.yumvar, indent=2)' )"`
	`# validated on centos 7`
	`basearch="$( echo "${raw}" \| sed -n '1!p' \| jq '.basearch' \| tr -d '"' )"`
	`releasever="$( echo "${raw}" \| sed -n '1!p' \| jq '.releasever' \| tr -d '"' )"`
	`#echo "${inurl}" \| awk -v "basearch=${basearch}" -v "releasever=${releasever}" '{gsub("\$basearch",basearch,$0);gsub("\$releasever",releasever,$0);print}'`
	`newurl="$( echo "${inurl}" \| sed -r -e "s/\\\$basearch/${basearch}/g;" -e "s/\\\$releasever/${releasever}/g;" )"`
	`test -n "${DEBUG}" && {`
	`echo "Interpreting ${inurl} -> ${newurl}"`
	`} 1>&2`
	`inurl="${newurl}"`
	`fi`

	`# clean up trailing slashes`
	`inurl="${inurl%%/}"`

	`# sync to workdir`
	`wget_verbose=--quiet`
	`test -n "${VERBOSE}" && unset wget_verbose`
	`{`
	`test "${DEBUG:-NONE}" = "FULL" && set -x`
	`echo "logfile=${logfile}"`
	`mkdir -p "${workdir}" ; cd "${workdir}"`

	`# This file lists the files that define a yum repo`
	`for word in repodata/repomd.xml ;`
	`do`
	`# coprmirror.sh will pass DRYRUN=metadata if it has anything set in dryrun, and we interpret "metadata" here, because we absolutely need the repomd.xml and associated files no matter what, even for a dry run.`
	`DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}"`
	`done`

	`# loop through the important files listed in that file.`
	`# read sha256sum, so we don't have to download these if we already have them.`
	`metadata_list="$( grep -iE '<checksum \|href' repodata/repomd.xml \| awk -F'"' '/checksum/{print $3} /href/{print $2}' \| awk "/^>/{gsub(\"^>\",\"\",\$0);gsub(\"<.*$\",\"\",\$0);a=\$0;} /repodata/{print a,\$0}" )"`
	`echo "${metadata_list}" \| while read sum word ;`
	`do`
	`DRYRUN="${DRYRUN:+metadata}" get_file "${inurl%%/}/${word}" "${sum}" 3>/dev/null`
	`done`

	`# go ahead and fetch the gpgkey if resign_repo=no`
	`if echo "${resign_repo}" \| grep -qiE 'yes\|\<y\>\|1\|true' ;`
	`then`
	`# do something`
	`echo "Resigning functionality not built yet. Skipping..."`
	`else`
	`if test -n "${gpgkey}" ;`
	`then`
	`# fetch the gpgkey`
	`# note: yum variable parsing not yet implemented here. I would need to turn the variable parsing into a function and call it here.`
	`# cannot use get_file because it uses a relative path evaluation`
	`#wget --output-file="${workdir}/pubkey.gpg" "${gpgkey}"`
	`get_file "${gpgkey}" "" "absolute"`
	`fi`
	`fi`

	`# COPR always provides a primary.xml.gz file, which lists the assets to download`
	`full_list="$( zgrep -iE 'checksum\|href' $( grep -oiE 'href=.primary.xml.gz.$' repodata/repomd.xml 2>/dev/null \| awk -F'"' '{print $2}' ; echo "none.XXXXXXX" ) 2>/dev/null \| awk "/checksum/{gsub(\".\\\">\",\"\",\$0);gsub(\"</.\",\"\",\$0);a=\$0;} /href/{gsub(\".=\\\"\",\"\",\$0);gsub(\"\\\"/.\",\"\",\$0);print a,\$0;}" )"`
	`#echo "FULL_LIST=${full_list}"`

	`# protect against the "metadata only" run`
	`test "${DRYRUN}" = "metadata" && DRYRUN=1`
	`echo "${full_list}" \| while read sum word ;`
	`do`
	`if echo "${word}" \| grep -qiE "\.src\.rpm" ;`
	`then`
	`# if a srpm, only get it if user has set include_sources.`
	`if echo "${include_sources}" \| grep -qiE 'yes\|\<y\>\|1\|true' ;`
	`then`
	`get_file "${inurl}/${word}" "${sum}"`
	`fi`
	`else`
	`# always get all regular rpms and other non-.src.rpm files if any`
	`get_file "${inurl}/${word}" "${sum}"`
	`fi`
	`done`

	`chown -R "${thisuser}:$( id -G "${thisuser}" \| awk '{print $1}' )" "${workdir}"`
	`} 2>&1 \| tee -a "${logfile}"`

And now you have the entire (3-package) yum repository for local use.

Rsync errors in my main Fedora mirror script

bgstack15 — Mon, 01 Aug 2022 12:38:56 GMT

So for just over a month I have noticed some errors in my logs for my main mirror sync scripts.

[2022-07-23T12:00:02Z]root@server3.ipa.internal.com: BEGIN errors
@ERROR: max connections (150) reached -- try again later
rsync error: error starting client-server protocol (code 5) at main.c(1661) [Receiver=3.1.3]
@ERROR: Unknown module 'fedora-enchilada'
rsync error: error starting client-server protocol (code 5) at main.c(1661) [Receiver=3.1.3]
@ERROR: max connections (150) reached -- try again later
rsync error: error starting client-server protocol (code 5) at main.c(1661) [Receiver=3.1.3]
[2022-07-23T12:00:40Z]root@server3.ipa.internal.com: END errors

I didn't think much of it, but I have now learned that each of these "max connection" errors means that my entire connection to the upstream mirror failed. Two of these were kernel.org's mirrors. Apparently now they're so busy/popular at the time I connect to them that the server refuses the connection.

So my on-prem Fedora mirrors were out of date. This problem frustrated me, so I had to find a new mirror. So here is the Fedora releases mirror list and here is EPEL mirror list.

So now that I've picked a new one, my connections work and my local mirror is up-to-date!

You see, nowadays I have only one Fedora workstation. Everything else is now running Devuan GNU+Linux to avoid systemd. If I ever reinstall my Fedora machine, it will get Devuan too. I love my SELinux, but I don't actually care about it for desktop systems. And I'd like to avoid the bloatware on my systems when possible.

Improvements to obsmirror

bgstack15 — Thu, 07 Apr 2022 12:58:25 GMT

Previous posts on the topic of mirroring the OBS apt repos locally:

I have improved the shell script that is the main obsmirror logic. I also split out the rebuild-apt-repo logic into its own script so I could call it separately.

	`#!/bin/sh`
	`# File: /etc/installed/obsmirror.sh`
	`# Location: https://gitlab.com/bgstack15/former-gists/tree/master/obsmirror.sh`
	`# Author: bgstack15`
	`# Startdate: 2020-03-03 08:43`
	`# SPDX-License-Identifier: CC-BY-SA-4.0`
	`# Title: Script that scrapes down OBS site to serve a copy to intranet`
	`# Purpose: save down my OBS site so I can serve it locally`
	`# History:`
	`# 2020-01-05 v1: begin which used httrack`
	`# 2020-02-28 v2: complete rewrite to exclude httrack`
	`# 2020-03-03 v3: complete rewrite to get explicit files and loop through their contents, and rebuild apt repo`
	`# 2020-03-13 add on-prompt notifications`
	`# 2022-04-01 22:33`
	`# Usage:`
	`# in a cron job: /etc/cron.d/mirror.cron`
	`# 50 12 * * * root /etc/installed/obsmirror.sh 1>/dev/null 2>&1`
	`# Reference:`
	`# https://software.opensuse.org//download.html?project=home%3Abgstack15&package=freefilesync`
	`# /mnt/public/www/internal/repo/devuan-deb/update-devuan-deb.sh`
	`# https://medium.com/sqooba/create-your-own-custom-and-authenticated-apt-repository-1e4a4cf0b864`
	`# https://unix.stackexchange.com/questions/113898/how-to-merge-two-files-based-on-the-matching-of-two-columns/113903#113903`
	`# sed\|sed to get line numbers printed https://stackoverflow.com/questions/52882594/insert-line-numbers-into-file-with-sed/52884598#52884598`
	`# Improve:`
	`# Documentation:`
	`# Download the release key and trust it.`
	`# curl -s http://repo.example.com/mirror/obs/Release.key \| apt-key add -`
	`# Use a sources.list.d/ file with contents:`
	`# deb https://repo.example.com/mirror/obs/ /`
	`# Dependencies:`
	`# binaries: wget sed awk`
	`# user: obsmirror`
	`umask 0002`
	`exec 8>&0`

	`test -n "${OBSMIRROR_CONF}" && . "${OBSMIRROR_CONF}"`
	`test -z "${logfile}" && logfile="/tmp/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log"`
	`test -z "${inurl}" && inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable"`
	`test -z "${workdir}" && workdir=/tmp/obs`
	`test -z "${thisuser}" && thisuser=obsmirror`
	`test -z "${tempdir}" && tempdir="$( TMPDIR="${TMPDIR:-/tmp}" mktemp -d )"`
	`# also use include_sources resign_repo gpg_passfile gpg_keyfile DEBUG`

	`exec 3>&1`
	`show() {`
	`printf "%s" "${*}" 1>&3`
	`}`

	`reset_show() {`
	`printf "\r%s" "${*}" 1>&3`
	`}`

	`get_file() {`
	`# call: get_file "${tu}" "${md5sum}"`
	`___tu="${1}"`
	`___sum1="${2}" # nominally from the locally-generated Packages from previous run`
	`___sum2="${3}" # from current set`
	`tn="${___tu##${inurl}}"`
	`tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"`
	`td="$( dirname "${tf}" )"`
	`test -d "${td}" \|\| mkdir -p "${td}"`
	`gotten="skipped "`
	`if test -z "${DRYRUN}" ;`
	`then`
	`# determine if file is good enough`
	`___matches_either=0`
	`___md5sum_file="$( md5sum "${tf}" 2>/dev/null \| awk '{print $1}' )"`
	`# sum1 check is disabled. If upstream obs rebuilds a package, we would never download the new package of the exact same name+version! So all my fancy sum1 logic is useless.`
	`#test "${___md5sum_file}" = "${___sum1}" && ___matches_either=1`
	`test "${___md5sum_file}" = "${___sum2}" && ___matches_either=1`
	`#test -n "${VERBOSE}" && printf "%s matches:%s\n" "${tn}" "${___matches_either}"`
	`if test -z "${___sum2}" \|\| test "${___matches_either}" = "0" ;`
	`then`
	`# so the checksum is empty, or the given checksum does not match the existing downloaded file`
	`# use the Link headers because provo-mirror sucks and presents a lot of 404s.`
	`# if Links header does not exist then this list will be short.`
	`___links="$( printf '%s\n%s\n' "${___tu}" "$( curl --head "${___tu}" --silent \| awk '/Link:/ && !/type=/{print $2}' \| tr -d '<>;' )" \| sed -e '/^\s*$/d' )"`
	`test -n "${VERBOSE}" && show "retrieving ${___tu}" 2>/dev/null \|\| :`
	`___valid=0`
	`___x=0`
	`while test ${___valid} -eq 0 && test ${___x} -lt $( echo "${___links}" \| wc -l ) ;`
	`do`
	`___x=$((___x+1))`
	`___tl="$( echo "${___links}" \| sed -n "${___x}p" 2>/dev/null )"`
	`test -n "${___tl}" && wget --content-disposition --no-verbose --quiet --output-document "${tf}" "${___tl}" && ___valid=1`
	`grep -qiE '404 Not Found' "${tf}" 2>/dev/null && ___valid=0`
	`! test -s "${tf}" && ___valid=0`
	`___matches_either=0`
	`___md5sum_file="$( mdtsum "${tf}" 2>/dev/null \| awk '{print $1}' )"`
	`#test "${___md5sum_file}" = "${___sum1}" && ___matches_either=1`
	`test "${___md5sum_file}" = "${___sum2}" && ___matches_either=1`
	`if test "${___matches_either}" = "1" ;`
	`then`
	`___valid=1`
	`fi`
	`done`
	`test ${___valid:-0} -eq 1 && gotten="DOWNLOADED"`
	`test ${___valid:-0} -eq 0 && gotten='x FAILED '`
	`fi`
	`fi`
	`test -n "${VERBOSE}" && reset_show 2>/dev/null \|\| :`
	`test -n "${VERBOSE}" && echo "${gotten} ${___tu} -> ${tf}"`
	`#echo "PAUSED: " ; read -u8 foo`
	`}`

	`wget_verbose=--quiet`
	`test -n "${VERBOSE}" && unset wget_verbose`
	`{`
	`test "${DEBUG:-NONE}" = "FULL" && set -x`
	`echo "logfile=${logfile}"`

	`# These files define an apt repo`
	`# archive the Packages file, which might be generated locally by rebuild-apt-repo.sh the previous time, and might have more useful md5sums of the packages`
	`/bin/cp -pf "${workdir}/Packages" "${tempdir}/Packages.$$"`
	`for word in InRelease Packages Packages.gz Release Release.gpg Release.key Sources Sources.gz ;`
	`do`
	`get_file "${inurl}/${word}"`
	`done`

	`# loop through named packages and download them`
	`# extract these 2 fields, from both old and new Packages files. Combine them into one line. Sed\|sed adds the line numbers so we can put the lines back in original order just because, and then sort.`
	`awk '/Filename:\|MD5/{print $2}' "${tempdir}/Packages.$$" \| xargs -n2 \| sed '=' \| sed 'N; s/\n/ /' \| sort -k2 > "${tempdir}/old_list"`
	`awk '/Filename:\|MD5/{print $2}' "${workdir}/Packages" \| xargs -n2 \| sed '=' \| sed 'N; s/\n/ /' \| sort -k2 > "${tempdir}/new_list"`
	`#awk '/Filename:\|MD5/{print $2}' "${workdir}/Packages" \| xargs -n2 \| while read word sum`
	`# The sort above was so join will work. Print these columns, sort by original line number, and then remove the lineno column with awk. Then split into these var names and process.`
	`join -j 2 -a 2 -o 2.1,2.2,1.3,2.3 "${tempdir}/old_list" "${tempdir}/new_list" \| sort -k1 \| awk '{$1="";print}' \| while read word sum_old sum_new`
	`do`
	`get_file "$( echo "${word}" \| sed -r -e "s@^\.@${inurl}@;" )" "${sum_old}" "${sum_new}"`
	`#echo "a=${a} b=${b}"`
	`done`

	`# loop through dsc, orig.tar.gz, and debian.tar.xz files`
	`test -n "${include_sources}" && {`
	`for word in $( sed -n -r -e '/Files:/,/^\s*$/{/^ /p;}' ${workdir}/Sources \| awk '{print $NF}' ) ;`
	`do`
	`get_file "${inurl}/${word}"`
	`done`
	`}`

	`test -n "${resign_repo}" && . /etc/installed/rebuild-apt-repo.sh`

	`chown -R "${thisuser}:$( id -G "${thisuser}" \| awk '{print $1}' )" "${workdir}"`
	`} 2>&1 \| tee -a "${logfile}"`

	`test -z "${NO_CLEAN}" && rm -rf "${tempdir:-NOTHINGTODEL}"`

Observe the ___sum1 lines that are commented out. I wrote a lot of logic to support reading the old (previous run's locally-generated) Packages file and accepting the .deb checksums from it. I realized that due to how I rebuild the exact same version of a package (yes, I'm a terrible person and I know it), accepting the old Packages's checksum for a file means that it would never download the newer build of the exact same version-release number. So I disabled allowing it from the old Packages file, which entirely defeats the purpose of my byzantine logic to join the two Packages entries' together and comparing against both checksums for a given file.

Here is the rebuild-apt-repo.sh script.

	`#!/bin/sh`
	`# Startdate: 2022-04-02 19:51`
	`# Purpose: To rebuild apt repo, primarily for obsmirror operations`
	`# Usage: Can be called by itself, with appropriate env vars, or from obsmirror.sh`
	`# ( . /etc/installed/obsmirror-cdemu.conf ; . /etc/installed/rebuild-apt-repo.sh ; )`
	`# Dependencies:`
	`# environment vars: workdir, gpg_passfile, gpg_keyfile`
	`# On CentOS7, gnupg2 package that supports --pinentry-mode loopback, 2.2.18-2.el7 from @copr:bgstack15:el7-gnupg2-debmirror`
	`# /usr/bin/dpkg-scanpackages`
	`# References:`
	`# obsmirror.sh`
	`# https://medium.com/sqooba/create-your-own-custom-and-authenticated-apt-repository-1e4a4cf0b864`

	`# rebuild release files`
	`echo "Rebuild apt repo in ${workdir}"`

	`repodir="${workdir}"`
	`cd "${repodir}"`
	`dpkg-scanpackages -m . > Packages`
	`gzip -9c < Packages > Packages.gz`
	`# create the Release file`
	`PKGS="$(wc -c Packages)"`
	`PKGS_GZ="$(wc -c Packages.gz)"`
	`old_headers1="$( grep -E '^(Archive\|Codename\|Origin\|Label\|Architectures):' Release )"`
	`old_headers2="$( grep -E '^(Description):' Release )"`
	`cat <<EOF > Release`
	`${old_headers1}`
	`Date: $(date -u '+%a, %d %b %Y %T %Z')`
	`${old_headers2}`
	`MD5Sum:`
	`$(md5sum Packages \| cut -d" " -f1) $PKGS`
	`$(md5sum Packages.gz \| cut -d" " -f1) $PKGS_GZ`
	`SHA1:`
	`$(sha1sum Packages \| cut -d" " -f1) $PKGS`
	`$(sha1sum Packages.gz \| cut -d" " -f1) $PKGS_GZ`
	`SHA256:`
	`$(sha256sum Packages \| cut -d" " -f1) $PKGS`
	`$(sha256sum Packages.gz \| cut -d" " -f1) $PKGS_GZ`
	`EOF`
	`test -e "${gpg_passfile}" && gpg --batch --yes --passphrase-file "${gpg_passfile}" --pinentry-mode loopback -abs -o Release.gpg Release`
	`test -e "${gpg_passfile}" && gpg --batch --yes --passphrase-file "${gpg_passfile}" --pinentry-mode loopback --clearsign -o InRelease Release`
	`# and because we are resigning it, replace Release.key with the one we used`
	`test -e "${gpg_keyfile}" && /bin/cp -pf "${gpg_keyfile}" Release.key`

To run this by itself, I recommend you run it in a sub shell, because you need to dot-source the conf file.

( . /etc/installed/obsmirror-cdemu.conf ; . /etc/installed/rebuild-apt-repo.sh ; )

And a reminder, the conf file looks like this:

# vim: syntax=sh
logfile="/var/server1/shares/public/Support/Systems/server1/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log"
inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable"
workdir=/var/server1/shares/public/www/mirror/obs
include_sources=
resign_repo=yes
gpg_passfile=/root/.gnupg/linuxadmin
gpg_keyfile=/var/server1/shares/public/www/internal/repo/deb/internaldeb.gpg
thisuser=obsmirror
VERBOSE=1

Mirror a copr, all architectures and releases and versions

bgstack15 — Fri, 20 Aug 2021 12:44:20 GMT

coprmirror is my yum mirror solution, with a wrapper to mirror specifically a named COPR repository.

Overview

COPR is a distro-run community offering that lets users build yum/dnf repositories with their own software (primarily if not exclusively in rpm format). COPR performs the builds, and then hosts the binary and source rpms for client machines. My project today downloads using native GNU/Linux tools the yum repositories that collectively make up a COPR, so each release-releasever-basearch triplet. The end goal is to have a local copy of the entire current yum repos. This does not copy the build assets or log files; just what a yum repository defines and also the gpg public key. Notably this utility needs the yum python packages present only for evaluating yum variables in the inurl (baseurl from a .repo file), so if you have a string literal as the inurl value, you can run this on a system that does not have yum installed.

Using

Configure coprmirror.conf from the provided .example file, and then run:

COPRMIRROR_CONF=coprmirror.conf VERBOSE=1 DEBUG=1 ./coprmirror.sh

Upstream

Original content The get_file function was improved after being imported from my Reference 1 script.

Alternatives

I felt like ansible and system are overkill, but if you like those, this is perfect for you: https://github.com/ganto/ansible-copr_reposync

Dependencies

wget, grep, awk, sed, jq

References

obsmirror Mirror an OBS repository locally — update 1 [this blog]

Package for CentOS 7: gnupg2-2.2.18-2.el7

bgstack15 — Sun, 22 Mar 2020 12:31:34 GMT

I have previously written about how I use debmirror on CentOS 7 to keep a local copy of the Devuan Ceres package pools. A recent problem I discovered, is that the recent update of debmirror in CentOS 7 changed the way it uses gpgv to validate the Release and associated files that define an apt repo. The version of gnupg2 in el7 is not sufficient for how debmirror-2.33-1.el7 invokes gpgv, so debmirror always chokes out in the proper apt fashion, because it cannot trust the origin repository. I took it upon myself to fork the Fedora sources for gnupg2 and related packages. With very minor changes, the packages compiled on el7 just fine! You can go connect to my COmmunity PRoject (copr) and upgrade these packages.

gnupg2.x86_64              2.2.18-2.el7   @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror
libassuan.x86_64           2.5.2-2.el7    @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror
libgcrypt-stackrpms.x86_64 1.8.5-1.el7    @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror
libgpg-error.x86_64        1.33-2.el7     @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror
libksba.x86_64             1.3.5-9.el7    @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror
npth.x86_64                1.6-2.el7      @copr:copr.fedorainfracloud.org:bgstack15:el7-gnupg2-debmirror

Using this version of gnupg2 will let debmirror operate correctly on CentOS 7. I seriously doubt many people in the world are running this kind of setup, but in case you are, here you go.

Mirror an OBS deb repository locally

bgstack15 — Mon, 27 Jan 2020 13:56:50 GMT

Story

I run an OBS repository for all my packages, and it is available at the main site: https://build.opensuse.org/project/show/home:bgstack15. But I wanted to mirror this for myself, so I don't have to configure all my systems to point outward to get updates. I already host a Devuan ceres mirror for myself, and so mirroring this Open Build System repository is the last step to be self-hosting entirely for all systems except the mirror server. I first dabbled with debmirror, but it kept wanting to try rsync despite my best configuration, plus it really insists on using the dists/ directory which isn't used in the OBS deb repo design. So, I researched scraping down a whole site, and I found httrack which exists to serve a local copy of an Internet site. Bingo! After a few hours of work, here is my solution for mirroring an OBS deb repo locally.

Solution

Create a user who will own the files and execute the httrack command, because httrack didn't want to be run as root. Also, this new user can't munge other data.

useradd obsmirror

Configure a script (available at gitlab)

#!/bin/sh
# File: /etc/installed/obsmirror.sh
# License: CC-BY-SA 4.0
# Author: bgstack15
# Startdate: 2020-01-05 18:01
# Title: Script that scrapes down OBS site to serve a copy to intranet
# Purpose: save down my OBS site so I can serve it locally
# History:
# Usage:
#    in a cron job: /etc/cron.d/mirror.cron
#       50  12  *   *   *   root    /etc/installed/obsmirror.sh 1>/dev/null 2>&1
# Reference:
#    https://unix.stackexchange.com/questions/114044/how-to-make-wget-download-recursive-combining-accept-with-exclude-directorie?rq=1
#    man 1 httrack
#    https://software.opensuse.org//download.html?project=home%3Abgstack15&package=freefilesync
# Improve:
#    use some text file as a list of recently-synced URLs, and if today's URL matches a recent one, then run the httrack with the --update flag. Probably keep a running list forever.
# Documentation:
#    Download the release key and trust it.
#       curl -s http://repo.example.com/mirror/obs/Release.key | apt-key add -
#    Use a sources.list.d/ file with contents:
#       deb https://repo.example.com/mirror/obs/ /
# Dependencies:
#    binaries: curl httrack grep head tr sed awk chmod chown find rm ln
#    user: obsmirror

logfile="/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log"
{
   test "${DEBUG:-NONE}" = "FULL" && set -x
   inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable"
   workdir=/tmp/obs-stage
   outdir=/var/www/mirror/obs
   thisuser=obsmirror
   echo "logfile=${logfile}"

   mkdir -p "${workdir}" ; chmod "0711" "${workdir}" ; chown "${thisuser}:$( id -Gn obsmirror )" "${workdir}" 
   cd "${workdir}"
   # get page contents
   step1="$( curl -s -L "${inurl}/all" )"
   # get first listed package
   step2="$( echo "${step1}" | grep --color=always -oE 'href="[a-zA-Z0-9_.+\-]+\.deb"' | head -n1 | grep -oE '".*"' | tr -d '"' )"
   # get full url to a package
   step3="$( curl -s -I "${inurl}/all/${step2}" | awk '/Location:/ {print $2}' )"
   # get directory of the mirror to save down
   step4="$( echo "${step3}" | sed -r -e "s/all\/${step2}//;" -e 's/\s*$//;' )"
   # get domain of full url
   domainname="$( echo "${step3}" | grep -oE '(ht|f)tps?:\/\/[^\/]+\/' | cut -d'/' -f3 )"
   echo "TARGET URL: ${step4}"
   test -z "${DRYRUN}" && {
      # clean workdir of specific domain name in use right now.
      echo su "${thisuser}" -c "rm -rf \"${workdir:-SOMETHING}/${domainname:-SOMETHING}\""
      su "${thisuser}" -c "rm -rf \"${workdir:-SOMETHING}/${domainname:-SOMETHING}\"*"
      # have to skip the orig.tar.gz files because they are large and slow down the sync process significantly.
      echo su "${thisuser}" -c "httrack \"${step4}\" -*.orig.t* -v --mirror --update -s0 -r3 -%e0 \"${workdir}\""
      time su "${thisuser}" -c "httrack ${step4} -*.orig.t* -v --mirror --update -s0 -r3 -%e0 ${workdir}"
   }
   # -s0 ignore robots.txt
   # -r3 only go down 3 links
   # -%e0 follow 0 links to external sites

   # find most recent directory of that level
   levelcount="$(( $( printf "%s" "${inurl}" | tr -dc '/' | wc -c ) - 1 ))"
   subdir="$( find "${workdir}" -mindepth "${levelcount}" -maxdepth "${levelcount}" -type d -name 'Debian_Unstable' -printf '%T@ %p\n' | sort -n -k1 | head -n1 | awk '{print $2}' )"

   # if the work directory actually synced
   if test -n "${subdir}" ;
   then

      printf "%s " "DIRECTORY SIZE:"
      du -sxBM "${subdir:-.}"
      mkdir -p "$( dirname "${outdir}" )"
      # get current target of symlink
      current_target="$( find "${outdir}" -maxdepth 0 -type l -printf '%l\n' )"

      # if the current link is pointing to a different directory than this subdir
      if test "${current_target}" != "${subdir}" ;
      then
         # then replace it with a link to this one
         test -L "${outdir}" && unlink "${outdir}"
         echo ln -sf "${subdir}" "${outdir}"
         ln -sf "${subdir}" "${outdir}"
      fi

   else
      echo "ERROR: No subdir found, so cannot update the symlink."
   fi

   # disable the index.html with all the httrack comments and original site links
   find "${workdir}" -iname '*index.html' -exec rm {} +
} 2>&1 | tee -a "${logfile}"

And place this in cron!

#       50  12  *   *   *   root    /etc/installed/obsmirror.sh 1>/dev/null 2>&1

Explanation of script

So the logic is a little convoluted, because the OBS front page actually redirects downloads to various mirrors where the files are kept. So I needed to learn what the actual site is, and then pull down that whole site. I couldn't just use httrack --getfiles because it makes just a flat directory, which breaks the Packages contents' accuracy to the paths of the package files. But I didn't want the whole complex directory structure, just the repository structure. So I make a symlink to it in my actual web contents location.

How I host a Devuan ceres mirror for myself

bgstack15 — Fri, 08 Nov 2019 14:18:17 GMT

Building on what I learned a long time ago for building an apt repo on CentOS, and some guidance from the Devuan forum, I have now validated my local Devuan mirror. I chose to use debmirror because it can be limited to just certain suites (that's Debian-speak for "version") and package repositories. I exclusively use Devuan ceres (testing) because I need the FreeIPA packages. And also CentOS had an rpm of debmirror. I run my script on Sunday and weekdays. Cron job:

0   9   *   *   0-5 root    /etc/installed/debmirror-devuan.sh 1>/dev/null 2>&1

And my script:

#!/usr/bin/env sh
# File: /etc/installed/debmirror-devuan.sh
# Startdate: 2019-10-02 21:08
# Usage: by cron
{ debmirror --keyring /var/storage1/shares/public/Support/Platforms/devuan/keyrings/devuan-archive-keyring.gpg --verbose --config-file=/etc/debmirror.conf.devuan-ceres ; } 2>&1 | tee -a /var/storage1/shares/public/Support/Systems/storage1/var/log/debmirror/debmirror.devuan.$( date "+%F" ).log

Pretty basic, right? I had to prep the keyrings directory with the contents of a devuan install's /usr/share/keyrings, but calling the debmirror was pretty easy. I pipe it to some logging as well. Of course the debmirror.conf for this is extremely relevant.

# File: debmirror.conf.devuan-ceres
# Location: storage1:/etc/
# Author: bgstack15
# Startdate: 2019-09-27 22:43
# Title: Debmirror config for Devuan Ceres i386 and amd64 only
# Purpose: To provide a local mirror of devuan unstable so my clients can traverse only the local network
# History:
# Usage:
#    { time sudo debmirror --keyring /var/storage1/shares/public/Support/Platforms/devuan/keyrings/devuan-archive-keyring.gpg --verbose --config-file=/etc/debmirror.conf.devuan-ceres ; } 2>&1 | tee -a /var/storage1/shares/public/Support/Systems/storage1/var/log/debmirror/debmirror.devuan.$( date "+%F" ).log
# Reference:
#    https://dev1galaxy.org/viewtopic.php?id=1152
#    https://dev1galaxy.org/viewtopic.php?id=1571
#    https://duckduckgo.com/?q=devuan+debmirror
#    http://pkgmaster.devuan.org/devuan_mirror_walkthrough.txt
#    http://ftp.debian.org/debian/dists/sid/main/
#    http://pkgmaster.devuan.org/devuan/dists/ceres/
#    http://pkgmaster.devuan.org/merged/dists/ceres/
#    how to get working keyring https://ilostmynotes.blogspot.com/2008/05/debmirror-of-ubuntu-archive-validating.html
# Improve:
# Documentation:
#    To get the gpg keyring, copy d2-02a:/usr/share/keyrings/ to /mnt/public/Support/Platforms/devuan/keyrings.
#    This setup will copy all the debian+devuan packages into one devuan/ directory. No amprolla or apache rewrites are necessary.
# Location of the local mirror (use with care)
$mirrordir="/mnt/narcissus/mirror/devuan";

# Output options
$verbose=0;
$progress=0;
$debug=0;

# Download options
$host="packages.devuan.org";
$user="anonymous";
$passwd="anonymous@";
$remoteroot="merged";
$download_method="http";
@dists="ceres";
@sections="main,main/debian-installer,contrib,non-free";
@arches="i386,amd64";
# @ignores="";
# @excludes="";
# @includes="";
# @excludes_deb_section="";
# @limit_priority="";
$omit_suite_symlinks=0;
$skippackages=0;
# @rsync_extra="doc,tools";
$i18n=0;
$getcontents=1;
$do_source=1;
$max_batch=0;
# @di_dists="dists";
# @di_archs="arches";

# Save mirror state between runs; value sets validity of cache in days
$state_cache_days=0;

# Security/Sanity options
$ignore_release_gpg=0;
$ignore_release=0;
$check_md5sums=0;
$ignore_small_errors=0;

# Cleanup
$cleanup=0;
$post_cleanup=1;

# Locking options
$timeout=300;

# Rsync options
$rsync_batch=200;
$rsync_options="-aIL --partial";

# FTP/HTTP options
$passive=0;
# $proxy="http://proxy:port/";

# Dry run
$dry_run=0;

# Don't keep diff files but use them
$diff_mode="use";

# The config file must return true or perl complains.
# Always copy this.
1;

References

Weblinks

Web searches

devuan debmirror