add obsmirror2 which is rewrite so v3

1 files changed, 71 insertions, 0 deletions

diff --git a/obsmirror.sh/obsmirror2.sh b/obsmirror.sh/obsmirror2.sh
new file mode 100755
index 0000000..71284dd
--- /dev/null
+++ b/obsmirror.sh/obsmirror2.sh
@@ -0,0 +1,71 @@
+#!/bin/sh
+# File: /etc/installed/obsmirror.sh
+# Author: bgstack15
+# SPDX-License-Identifier: CC-BY-SA-4.0
+# Startdate: 2020-03-03 08:43
+# Title: Script that scrapes down OBS site to serve a copy to intranet
+# Purpose: save down my OBS site so I can serve it locally
+# History:
+#    2020-01-05 v1: begin which used httrack
+#    2020-02-28 v2: complete rewrite to exclude httrack
+#    2020-03-03 v3: complete rewrite to get explicit files and loop through their contents
+# Usage:
+#    in a cron job: /etc/cron.d/mirror.cron
+#       50	12	*	*	*	root	/etc/installed/obsmirror.sh 1>/dev/null 2>&1
+# Reference:
+#    https://software.opensuse.org//download.html?project=home%3Abgstack15&package=freefilesync
+# Improve:
+# Documentation:
+#    Download the release key and trust it.
+#       curl -s http://repo.example.com/mirror/obs/Release.key | apt-key add -
+#    Use a sources.list.d/ file with contents:
+#       deb https://repo.example.com/mirror/obs/ /
+# Dependencies:
+#    binaries: wget sed awk
+#    user: obsmirror
+umask 0002
+
+test -n "${OBSMIRROR_CONF}" && . "${OBSMIRROR_CONF}"
+test -z "${logfile}" && logfile="/tmp/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log"
+test -z "${inurl}" && inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable"
+test -z "${workdir}" && workdir=/tmp/obs
+# also use include_sources DEBUG
+
+get_file() {
+   # call: get_file "${tu}" "${md5sum}"
+   ___tu="${1}"
+   tn="$( basename "${___tu}" )"
+   tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"
+   td="$( dirname "${tf}" )"
+   test -d "${td}" || mkdir -p "${td}"
+   test -n "${DRYRUN}" && test -n "${VERBOSE}" && echo "${___tu} -> ${tf}"
+   test -z "${DRYRUN}" && wget --content-disposition --no-verbose ${wget_verbose} -O "${tf}" "${___tu}"
+}
+
+wget_verbose=--quiet
+test -n "${VERBOSE}" && unset wget_verbose
+{
+   test "${DEBUG:-NONE}" = "FULL" && set -x
+   echo "logfile=${logfile}"
+
+   # These files define an apt repo
+   for word in InRelease Packages Packages.gz Release Release.gpg Release.key Sources Sources.gz ;
+   do
+      get_file "${inurl}/${word}"
+   done
+
+   # loop through named packages and download them
+   for word in $( awk '/Filename:/{print $2}' "${workdir}/Packages" ) ;
+   do
+      get_file "$( echo "${word}" | sed -r -e "s@^\.@${inurl}@;" )"
+   done
+
+   # loop through dsc, orig.tar.gz, and debian.tar.xz files
+   test -n "${include_sources}" && {
+      for word in $( sed -n -r -e '/Files:/,/^\s*$/{/^ /p;}' ${workdir}/Sources | awk '{print $NF}' ) ;
+      do
+         get_file "${inurl}/${word}"
+      done
+   }
+
+} 2>&1 | tee -a "${logfile}"