diff options
| -rwxr-xr-x | obsmirror.sh/obsmirror2.sh | 71 |
1 files changed, 71 insertions, 0 deletions
diff --git a/obsmirror.sh/obsmirror2.sh b/obsmirror.sh/obsmirror2.sh new file mode 100755 index 0000000..71284dd --- /dev/null +++ b/obsmirror.sh/obsmirror2.sh @@ -0,0 +1,71 @@ +#!/bin/sh +# File: /etc/installed/obsmirror.sh +# Author: bgstack15 +# SPDX-License-Identifier: CC-BY-SA-4.0 +# Startdate: 2020-03-03 08:43 +# Title: Script that scrapes down OBS site to serve a copy to intranet +# Purpose: save down my OBS site so I can serve it locally +# History: +# 2020-01-05 v1: begin which used httrack +# 2020-02-28 v2: complete rewrite to exclude httrack +# 2020-03-03 v3: complete rewrite to get explicit files and loop through their contents +# Usage: +# in a cron job: /etc/cron.d/mirror.cron +# 50 12 * * * root /etc/installed/obsmirror.sh 1>/dev/null 2>&1 +# Reference: +# https://software.opensuse.org//download.html?project=home%3Abgstack15&package=freefilesync +# Improve: +# Documentation: +# Download the release key and trust it. +# curl -s http://repo.example.com/mirror/obs/Release.key | apt-key add - +# Use a sources.list.d/ file with contents: +# deb https://repo.example.com/mirror/obs/ / +# Dependencies: +# binaries: wget sed awk +# user: obsmirror +umask 0002 + +test -n "${OBSMIRROR_CONF}" && . "${OBSMIRROR_CONF}" +test -z "${logfile}" && logfile="/tmp/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log" +test -z "${inurl}" && inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable" +test -z "${workdir}" && workdir=/tmp/obs +# also use include_sources DEBUG + +get_file() { + # call: get_file "${tu}" "${md5sum}" + ___tu="${1}" + tn="$( basename "${___tu}" )" + tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )" + td="$( dirname "${tf}" )" + test -d "${td}" || mkdir -p "${td}" + test -n "${DRYRUN}" && test -n "${VERBOSE}" && echo "${___tu} -> ${tf}" + test -z "${DRYRUN}" && wget --content-disposition --no-verbose ${wget_verbose} -O "${tf}" "${___tu}" +} + +wget_verbose=--quiet +test -n "${VERBOSE}" && unset wget_verbose +{ + test "${DEBUG:-NONE}" = "FULL" && set -x + echo "logfile=${logfile}" + + # These files define an apt repo + for word in InRelease Packages Packages.gz Release Release.gpg Release.key Sources Sources.gz ; + do + get_file "${inurl}/${word}" + done + + # loop through named packages and download them + for word in $( awk '/Filename:/{print $2}' "${workdir}/Packages" ) ; + do + get_file "$( echo "${word}" | sed -r -e "s@^\.@${inurl}@;" )" + done + + # loop through dsc, orig.tar.gz, and debian.tar.xz files + test -n "${include_sources}" && { + for word in $( sed -n -r -e '/Files:/,/^\s*$/{/^ /p;}' ${workdir}/Sources | awk '{print $NF}' ) ; + do + get_file "${inurl}/${word}" + done + } + +} 2>&1 | tee -a "${logfile}" |