blob: 71284ddeb00b62ccf0a6aa9cb068a912a73e4874 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
|
#!/bin/sh
# File: /etc/installed/obsmirror.sh
# Author: bgstack15
# SPDX-License-Identifier: CC-BY-SA-4.0
# Startdate: 2020-03-03 08:43
# Title: Script that scrapes down OBS site to serve a copy to intranet
# Purpose: save down my OBS site so I can serve it locally
# History:
# 2020-01-05 v1: begin which used httrack
# 2020-02-28 v2: complete rewrite to exclude httrack
# 2020-03-03 v3: complete rewrite to get explicit files and loop through their contents
# Usage:
# in a cron job: /etc/cron.d/mirror.cron
# 50 12 * * * root /etc/installed/obsmirror.sh 1>/dev/null 2>&1
# Reference:
# https://software.opensuse.org//download.html?project=home%3Abgstack15&package=freefilesync
# Improve:
# Documentation:
# Download the release key and trust it.
# curl -s http://repo.example.com/mirror/obs/Release.key | apt-key add -
# Use a sources.list.d/ file with contents:
# deb https://repo.example.com/mirror/obs/ /
# Dependencies:
# binaries: wget sed awk
# user: obsmirror
umask 0002
test -n "${OBSMIRROR_CONF}" && . "${OBSMIRROR_CONF}"
test -z "${logfile}" && logfile="/tmp/var/log/obsmirror/obsmirror.$( date "+%FT%H%M%S" ).log"
test -z "${inurl}" && inurl="http://download.opensuse.org/repositories/home:/bgstack15/Debian_Unstable"
test -z "${workdir}" && workdir=/tmp/obs
# also use include_sources DEBUG
get_file() {
# call: get_file "${tu}" "${md5sum}"
___tu="${1}"
tn="$( basename "${___tu}" )"
tf="${workdir}/${tn}" ; tf="$( readlink -m "${tf}" )"
td="$( dirname "${tf}" )"
test -d "${td}" || mkdir -p "${td}"
test -n "${DRYRUN}" && test -n "${VERBOSE}" && echo "${___tu} -> ${tf}"
test -z "${DRYRUN}" && wget --content-disposition --no-verbose ${wget_verbose} -O "${tf}" "${___tu}"
}
wget_verbose=--quiet
test -n "${VERBOSE}" && unset wget_verbose
{
test "${DEBUG:-NONE}" = "FULL" && set -x
echo "logfile=${logfile}"
# These files define an apt repo
for word in InRelease Packages Packages.gz Release Release.gpg Release.key Sources Sources.gz ;
do
get_file "${inurl}/${word}"
done
# loop through named packages and download them
for word in $( awk '/Filename:/{print $2}' "${workdir}/Packages" ) ;
do
get_file "$( echo "${word}" | sed -r -e "s@^\.@${inurl}@;" )"
done
# loop through dsc, orig.tar.gz, and debian.tar.xz files
test -n "${include_sources}" && {
for word in $( sed -n -r -e '/Files:/,/^\s*$/{/^ /p;}' ${workdir}/Sources | awk '{print $NF}' ) ;
do
get_file "${inurl}/${word}"
done
}
} 2>&1 | tee -a "${logfile}"
|
