diff options
21 files changed, 126 insertions, 165 deletions
diff --git a/etc/mirror/README.txt b/etc/mirror/docs/README.txt index 0a5253f..6224d84 100644 --- a/etc/mirror/README.txt +++ b/etc/mirror/docs/README.txt @@ -1,5 +1,5 @@ File: etc/mirror/README.txt -Package: mirror 1.0-5 +Package: mirror 1.0-6 Author: bgstack15@gmail.com Startdate: 2016-06-08 Title: Readme file for mirror @@ -38,6 +38,10 @@ chmod +x /etc/mirror/scripts/{centos,fedora-epel,fedora-releases,fedora-updates, Included in this package are some apache configs. For a nice intro page, a template is provided at /etc/mirror/inc/HEADER.html +## Maintaining your mirror server +Fedora only keeps current the two latest versions. +Ubuntu keeps certain versions current: check http://releases.ubuntu.com/ for which versions. + ### USAGE BLOCK usage: mirror-master.sh [-duV] [ -f | --file /etc/mirror/mirror.conf ] [ --scriptsdir /etc/mirror/scripts ] [ scriptname ] version ${mirrormasterversion} @@ -78,3 +82,6 @@ Added repos for linuxmint 2016-11-11 mirror 1.0-5 adding favicon, licensed CC BY 3.0 by Lorc <http://lorcblog.blogspot.com/> + +2017-01-11 mirror-1.0-6 +Updated all scripts for the bgscripts-1.1-28 directory migration to /usr/share/bgscripts diff --git a/etc/mirror/docs/files-for-versioning.txt b/etc/mirror/docs/files-for-versioning.txt new file mode 100644 index 0000000..e13cc85 --- /dev/null +++ b/etc/mirror/docs/files-for-versioning.txt @@ -0,0 +1,3 @@ +docs/README.txt +docs/mirror.spec +inc/scrub.txt diff --git a/etc/mirror/docs/mirror.spec b/etc/mirror/docs/mirror.spec index 064dcf8..7b1193f 100644 --- a/etc/mirror/docs/mirror.spec +++ b/etc/mirror/docs/mirror.spec @@ -3,7 +3,7 @@ Summary: Mirror Name: mirror Version: 1.0 -Release: 5 +Release: 6 License: CC BY-SA 4.0 Group: Applications/System Source: mirror.tgz @@ -12,7 +12,7 @@ URL: bgstack15@gmail.com #Vendor: Packager: Bgstack15 <bgstack15@gmail.com> Buildarch: noarch -PreReq: bgscripts >= 1.1-17 +PreReq: bgscripts >= 1.1-28 Requires: httpd >= 2.2 %description @@ -39,7 +39,6 @@ rm -rf ${buildroot} %config /etc/sudoers.d/50_mirror-sudo %config /etc/cron.d/mirror.cron /etc/mirror/mirror-master.sh -%doc %attr(444, -, -) /etc/mirror/README.txt /etc/mirror/scripts/ssh /etc/mirror/scripts/libreoffice /etc/mirror/scripts/linuxmint-repos @@ -48,6 +47,7 @@ rm -rf ${buildroot} /etc/mirror/scripts/rsyslog /etc/mirror/scripts/centos /etc/mirror/scripts/fedora-epel +/etc/mirror/scripts/old-scripts.tgz /etc/mirror/scripts/ubuntu-releases /etc/mirror/scripts/linuxmint-isos /etc/mirror/scripts/fedora-releases @@ -58,16 +58,16 @@ rm -rf ${buildroot} /etc/mirror/docs/debian/postrm /etc/mirror/docs/debian/control /etc/mirror/docs/debian/preinst +%doc %attr(444, -, -) /etc/mirror/docs/README.txt /etc/mirror/docs/mirror.spec -/etc/mirror/inc/scrub.py -/etc/mirror/inc/scrub.pyc -/etc/mirror/inc/scrub.pyo +%doc %attr(444, -, -) /etc/mirror/docs/packaging.txt +%doc %attr(444, -, -) /etc/mirror/docs/files-for-versioning.txt %config /etc/mirror/inc/deploy.conf +/etc/mirror/inc/pack /etc/mirror/inc/localize_git.sh /etc/mirror/inc/deploy.sh %doc %attr(444, -, -) /etc/mirror/inc/scrub.txt /etc/mirror/inc/rsync -%doc %attr(444, -, -) /etc/mirror/packaging.txt /etc/mirror/examples/favicon.ico /etc/mirror/examples/FOOTER.html /etc/mirror/examples/HEADER.html diff --git a/etc/mirror/packaging.txt b/etc/mirror/docs/packaging.txt index a97d4c5..a97d4c5 100644 --- a/etc/mirror/packaging.txt +++ b/etc/mirror/docs/packaging.txt diff --git a/etc/mirror/inc/deploy.conf b/etc/mirror/inc/deploy.conf index 4993ba1..5f2e3d6 100644 --- a/etc/mirror/inc/deploy.conf +++ b/etc/mirror/inc/deploy.conf @@ -34,7 +34,7 @@ flavor=centos # base location. Required to function. location=/mnt/mirror/bgscripts -# packagedir is the first place the script checks before placing a destination file. The location is a backup location. The script will evaluate packagedir, so you can use variables here. +# packagedir is the first place the script checks before placing a destination file. $location is a backup spot if packagedir is undefined. The script will evaluate packagedir, so you can use variables here. # default packagedir=<undefined> # packagedir=/mnt/mirror/bgscripts/${packagename} diff --git a/etc/mirror/inc/localize_git.sh b/etc/mirror/inc/localize_git.sh index 35b7ce6..160f168 100755 --- a/etc/mirror/inc/localize_git.sh +++ b/etc/mirror/inc/localize_git.sh @@ -1,2 +1,2 @@ #!/bin/sh -\cp -pRf /home/work/mirror.clean/.git /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-5/ +\cp -pRf /home/work/mirror.clean/.git /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-6/ diff --git a/etc/mirror/inc/pack b/etc/mirror/inc/pack new file mode 100755 index 0000000..64b1121 --- /dev/null +++ b/etc/mirror/inc/pack @@ -0,0 +1,93 @@ +#!/bin/bash +. /usr/share/bgscripts/framework.sh + +type=""; + +case "${thisflavor}" in + redhat|rhel|centos|fedora|korora) type=rpm;; + debian|ubuntu) type=dpkg;; + *) type=targz;; +esac + +echo " $@ " | grep -qiE -- "help|usage|\s-h\s|\s-\?" 1>/dev/null 2>&1 && { + # display help and exit + less -F <<EOF +pack utility version 2017-01-11b-mirror +usage: pack [ rpm | deb | tar | scrub ] [ --debug | -d {0-10} ] +Provides a single command for building a package. This script is customized to each package. +optional arguments: + [ rpm | deb | tar | scrub ] Build that type of package. Scrub calls the scrub.py script. + The default depends on the local os flavor. This system is "${thisflavor}" + [ --debug {0-10} | -d {0-10} ] Display package type to build and exit. Debuglev not implemented here. +EOF +exit 1 +} + +# Derive package name and version number, for my simple package versions only. +fullname="$( pwd | sed -e 's/.*\(SOURCES\|deb\)\///;s/\/.*//;' )" +version="$( echo "${fullname}" | grep -oiE -- "-[0-9]{0,3}\.[0-9a-zA-Z]{0,5}-[0-9]{0,4}" | sed -e 's/^-//;' )" +shortversion="$( echo "${version}" | sed -e 's/-.*//;' )" +package="$( echo "${fullname}" | sed -e "s/-${version}//;" )" + +echo " $@ " | grep -qiE -- "rpm|rhel|redhat|centos|fedora|korora" 1>/dev/null 2>&1 && type=rpm +echo " $@ " | grep -qiE -- "debian|ubuntu|deb\s|dpkg" 1>/dev/null 2>&1 && type=dpkg +echo " $@ " | grep -qiE -- "tar|tgz|gz" 1>/dev/null 2>&1 && type=targz +echo " $@ " | grep -qiE -- "scrub" 1>/dev/null 2>&1 && type=scrub + +# Display information +echo "packaging ${package}-${version} as ${type}" +echo " $@ " | grep -qiE -- "debug|\s-d[ 0-9]{0,3}\s" 1>/dev/null 2>&1 && exit 0 + +case "${type}" in + rpm) + +########## PACKAGING for rhel/centos +# if you copy-paste this, be sure to define package, version, shortversion +rpmbuilddir=~/rpmbuild/ +packagespecfile="${package}-${version}/etc/${package}/docs/${package}.spec" +sed -i -n '1,/^\%files$/p;' "${rpmbuilddir}/SOURCES/${packagespecfile}" +cd "${rpmbuilddir}/SOURCES/${package}-${version}" +find * ! -type d ! -regex '.*?.swp' ! -regex '.*?DEBIAN.*?' | sed -e 's/^/\//;' -e 's/\(.*\.txt\)/%doc %attr(444, -, -) \1/;' -e 's/\(.*\.\(conf\|cron\)\|.*sudoers\.d\)/%config \1/;' -e 's/\(.*bin.*\)/%verify(link) \1/;' -e 's/\(.*\)\.py$/\1\.py\n\1\.pyc\n\1\.pyo/;' >> "${rpmbuilddir}/SOURCES/${packagespecfile}" +rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}" +cp -prf "${rpmbuilddir}/SOURCES/${package}-${version}" "${rpmbuilddir}/SOURCES/${package}-${shortversion}" +rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}/DEBIAN" +cd "${rpmbuilddir}/SOURCES" +rm -rf "${package}.tgz"; tar -zc --exclude='.git' -f "${package}.tgz" "${package}-${shortversion}" && rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}/" +cp -p "${rpmbuilddir}/SOURCES/${packagespecfile}" "${rpmbuilddir}/SPECS" +cd "${rpmbuilddir}/RPMS/noarch" +rpmbuild -bb "${rpmbuilddir}/SPECS/${package}.spec" +rm -rf "${rpmbuilddir}/SOURCES/${package}.tgz" + + ;; + dpkg) + +########## PACKAGING for ubuntu +# You need package dpkg-dev to build packages. +# if you copy-paste this, be sure to define package, version +debdir=~/deb +echo "Gotta say unh. Deb format for ${package} has not yet been implemented. Aborted." +exit 1 + + ;; + targz) + +########## PACKAGING in a master.tgz +# if you copy-paste this, be sure to define package, version +cd ~/deb 2>/dev/null || cd ~/rpmbuild/SOURCES +rm -rf "./${package}-${version}.master.tgz" +tar -zcf "${package}-${version}.master.tgz" "${package}-${version}/" + + ;; + scrub) + +########## SCRUB for publication +# if you copy-paste this, be sure to define package, version +cd ~/deb 2>/dev/null || cd ~/rpmbuild/SOURCES +cd "$( find . -name "*scrub.txt" | grep -iE "${package}-${version}\/" | xargs dirname )" +/usr/share/bgscripts/scrub.py + + ;; + unknown) + echo "error: check $0 for errors on type ${type}." 1>&2 + ;; +esac diff --git a/etc/mirror/inc/scrub.py b/etc/mirror/inc/scrub.py deleted file mode 100755 index 01ddbb0..0000000 --- a/etc/mirror/inc/scrub.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/bin/env python3 -# Filename: scrub.py -# Location: Various -# Author: bgstack15@gmail.com -# Startdate: 2016-09-28 -# Title: Script that Simultaneously Copies and Scrubs a Directory -# Purpose: Prepare projects for publication by removing private information like usernames and hostnames -# Package: Various -# History: -# 2016-10-03 working on batch rename files -# 2016-10-20 added not ".tgz" in source.name -# 2016-10-27 Fixed error when trying chmod on a symlink -# 2016-10-31 Handle symlinks by duplicating the link and stopping there. -# Also separated directory, filename renaming tasks -# Usage: -# Store this file with any package that gets published. Adjust scrub.txt in local directory. -# # First line: source directory Second line: target directory. WILL BE OVERWRITTEN! -# /etc/ansible -# /home/bjones/ansible.clean -# # Rest of the lines are "OLD WORD" "NEW WORD" -# bjones bgstack15 -# rsmith rmstack15 -# Reference: -# http://stackoverflow.com/questions/79968/split-a-string-by-spaces-preserving-quoted-substrings-in-python/524796#524796 -# http://stackoverflow.com/questions/6706953/python-using-subprocess-to-call-sed#6707003 -# http://stackoverflow.com/questions/6584871/remove-last-character-if-its-a-backslash/6584893#6584893 -# http://stackoverflow.com/questions/2212643/python-recursive-folder-read/2212728#2212728 -# parallel lists: http://stackoverflow.com/questions/1663807/how-can-i-iterate-through-two-lists-in-parallel-in-python -# file renames http://stackoverflow.com/questions/225735/batch-renaming-of-files-in-a-directory/7917798#7917798 -# Improve: -# Add option to specify scrub file -# Add exclude option to scrub file, such as .git and so on -# Accept CLI options like source, destination, even exclusions? -# Add flag for performing file renames as well, or file renames only -import re, shlex, os, sys, shutil -from pathlib import Path - -# scrubpy version -scrubpyversion = "2016-11-11a" - -# Define functions - -def removeComments(string): - #string = re.sub(re.compile("/\*.*?\*/",re.DOTALL ) ,"", string) - #string = re.sub(re.compile("//.*?\n" ) ,"" ,string) - pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|(//|#)[^\r\n]*$)" - regex = re.compile(pattern, re.MULTILINE|re.DOTALL) - def _replacer(match): - if match.group(2) is not None: - return "" - else: - return match.group(1) - return regex.sub(_replacer, string) - -def isValidFile(_thisstring): - # return true if not png, tgz, or other non-text file - _isValidFile=True - if re.compile('.*\.(tgz|png|gif|jpg|pyc|pyo|git|swp|ico)').match(_thisstring): - _isValidFile=False - #print( _thisstring + ": " + str(_isValidFile) ) - return _isValidFile - -# Main code -stringfile = open('scrub.txt','r') -count=0 -thisdir="" -newdir="" -oldstrings=[] -newstrings=[] - -while True: - x = stringfile.readline().rstrip() - count += 1 - if not x: break - x = removeComments(x) - #print("x=" + x) - y = shlex.split (x) - if len(y) >= 1: - if thisdir == "": - thisdir = y[0] - elif newdir == "": - newdir = y[0] - if len(y) >= 2: - #print("y[0]=" + y[0] + "\t and y[1]=" + y[1]) - oldstrings.append(y[0]) - newstrings.append(y[1]) - -# After the file is done -stringfile.close() -#newdir = thisdir.rstrip('\/') + ".scrubbed/" - -if False: - print("\nthisdir=" + thisdir) - print("newdir=" + newdir + '\n') - print("oldstrings are:") - print(oldstrings) - print("newstrings are:") - print(newstrings) - -# Clean scrubbed directory -try: - shutil.rmtree(newdir) -except: - foo=1 - -shutil.copytree(thisdir,newdir,symlinks=True) - -# Execute substitutions -for rootfolder, subdirs, files in os.walk(thisdir): - for filename in files: - sourcepath = os.path.join(rootfolder, filename) - destdir = rootfolder.replace(thisdir.rstrip('\/'),newdir.rstrip('\/')) - destfile = os.path.join(destdir, filename) - if os.path.islink( sourcepath ): - _symlinktarget = os.readlink( sourcepath ) - try: - os.symlink( _symlinktarget, destfile ) - except Exception as e: - pass - else: - with open( sourcepath, "r" ) as source: - #if not ".swp" in source.name and not ".git" in source.name and not ".tgz" in source.name: - if isValidFile( source.name ): - #print("sourcefile=" + source.name) - #print("destfile=" + destfile + '\n') - with open( destfile, "w") as target: - data = source.read() - for oldword, newword in zip(oldstrings, newstrings): - data = data.replace(oldword,newword) - changed = data - target.write(changed) - -# Execute directory renames -for rootfolder, subdirs, files in os.walk(newdir): - for subdir in subdirs: - oldpath = os.path.join(rootfolder, subdir) - for oldword, newword in zip(oldstrings, newstrings): - if oldword in oldpath: - os.rename(oldpath, oldpath.replace(oldword,newword)) - -# Execute file renames -# Used "file renames" reference, as well as the structure of directory traversal used earlier, which was from a different source. -for rootfolder, subdirs, files in os.walk(newdir): - for filename in files: - oldpath = os.path.join(rootfolder, filename) - for oldword, newword in zip(oldstrings, newstrings): - if oldword in oldpath: - #print("oldword=" + oldword + "\toldpath=" + oldpath) - os.rename(oldpath, oldpath.replace(oldword,newword)) diff --git a/etc/mirror/inc/scrub.txt b/etc/mirror/inc/scrub.txt index 153263f..9f1a86d 100644 --- a/etc/mirror/inc/scrub.txt +++ b/etc/mirror/inc/scrub.txt @@ -1,15 +1,21 @@ -# First line: source directory Second line: target directory. WILL BE OVERWRITTEN! -/home/bgstack15/rpmbuild/SOURCES/mirror-1.0-5/ -/home/work/mirror.clean +# for scrubpy version 2016-11-16a or newer +source /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-6/ +target /home/work/mirror.clean +ignore "tgz png gif jpg pyc pyo git swp ico" # rest of the lines are "OLD WORD" "NEW WORD" BGSTACK15 BGSTACK15 Bgstack15 Bgstack15 bgstack15 bgstack15 +Bname Bname +bname bname +Stack15 Stack15 +stack15 stack15 Alice Alice User1 User1 auser1 auser1 example example EXAMPLE EXAMPLE ".com" ".com" +".COM" ".COM" "203.0." "203.0." one one diff --git a/etc/mirror/scripts/centos b/etc/mirror/scripts/centos index b812bde..b812bde 100644..100755 --- a/etc/mirror/scripts/centos +++ b/etc/mirror/scripts/centos diff --git a/etc/mirror/scripts/fedora-epel b/etc/mirror/scripts/fedora-epel index 88f0c77..88f0c77 100644..100755 --- a/etc/mirror/scripts/fedora-epel +++ b/etc/mirror/scripts/fedora-epel diff --git a/etc/mirror/scripts/fedora-releases b/etc/mirror/scripts/fedora-releases index 8f33c46..b7d0928 100644..100755 --- a/etc/mirror/scripts/fedora-releases +++ b/etc/mirror/scripts/fedora-releases @@ -4,4 +4,5 @@ source /etc/mirror/inc/rsync -rmirror mirrors.kernel.org::fedora/releases fedora/linux/releases 21/ test/ */Live */debug */armhfp +rmirror mirrors.kernel.org::fedora/releases fedora/linux/releases 21/ 22/ 23/ test/ */Live */debug */armhfp + diff --git a/etc/mirror/scripts/fedora-updates b/etc/mirror/scripts/fedora-updates index 44cfe02..4ba729d 100644..100755 --- a/etc/mirror/scripts/fedora-updates +++ b/etc/mirror/scripts/fedora-updates @@ -4,4 +4,4 @@ source /etc/mirror/inc/rsync -rmirror mirror.lstn.net::fedora-enchilada/updates fedora/linux/updates '[89]/' '1[0-9]/' 20/ testing/ */ppc */ppc64 */armhfp +rmirror mirror.lstn.net::fedora-enchilada/updates fedora/linux/updates '[89]/' '1[0-9]/' '2[0-3]/' testing/ */ppc */ppc64 */armhfp diff --git a/etc/mirror/scripts/libreoffice b/etc/mirror/scripts/libreoffice index cf88808..cf88808 100644..100755 --- a/etc/mirror/scripts/libreoffice +++ b/etc/mirror/scripts/libreoffice diff --git a/etc/mirror/scripts/linuxmint-isos b/etc/mirror/scripts/linuxmint-isos index 674f96e..674f96e 100644..100755 --- a/etc/mirror/scripts/linuxmint-isos +++ b/etc/mirror/scripts/linuxmint-isos diff --git a/etc/mirror/scripts/linuxmint-repos b/etc/mirror/scripts/linuxmint-repos index d73136c..d73136c 100644..100755 --- a/etc/mirror/scripts/linuxmint-repos +++ b/etc/mirror/scripts/linuxmint-repos diff --git a/etc/mirror/scripts/old-scripts.tgz b/etc/mirror/scripts/old-scripts.tgz Binary files differnew file mode 100644 index 0000000..c4abc34 --- /dev/null +++ b/etc/mirror/scripts/old-scripts.tgz diff --git a/etc/mirror/scripts/putty b/etc/mirror/scripts/putty index 82e7edd..82e7edd 100644..100755 --- a/etc/mirror/scripts/putty +++ b/etc/mirror/scripts/putty diff --git a/etc/mirror/scripts/ssh b/etc/mirror/scripts/ssh index d08f629..d08f629 100644..100755 --- a/etc/mirror/scripts/ssh +++ b/etc/mirror/scripts/ssh diff --git a/etc/mirror/scripts/ubuntu b/etc/mirror/scripts/ubuntu index 9571444..b575ba6 100644..100755 --- a/etc/mirror/scripts/ubuntu +++ b/etc/mirror/scripts/ubuntu @@ -4,4 +4,4 @@ source /etc/mirror/inc/rsync -rmirror archive.ubuntu.com::ubuntu ubuntu/ubuntu *breezy* *warty* *hoary* *dapper* *edgy* *feisty* gutsy* intrepid* jaunty* karmic* maverick* natty* oneiric* quantal* raring* saucy* *[_-]powerpc* *[_-]sparc* *[_-]ia64* pool/**/*.tar.* *.diff.gz* *.dsc source/ +rmirror archive.ubuntu.com::ubuntu ubuntu/ubuntu *breezy* *warty* *hoary* *dapper* *edgy* *feisty* gutsy* intrepid* jaunty* karmic* lucid* maverick* natty* oneiric* quantal* raring* saucy* vivid* wily* *[_-]powerpc* *[_-]sparc* *[_-]ia64* pool/**/*.tar.* *.diff.gz* *.dsc source/ diff --git a/etc/mirror/scripts/ubuntu-releases b/etc/mirror/scripts/ubuntu-releases index 14a2406..14a2406 100644..100755 --- a/etc/mirror/scripts/ubuntu-releases +++ b/etc/mirror/scripts/ubuntu-releases |