preparing for bgscripts-1.1-28

21 files changed, 126 insertions, 165 deletions

diff --git a/etc/mirror/README.txt b/etc/mirror/docs/README.txt
index 0a5253f..6224d84 100644
--- a/etc/mirror/README.txt
+++ b/etc/mirror/docs/README.txt
@@ -1,5 +1,5 @@
 File: etc/mirror/README.txt
-Package: mirror 1.0-5
+Package: mirror 1.0-6
 Author: bgstack15@gmail.com
 Startdate: 2016-06-08
 Title: Readme file for mirror
@@ -38,6 +38,10 @@ chmod +x /etc/mirror/scripts/{centos,fedora-epel,fedora-releases,fedora-updates,
 Included in this package are some apache configs.
 For a nice intro page, a template is provided at /etc/mirror/inc/HEADER.html
 
+## Maintaining your mirror server
+Fedora only keeps current the two latest versions.
+Ubuntu keeps certain versions current: check http://releases.ubuntu.com/ for which versions.
+
 ### USAGE BLOCK
 usage: mirror-master.sh [-duV] [ -f | --file /etc/mirror/mirror.conf ] [ --scriptsdir /etc/mirror/scripts ] [ scriptname ]
 version ${mirrormasterversion}
@@ -78,3 +82,6 @@ Added repos for linuxmint
 
 2016-11-11 mirror 1.0-5
 adding favicon, licensed CC BY 3.0 by Lorc <http://lorcblog.blogspot.com/>
+
+2017-01-11 mirror-1.0-6
+Updated all scripts for the bgscripts-1.1-28 directory migration to /usr/share/bgscripts
diff --git a/etc/mirror/docs/files-for-versioning.txt b/etc/mirror/docs/files-for-versioning.txt
new file mode 100644
index 0000000..e13cc85
--- /dev/null
+++ b/etc/mirror/docs/files-for-versioning.txt
@@ -0,0 +1,3 @@
+docs/README.txt
+docs/mirror.spec
+inc/scrub.txt
diff --git a/etc/mirror/docs/mirror.spec b/etc/mirror/docs/mirror.spec
index 064dcf8..7b1193f 100644
--- a/etc/mirror/docs/mirror.spec
+++ b/etc/mirror/docs/mirror.spec
@@ -3,7 +3,7 @@
 Summary: Mirror
 Name: mirror
 Version: 1.0
-Release: 5
+Release: 6
 License: CC BY-SA 4.0
 Group: Applications/System
 Source: mirror.tgz
@@ -12,7 +12,7 @@ URL: bgstack15@gmail.com
 #Vendor:
 Packager: Bgstack15 <bgstack15@gmail.com>
 Buildarch: noarch
-PreReq: bgscripts >= 1.1-17
+PreReq: bgscripts >= 1.1-28
 Requires: httpd >= 2.2
 
 %description
@@ -39,7 +39,6 @@ rm -rf ${buildroot}
 %config /etc/sudoers.d/50_mirror-sudo
 %config /etc/cron.d/mirror.cron
 /etc/mirror/mirror-master.sh
-%doc %attr(444, -, -) /etc/mirror/README.txt
 /etc/mirror/scripts/ssh
 /etc/mirror/scripts/libreoffice
 /etc/mirror/scripts/linuxmint-repos
@@ -48,6 +47,7 @@ rm -rf ${buildroot}
 /etc/mirror/scripts/rsyslog
 /etc/mirror/scripts/centos
 /etc/mirror/scripts/fedora-epel
+/etc/mirror/scripts/old-scripts.tgz
 /etc/mirror/scripts/ubuntu-releases
 /etc/mirror/scripts/linuxmint-isos
 /etc/mirror/scripts/fedora-releases
@@ -58,16 +58,16 @@ rm -rf ${buildroot}
 /etc/mirror/docs/debian/postrm
 /etc/mirror/docs/debian/control
 /etc/mirror/docs/debian/preinst
+%doc %attr(444, -, -) /etc/mirror/docs/README.txt
 /etc/mirror/docs/mirror.spec
-/etc/mirror/inc/scrub.py
-/etc/mirror/inc/scrub.pyc
-/etc/mirror/inc/scrub.pyo
+%doc %attr(444, -, -) /etc/mirror/docs/packaging.txt
+%doc %attr(444, -, -) /etc/mirror/docs/files-for-versioning.txt
 %config /etc/mirror/inc/deploy.conf
+/etc/mirror/inc/pack
 /etc/mirror/inc/localize_git.sh
 /etc/mirror/inc/deploy.sh
 %doc %attr(444, -, -) /etc/mirror/inc/scrub.txt
 /etc/mirror/inc/rsync
-%doc %attr(444, -, -) /etc/mirror/packaging.txt
 /etc/mirror/examples/favicon.ico
 /etc/mirror/examples/FOOTER.html
 /etc/mirror/examples/HEADER.html
diff --git a/etc/mirror/packaging.txt b/etc/mirror/docs/packaging.txt
index a97d4c5..a97d4c5 100644
--- a/etc/mirror/packaging.txt
+++ b/etc/mirror/docs/packaging.txt
diff --git a/etc/mirror/inc/deploy.conf b/etc/mirror/inc/deploy.conf
index 4993ba1..5f2e3d6 100644
--- a/etc/mirror/inc/deploy.conf
+++ b/etc/mirror/inc/deploy.conf
@@ -34,7 +34,7 @@ flavor=centos
 # base location. Required to function.
 location=/mnt/mirror/bgscripts
 
-# packagedir is the first place the script checks before placing a destination file. The location is a backup location. The script will evaluate packagedir, so you can use variables here.
+# packagedir is the first place the script checks before placing a destination file. $location is a backup spot if packagedir is undefined. The script will evaluate packagedir, so you can use variables here.
 # default packagedir=<undefined>
 # packagedir=/mnt/mirror/bgscripts/${packagename}
 
diff --git a/etc/mirror/inc/localize_git.sh b/etc/mirror/inc/localize_git.sh
index 35b7ce6..160f168 100755
--- a/etc/mirror/inc/localize_git.sh
+++ b/etc/mirror/inc/localize_git.sh
@@ -1,2 +1,2 @@
 #!/bin/sh
-\cp -pRf /home/work/mirror.clean/.git /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-5/
+\cp -pRf /home/work/mirror.clean/.git /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-6/
diff --git a/etc/mirror/inc/pack b/etc/mirror/inc/pack
new file mode 100755
index 0000000..64b1121
--- /dev/null
+++ b/etc/mirror/inc/pack
@@ -0,0 +1,93 @@
+#!/bin/bash
+. /usr/share/bgscripts/framework.sh
+
+type="";
+
+case "${thisflavor}" in
+   redhat|rhel|centos|fedora|korora) type=rpm;;
+   debian|ubuntu) type=dpkg;;
+   *) type=targz;;
+esac
+
+echo " $@ " | grep -qiE -- "help|usage|\s-h\s|\s-\?" 1>/dev/null 2>&1 && {
+   # display help and exit
+   less -F <<EOF
+pack utility version 2017-01-11b-mirror
+usage: pack [ rpm | deb | tar | scrub ] [ --debug | -d {0-10} ]
+Provides a single command for building a package. This script is customized to each package.
+optional arguments:
+ [ rpm | deb | tar | scrub ]    Build that type of package. Scrub calls the scrub.py script.
+                                The default depends on the local os flavor. This system is "${thisflavor}"
+ [ --debug {0-10} | -d {0-10} ] Display package type to build and exit. Debuglev not implemented here.
+EOF
+exit 1
+}
+
+# Derive package name and version number, for my simple package versions only.
+fullname="$( pwd | sed -e 's/.*\(SOURCES\|deb\)\///;s/\/.*//;' )"
+version="$( echo "${fullname}" | grep -oiE -- "-[0-9]{0,3}\.[0-9a-zA-Z]{0,5}-[0-9]{0,4}" | sed -e 's/^-//;' )"
+shortversion="$( echo "${version}" | sed -e 's/-.*//;' )"
+package="$( echo "${fullname}" | sed -e "s/-${version}//;" )"
+
+echo " $@ " | grep -qiE -- "rpm|rhel|redhat|centos|fedora|korora" 1>/dev/null 2>&1 && type=rpm
+echo " $@ " | grep -qiE -- "debian|ubuntu|deb\s|dpkg" 1>/dev/null 2>&1 && type=dpkg
+echo " $@ " | grep -qiE -- "tar|tgz|gz" 1>/dev/null 2>&1 && type=targz
+echo " $@ " | grep -qiE -- "scrub" 1>/dev/null 2>&1 && type=scrub
+
+# Display information
+echo "packaging ${package}-${version} as ${type}"
+echo " $@ " | grep -qiE -- "debug|\s-d[ 0-9]{0,3}\s" 1>/dev/null 2>&1 && exit 0
+
+case "${type}" in
+   rpm)
+
+########## PACKAGING for rhel/centos
+# if you copy-paste this, be sure to define package, version, shortversion
+rpmbuilddir=~/rpmbuild/
+packagespecfile="${package}-${version}/etc/${package}/docs/${package}.spec"
+sed -i -n '1,/^\%files$/p;' "${rpmbuilddir}/SOURCES/${packagespecfile}"
+cd "${rpmbuilddir}/SOURCES/${package}-${version}"
+find * ! -type d ! -regex '.*?.swp' ! -regex '.*?DEBIAN.*?' | sed -e 's/^/\//;' -e 's/\(.*\.txt\)/%doc %attr(444, -, -) \1/;' -e 's/\(.*\.\(conf\|cron\)\|.*sudoers\.d\)/%config \1/;' -e 's/\(.*bin.*\)/%verify(link) \1/;' -e 's/\(.*\)\.py$/\1\.py\n\1\.pyc\n\1\.pyo/;' >> "${rpmbuilddir}/SOURCES/${packagespecfile}"
+rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}"
+cp -prf "${rpmbuilddir}/SOURCES/${package}-${version}" "${rpmbuilddir}/SOURCES/${package}-${shortversion}"
+rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}/DEBIAN"
+cd "${rpmbuilddir}/SOURCES"
+rm -rf "${package}.tgz"; tar -zc --exclude='.git' -f "${package}.tgz" "${package}-${shortversion}" && rm -rf "${rpmbuilddir}/SOURCES/${package}-${shortversion}/"
+cp -p "${rpmbuilddir}/SOURCES/${packagespecfile}" "${rpmbuilddir}/SPECS"
+cd "${rpmbuilddir}/RPMS/noarch"
+rpmbuild -bb "${rpmbuilddir}/SPECS/${package}.spec"
+rm -rf "${rpmbuilddir}/SOURCES/${package}.tgz"
+
+      ;;
+   dpkg)
+
+########## PACKAGING for ubuntu
+# You need package dpkg-dev to build packages.
+# if you copy-paste this, be sure to define package, version
+debdir=~/deb
+echo "Gotta say unh. Deb format for ${package} has not yet been implemented. Aborted."
+exit 1
+
+      ;;
+   targz)
+
+########## PACKAGING in a master.tgz
+# if you copy-paste this, be sure to define package, version
+cd ~/deb 2>/dev/null || cd ~/rpmbuild/SOURCES
+rm -rf "./${package}-${version}.master.tgz"
+tar -zcf "${package}-${version}.master.tgz" "${package}-${version}/"
+
+      ;;
+   scrub)
+      
+########## SCRUB for publication
+# if you copy-paste this, be sure to define package, version
+cd ~/deb 2>/dev/null || cd ~/rpmbuild/SOURCES
+cd "$( find . -name "*scrub.txt" | grep -iE "${package}-${version}\/" | xargs dirname )"
+/usr/share/bgscripts/scrub.py
+
+      ;;
+   unknown)
+      echo "error: check $0 for errors on type ${type}." 1>&2
+      ;;
+esac
diff --git a/etc/mirror/inc/scrub.py b/etc/mirror/inc/scrub.py
deleted file mode 100755
index 01ddbb0..0000000
--- a/etc/mirror/inc/scrub.py
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/bin/env python3
-# Filename: scrub.py
-# Location: Various
-# Author: bgstack15@gmail.com
-# Startdate: 2016-09-28
-# Title: Script that Simultaneously Copies and Scrubs a Directory
-# Purpose: Prepare projects for publication by removing private information like usernames and hostnames
-# Package: Various
-# History:
-#    2016-10-03 working on batch rename files
-#    2016-10-20 added not ".tgz" in source.name
-#    2016-10-27 Fixed error when trying chmod on a symlink
-#    2016-10-31 Handle symlinks by duplicating the link and stopping there.
-#               Also separated directory, filename renaming tasks
-# Usage:
-#    Store this file with any package that gets published. Adjust scrub.txt in local directory.
-#  # First line: source directory      Second line: target directory. WILL BE OVERWRITTEN!
-#  /etc/ansible
-#  /home/bjones/ansible.clean
-#  # Rest of the lines are "OLD WORD" "NEW WORD"
-#  bjones bgstack15
-#  rsmith rmstack15
-# Reference:
-#    http://stackoverflow.com/questions/79968/split-a-string-by-spaces-preserving-quoted-substrings-in-python/524796#524796
-#    http://stackoverflow.com/questions/6706953/python-using-subprocess-to-call-sed#6707003
-#    http://stackoverflow.com/questions/6584871/remove-last-character-if-its-a-backslash/6584893#6584893
-#    http://stackoverflow.com/questions/2212643/python-recursive-folder-read/2212728#2212728
-#    parallel lists: http://stackoverflow.com/questions/1663807/how-can-i-iterate-through-two-lists-in-parallel-in-python
-#    file renames http://stackoverflow.com/questions/225735/batch-renaming-of-files-in-a-directory/7917798#7917798
-# Improve:
-#    Add option to specify scrub file
-#    Add exclude option to scrub file, such as .git and so on
-#    Accept CLI options like source, destination, even exclusions?
-#    Add flag for performing file renames as well, or file renames only
-import re, shlex, os, sys, shutil
-from pathlib import Path
-
-# scrubpy version
-scrubpyversion = "2016-11-11a"
-
-# Define functions
-
-def removeComments(string):
-   #string = re.sub(re.compile("/\*.*?\*/",re.DOTALL ) ,"", string)
-   #string = re.sub(re.compile("//.*?\n" ) ,"" ,string)
-   pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|(//|#)[^\r\n]*$)"
-   regex = re.compile(pattern, re.MULTILINE|re.DOTALL)
-   def _replacer(match):
-      if match.group(2) is not None:
-         return ""
-      else:
-         return match.group(1)
-   return regex.sub(_replacer, string)
-
-def isValidFile(_thisstring):
-   # return true if not png, tgz, or other non-text file
-   _isValidFile=True
-   if re.compile('.*\.(tgz|png|gif|jpg|pyc|pyo|git|swp|ico)').match(_thisstring):
-      _isValidFile=False
-   #print( _thisstring + ": " + str(_isValidFile) )
-   return _isValidFile
-
-# Main code
-stringfile = open('scrub.txt','r')
-count=0
-thisdir=""
-newdir=""
-oldstrings=[]
-newstrings=[]
-
-while True:
-   x = stringfile.readline().rstrip()
-   count += 1
-   if not x: break
-   x = removeComments(x)
-   #print("x=" + x)
-   y = shlex.split (x)
-   if len(y) >= 1:
-      if thisdir == "":
-         thisdir = y[0]
-      elif newdir == "":
-         newdir = y[0]
-   if len(y) >= 2: 
-      #print("y[0]=" + y[0] + "\t and y[1]=" + y[1])
-      oldstrings.append(y[0])
-      newstrings.append(y[1])
-
-# After the file is done
-stringfile.close()
-#newdir = thisdir.rstrip('\/') + ".scrubbed/"
-
-if False:
-   print("\nthisdir=" + thisdir)
-   print("newdir=" + newdir + '\n')
-   print("oldstrings are:")
-   print(oldstrings)
-   print("newstrings are:")
-   print(newstrings)
-
-# Clean scrubbed directory
-try:
-   shutil.rmtree(newdir)
-except:
-   foo=1
-
-shutil.copytree(thisdir,newdir,symlinks=True)
-
-# Execute substitutions
-for rootfolder, subdirs, files in os.walk(thisdir):
-   for filename in files:
-      sourcepath = os.path.join(rootfolder, filename)
-      destdir = rootfolder.replace(thisdir.rstrip('\/'),newdir.rstrip('\/'))
-      destfile = os.path.join(destdir, filename)
-      if os.path.islink( sourcepath ):
-         _symlinktarget = os.readlink( sourcepath )
-         try:
-            os.symlink( _symlinktarget, destfile )
-         except Exception as e:
-            pass
-      else:
-         with open( sourcepath, "r" ) as source:
-            #if not ".swp" in source.name and not ".git" in source.name and not ".tgz" in source.name:
-            if isValidFile( source.name ):
-               #print("sourcefile=" + source.name)
-               #print("destfile=" + destfile + '\n')
-               with open( destfile, "w") as target:
-                  data = source.read()
-                  for oldword, newword in zip(oldstrings, newstrings):
-                     data = data.replace(oldword,newword)
-                  changed = data
-                  target.write(changed)
-
-# Execute directory renames
-for rootfolder, subdirs, files in os.walk(newdir):
-   for subdir in subdirs:
-      oldpath = os.path.join(rootfolder, subdir)
-      for oldword, newword in zip(oldstrings, newstrings):
-         if oldword in oldpath:
-            os.rename(oldpath, oldpath.replace(oldword,newword))
-
-# Execute file renames
-# Used "file renames" reference, as well as the structure of directory traversal used earlier, which was from a different source.
-for rootfolder, subdirs, files in os.walk(newdir):
-   for filename in files:
-      oldpath = os.path.join(rootfolder, filename)
-      for oldword, newword in zip(oldstrings, newstrings):
-         if oldword in oldpath:
-            #print("oldword=" + oldword + "\toldpath=" + oldpath)
-            os.rename(oldpath, oldpath.replace(oldword,newword))
diff --git a/etc/mirror/inc/scrub.txt b/etc/mirror/inc/scrub.txt
index 153263f..9f1a86d 100644
--- a/etc/mirror/inc/scrub.txt
+++ b/etc/mirror/inc/scrub.txt
@@ -1,15 +1,21 @@
-# First line: source directory		Second line: target directory. WILL BE OVERWRITTEN!
-/home/bgstack15/rpmbuild/SOURCES/mirror-1.0-5/
-/home/work/mirror.clean
+# for scrubpy version 2016-11-16a or newer
+source /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-6/
+target /home/work/mirror.clean
+ignore "tgz png gif jpg pyc pyo git swp ico"
 # rest of the lines are "OLD WORD" "NEW WORD"
 BGSTACK15	BGSTACK15
 Bgstack15	Bgstack15
 bgstack15 bgstack15
+Bname	Bname
+bname	bname
+Stack15	Stack15
+stack15	stack15
 Alice	Alice
 User1	User1
 auser1	auser1
 example	example
 EXAMPLE	EXAMPLE
 ".com"	".com"
+".COM"	".COM"
 "203.0."		"203.0."
 one	one
diff --git a/etc/mirror/scripts/centos b/etc/mirror/scripts/centos
index b812bde..b812bde 100644..100755
--- a/etc/mirror/scripts/centos
+++ b/etc/mirror/scripts/centos
diff --git a/etc/mirror/scripts/fedora-epel b/etc/mirror/scripts/fedora-epel
index 88f0c77..88f0c77 100644..100755
--- a/etc/mirror/scripts/fedora-epel
+++ b/etc/mirror/scripts/fedora-epel
diff --git a/etc/mirror/scripts/fedora-releases b/etc/mirror/scripts/fedora-releases
index 8f33c46..b7d0928 100644..100755
--- a/etc/mirror/scripts/fedora-releases
+++ b/etc/mirror/scripts/fedora-releases
@@ -4,4 +4,5 @@
 
 source /etc/mirror/inc/rsync
 
-rmirror mirrors.kernel.org::fedora/releases fedora/linux/releases 21/ test/ */Live */debug */armhfp
+rmirror mirrors.kernel.org::fedora/releases fedora/linux/releases 21/ 22/ 23/ test/ */Live */debug */armhfp
+
diff --git a/etc/mirror/scripts/fedora-updates b/etc/mirror/scripts/fedora-updates
index 44cfe02..4ba729d 100644..100755
--- a/etc/mirror/scripts/fedora-updates
+++ b/etc/mirror/scripts/fedora-updates
@@ -4,4 +4,4 @@
 
 source /etc/mirror/inc/rsync
 
-rmirror mirror.lstn.net::fedora-enchilada/updates fedora/linux/updates '[89]/' '1[0-9]/' 20/ testing/ */ppc */ppc64 */armhfp
+rmirror mirror.lstn.net::fedora-enchilada/updates fedora/linux/updates '[89]/' '1[0-9]/' '2[0-3]/' testing/ */ppc */ppc64 */armhfp
diff --git a/etc/mirror/scripts/libreoffice b/etc/mirror/scripts/libreoffice
index cf88808..cf88808 100644..100755
--- a/etc/mirror/scripts/libreoffice
+++ b/etc/mirror/scripts/libreoffice
diff --git a/etc/mirror/scripts/linuxmint-isos b/etc/mirror/scripts/linuxmint-isos
index 674f96e..674f96e 100644..100755
--- a/etc/mirror/scripts/linuxmint-isos
+++ b/etc/mirror/scripts/linuxmint-isos
diff --git a/etc/mirror/scripts/linuxmint-repos b/etc/mirror/scripts/linuxmint-repos
index d73136c..d73136c 100644..100755
--- a/etc/mirror/scripts/linuxmint-repos
+++ b/etc/mirror/scripts/linuxmint-repos
diff --git a/etc/mirror/scripts/old-scripts.tgz b/etc/mirror/scripts/old-scripts.tgz
new file mode 100644
index 0000000..c4abc34
--- /dev/null
+++ b/etc/mirror/scripts/old-scripts.tgz
diff --git a/etc/mirror/scripts/putty b/etc/mirror/scripts/putty
index 82e7edd..82e7edd 100644..100755
--- a/etc/mirror/scripts/putty
+++ b/etc/mirror/scripts/putty
diff --git a/etc/mirror/scripts/ssh b/etc/mirror/scripts/ssh
index d08f629..d08f629 100644..100755
--- a/etc/mirror/scripts/ssh
+++ b/etc/mirror/scripts/ssh
diff --git a/etc/mirror/scripts/ubuntu b/etc/mirror/scripts/ubuntu
index 9571444..b575ba6 100644..100755
--- a/etc/mirror/scripts/ubuntu
+++ b/etc/mirror/scripts/ubuntu
@@ -4,4 +4,4 @@
 
 source /etc/mirror/inc/rsync
 
-rmirror archive.ubuntu.com::ubuntu ubuntu/ubuntu *breezy* *warty* *hoary* *dapper* *edgy* *feisty* gutsy* intrepid* jaunty* karmic* maverick* natty* oneiric* quantal* raring* saucy*  *[_-]powerpc* *[_-]sparc* *[_-]ia64* pool/**/*.tar.* *.diff.gz* *.dsc source/
+rmirror archive.ubuntu.com::ubuntu ubuntu/ubuntu *breezy* *warty* *hoary* *dapper* *edgy* *feisty* gutsy* intrepid* jaunty* karmic* lucid* maverick* natty* oneiric* quantal* raring* saucy* vivid* wily*  *[_-]powerpc* *[_-]sparc* *[_-]ia64* pool/**/*.tar.* *.diff.gz* *.dsc source/
diff --git a/etc/mirror/scripts/ubuntu-releases b/etc/mirror/scripts/ubuntu-releases
index 14a2406..14a2406 100644..100755
--- a/etc/mirror/scripts/ubuntu-releases
+++ b/etc/mirror/scripts/ubuntu-releases