diff options
47 files changed, 1436 insertions, 0 deletions
diff --git a/etc/cron.d/mirror.cron b/etc/cron.d/mirror.cron new file mode 100644 index 0000000..63d22ac --- /dev/null +++ b/etc/cron.d/mirror.cron @@ -0,0 +1,4 @@ +# File: /etc/cron.d/mirror.cron +# Package: mirror +# Purpose: Run mirror right after work hours each day +05 17 * * * root /etc/mirror/mirror-master.sh diff --git a/etc/httpd/sites/10.1.8.63.conf b/etc/httpd/sites/10.1.8.63.conf new file mode 100644 index 0000000..7afda1b --- /dev/null +++ b/etc/httpd/sites/10.1.8.63.conf @@ -0,0 +1,5 @@ +# File: /etc/httpd/sites/{ipaddress}.conf +# Package: mirror +# Purpose: A template for the mirror apache listeners. This is probably redundant, but in case it's needed, fix the IP address here and in the filename. +Listen 10.1.8.63:80 +Listen 10.1.8.63:443 diff --git a/etc/httpd/sites/mirror.conf b/etc/httpd/sites/mirror.conf new file mode 100644 index 0000000..5f2e761 --- /dev/null +++ b/etc/httpd/sites/mirror.conf @@ -0,0 +1,65 @@ +# File: /etc/httpd/sites/mirror.conf +# Package: mirror +# Title: Apache mirror config +# Purpose: To provide a standard mirror http and https site +# Reference: +# Instructions: +# you can make a simple IP listener conf with the following +# thisip=$( ifconfig 2>/dev/null | awk '/Bcast|broadcast/{print $2}' | tr -cd '[^0-9\.\n]' | head -n1) +# printf "Listen %s\nListen %s" "${thisip}:80" "${thisip}:443" > /etc/httpd/sites/${thisip}.conf +# Be sure to fix the IP addresses here for the virtual hosts. +# Also ensure /etc/hosts has the right hostnames configured for the ssl vhost below +# Document: Below this line + +<VirtualHost 10.1.8.63:80> + + ServerName mirror.example.com:80 + ServerAlias mirror brass mirror-brass mirrorbrass brassmirror *.mirror.example.com brass.example.com + # this server also provides bgscripts.repo at http://mirror.example.com/bgscripts which is /var/www/html/mirror.example.com/bgscripts + + DocumentRoot "/var/www/html/mirror.example.com" + + Alias /mirror /mnt/mirror + + # Index options + Options +Indexes + IndexOptions IgnoreCase FancyIndexing FoldersFirst NameWidth=* DescriptionWidth=* SuppressHTMLPreamble + IndexIgnore FOOTER.html repodata tivoli *.so bgscripts + ReadmeName FOOTER.html + + # Useful additions for the mirror + AddIcon /icons/rpm.png .rpm + AddIcon /icons/repo.png .repo + AddType application/octet-stream .iso + + <Directory "/var/www/html/mirror.example.com"> + AllowOverride None + Order allow,deny + Allow from all + Options Indexes FollowSymLinks + </Directory> + + # Allows "centos.mirror.example.com" redirection to "mirror.example.com/centos" behavior + RewriteEngine On + RewriteCond %{HTTP_HOST} ^([^.]*)\.mirror\.example\.com$ + RewriteRule /(.*) http://mirror.example.com/%1/$1 [R,L] + +</VirtualHost> + +<VirtualHost 10.1.8.63:443> + SSLEngine on + SSLProtocol all -SSLv2 -SSLv3 + SSLHonorCipherOrder on + SSLCipherSuite "EECDH+ECDSA+AESGCM EECDH+aRSA+AESGCM EECDH+ECDSA+SHA384 EECDH+ECDSA+SHA256 EECDH+aRSA+SHA384 EECDH+aRSA+SHA256 EECDH+aRSA+RC4 EECDH EDH+aRSA !RC4 !aNULL !eNULL !LOW !3DES !MD5 !EXP !PSK !SRP !DSS" + SSLCertificateFile /etc/pki/tls/certs/wildcard.crt + SSLCertificateKeyFile /etc/pki/tls/private/wildcard.key + SSLCACertificateFile /etc/pki/tls/certs/rapidssl.crt + + LogLevel warn + # Reverse proxy all ssl traffic to the nonencrypted site. This is a template for any and all sites. + # Observe that this depends on /etc/hosts or DNS to look up the variable SERVER_NAME + SSLProxyEngine On + RewriteEngine On + RewriteRule ^/(.*)$ http://%{SERVER_NAME}/$1 [P,L] + +</VirtualHost> diff --git a/etc/httpd/sites/mntscripts.conf b/etc/httpd/sites/mntscripts.conf new file mode 100644 index 0000000..31cdc6b --- /dev/null +++ b/etc/httpd/sites/mntscripts.conf @@ -0,0 +1,24 @@ +<VirtualHost 10.1.8.63:80> + + ServerName mntscripts.example.com:80 + ServerAlias mntscripts scripts scripts.example.com + + DocumentRoot "/var/www/html/mntscripts.example.com" + + AddType application/octet-stream .iso + + #Alias /mirror /mnt/mirror + + Options +Indexes + IndexOptions FancyIndexing NameWidth=* DescriptionWidth=* + IndexIgnore FOOTER.html repodata + ReadmeName FOOTER.html + + <Directory "/var/www/html/mntscripts.example.com"> + AllowOverride None + Order allow,deny + Allow from all + Options Indexes FollowSymLinks + </Directory> + +</VirtualHost> diff --git a/etc/mirror/README.txt b/etc/mirror/README.txt new file mode 100644 index 0000000..e56418d --- /dev/null +++ b/etc/mirror/README.txt @@ -0,0 +1,77 @@ +File: etc/mirror/README.txt +Package: mirror 1.0-3 +Author: bgstack15@gmail.com +Startdate: 2016-06-08 +Title: Readme file for mirror +Purpose: All packages should come with a readme +Usage: Read it. +Reference: README.txt +Improve: +Document: Below this line + +### WELCOME + +mirror is basically a collection of scripts that provide the sync instructions to make a local repository of software. + +## Steps to take before using mirror for the first time +Configure these files: +/etc/httpd/sites/mirror.conf +/etc/mirror/inc/rsync +/etc/mirror/mirror.conf + +Select a storage directory. On the EXAMPLE mirror, we used an nfs mount for multiple locations. +--- BEGIN excerpt from mirror.example.com:/etc/fstab +linux-nfs2.example.com:/vol/linux_mirror /mnt/realmirror nfs defaults,uid=1539249479 0 0 +/mnt/realmirror/mirror /mnt/mirror none bind 0 0 +/mnt/mirror /var/www/html/mirror.example.com none bind 0 0 +--- END + +## Scripts to run +The files called by mirror-master are the ones that actually do the source and target selection and file operations. By default they are located in /etc/mirror/scripts/ +To enable a script, you must mark the executable flag: +chmod +x /etc/mirror/scripts/filetostart + +It is recommended to start with certain ones enabled. You can use this command: +chmod +x /etc/mirror/scripts/{centos,fedora-epel,fedora-releases,fedora-updates,libreoffice,putty,ssh,ubuntu,ubuntu-releases} + +## Configuring the web server +Included in this package are some apache configs. +For a nice intro page, a template is provided at /etc/mirror/inc/HEADER.html + +### CHANGELOG +1.0-2 2016-06-10 +uses config file /etc/mirror/mirror.conf + +1.0-3 2016-07-14 +added supplementary script /etc/mirror/inc/deploy.sh +New feature: deploy +See its config file /etc/mirror/inc/deploy.conf + +2016-10-28 mirror 1.0-4 +Adding elemants for safe publishing on github +Added repos for linuxmint + +### USAGE BLOCK +usage: mirror-master.sh [-duV] [ -f | --file /etc/mirror/mirror.conf ] [ --scriptsdir /etc/mirror/scripts ] [ scriptname ] +version ${mirrormasterversion} + -d debug Show debugging info, including parsed variables. + -u usage Show this usage block. + -V version Show script version number. + -f file Use specified config file. Default is /etc/mirror/mirror.conf + --scriptsdir Use specified scripts directory. Will override anything in the called conf file +Return values: +0 Normal +1 Help or version info displayed +2 mirror-master is already running +3 Unable to modify important file +4 Unable to find dependency +5 Not run as root or sudo +Examples: + mirror-master.sh centos + This command will run only the scripts/centos file. + mirror-master.sh centos putty + This command will run only centos and putty files. + mirror-master.sh + This command will run all files with o+x perms in scriptsdir directory. + +### REFERENCE diff --git a/etc/mirror/docs/debian/conffiles b/etc/mirror/docs/debian/conffiles new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/conffiles diff --git a/etc/mirror/docs/debian/control b/etc/mirror/docs/debian/control new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/control diff --git a/etc/mirror/docs/debian/postinst b/etc/mirror/docs/debian/postinst new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/postinst diff --git a/etc/mirror/docs/debian/postrm b/etc/mirror/docs/debian/postrm new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/postrm diff --git a/etc/mirror/docs/debian/preinst b/etc/mirror/docs/debian/preinst new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/preinst diff --git a/etc/mirror/docs/debian/prerm b/etc/mirror/docs/debian/prerm new file mode 100755 index 0000000..e69de29 --- /dev/null +++ b/etc/mirror/docs/debian/prerm diff --git a/etc/mirror/docs/mirror.spec b/etc/mirror/docs/mirror.spec new file mode 100644 index 0000000..abcd38c --- /dev/null +++ b/etc/mirror/docs/mirror.spec @@ -0,0 +1,87 @@ +# +# spec file for Mirror compiled by hand +Summary: Mirror +Name: mirror +Version: 1.0 +Release: 4 +License: CC BY-SA 4.0 +Group: Applications/System +Source: mirror.tgz +URL: bgstack15@gmail.com +#Distribution: +#Vendor: +Packager: Bgstack15 <bgstack15@gmail.com> +Buildarch: noarch +PreReq: bgscripts >= 1.1-17 +Requires: httpd >= 2.2 + +%description +Mirror is the rpm package for the official mirror.example.com installation. + +%prep +%setup + +%build + +%install +rsync -a . %{buildroot}/ + +%clean +rm -rf ${buildroot} + +%pre + +%post + +%preun + +%files +%config /etc/sudoers.d/50_mirror-sudo +%config /etc/cron.d/mirror.cron +/etc/mirror/mirror-master.sh +%doc %attr(444, -, -) /etc/mirror/README.txt +/etc/mirror/scripts/ssh +/etc/mirror/scripts/libreoffice +/etc/mirror/scripts/linuxmint-repos +/etc/mirror/scripts/ubuntu +/etc/mirror/scripts/putty +/etc/mirror/scripts/rsyslog +/etc/mirror/scripts/centos +/etc/mirror/scripts/fedora-epel +/etc/mirror/scripts/ubuntu-releases +/etc/mirror/scripts/linuxmint-isos +/etc/mirror/scripts/fedora-releases +/etc/mirror/scripts/fedora-updates +/etc/mirror/docs/debian/postinst +/etc/mirror/docs/debian/conffiles +/etc/mirror/docs/debian/prerm +/etc/mirror/docs/debian/postrm +/etc/mirror/docs/debian/control +/etc/mirror/docs/debian/preinst +/etc/mirror/docs/mirror.spec +/etc/mirror/inc/scrub.py +/etc/mirror/inc/scrub.pyc +/etc/mirror/inc/scrub.pyo +%config /etc/mirror/inc/deploy.conf +/etc/mirror/inc/localize_git.sh +/etc/mirror/inc/deploy.sh +%doc %attr(444, -, -) /etc/mirror/inc/scrub.txt +/etc/mirror/inc/rsync +%doc %attr(444, -, -) /etc/mirror/packaging.txt +/etc/mirror/examples/HEADER.html +/etc/mirror/examples/example-debian/example-debian.list +/etc/mirror/examples/example-debian/update-example-debian.sh +/etc/mirror/examples/example-debian/FOOTER.html +/etc/mirror/examples/example-debian/HEADER.html +/etc/mirror/examples/example-debian/example-debian.gpg +/etc/mirror/examples/example-rpm/FOOTER.html +/etc/mirror/examples/example-rpm/HEADER.html +/etc/mirror/examples/example-rpm/example-rpm.repo +/etc/mirror/examples/example-rpm/update-example-rpm.sh +%config /etc/mirror/mirror.conf +%config /etc/httpd/sites/10.1.8.63.conf +%config /etc/httpd/sites/mntscripts.conf +%config /etc/httpd/sites/mirror.conf +%verify(link) /usr/local/bin/deploy +/usr/share/httpd/icons/repo.png +/usr/share/httpd/icons/rpm.png diff --git a/etc/mirror/examples/HEADER.html b/etc/mirror/examples/HEADER.html new file mode 100644 index 0000000..d44fa8c --- /dev/null +++ b/etc/mirror/examples/HEADER.html @@ -0,0 +1,45 @@ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" + "http://www.w3.org/TR/REC-html40/loose.dtd"> +<html> + +<head><title>Welcome to mirror.example.com</title></head> + +<body> +<center><h1>Welcome to mirror.example.com</h1> +<!--<h6>migrated to current host 2015-12</h6>--> +</center> +<hr> + +This is IT's <a href="http://www.opensource.org/">Open Source</a> software +mirror. We have gobs of software here to save you time downloading it from the +internet. If you need ftp access, try <a +href="ftp://mirror.example.com">ftp://mirror.example.com</a>. + +<p> + +If there is a project you would like to see mirrored, let us know at the e-mail +address at the bottom and we'll see what we can do. We can't guarantee your +favorite project will be mirrored, but if you have a reasonable request, we +will most likely be able to comply. + +<p> + +<strong>WARNING!</strong> There is no guarantee that this box will always be +up or that the software contained on it will be up-to-date, bug-free, safe, +uncorrupted or complete. Use this box and the software contained on it +<em>at your own risk</em>. +Due to the nature of maintaining this box, IT cannot take any responsibility +for problems or damage incurred by the software from this box. Please +<em>DO NOT </em> contact the Helpdesk with problems or questions concerning +this box. If you have questions, comments, or problem reports, contact <a href="mailto:mirror@example.com">mirror Administrator</a>. However, as this is an "unofficial" +IT project, any response may be highly delayed. +<p> +That being said, have fun! +E-Mail the administrator at: <a href="mailto:mirror@example.com">mirror@example.com</a>. +<hr> +<table style="width:50%"> +<tr><td> +Force http: <a href="http://mirror.example.com">http://mirror.example.com</a></td><td> +Force https: <a href="https://mirror.example.com">https://mirror.example.com</a></td></tr></table> +</body> +</html> diff --git a/etc/mirror/examples/example-debian/FOOTER.html b/etc/mirror/examples/example-debian/FOOTER.html new file mode 100644 index 0000000..7879e1c --- /dev/null +++ b/etc/mirror/examples/example-debian/FOOTER.html @@ -0,0 +1,4 @@ +<html> +<body> +</body> +</html> diff --git a/etc/mirror/examples/example-debian/HEADER.html b/etc/mirror/examples/example-debian/HEADER.html new file mode 100644 index 0000000..6ffade1 --- /dev/null +++ b/etc/mirror/examples/example-debian/HEADER.html @@ -0,0 +1,13 @@ +<html> +<title>mirror.example.com/ubuntu/example-debian repository</title> +<body> +<h1>mirror.example.com/bgscripts</h1> +<hr> +<p>You can install the example-debian repo in the /etc/apt/sources.list.d/ directory with these command: +<pre> +sudo wget --quiet http://mirror.example.com/ubuntu/example-debian/example-debian.gpg -O /root/example-debian.gpg +sudo apt-key add /root/example-debian.gpg +sudo wget --quiet http://mirror.example.com/ubuntu/example-debian/example-debian.list -O /etc/apt/sources.list.d/example-debian.list +</pre> +</body> +</html> diff --git a/etc/mirror/examples/example-debian/example-debian.gpg b/etc/mirror/examples/example-debian/example-debian.gpg new file mode 100644 index 0000000..d64a765 --- /dev/null +++ b/etc/mirror/examples/example-debian/example-debian.gpg @@ -0,0 +1,51 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2.0.22 (GNU/Linux) + +mQINBFdPNSABEACd9U4/KOjZjXj1S7ehqwpSNyM/bvhNaGjSq/jEj6WaxcgwC0Ot +gN7PVIJRnIcgUGf2MiK79XeX4ZJFZQ9KdYeN38hKEaaMLGW+U3KN84fnt/6DSExy +HFl4UBshtVbg9yPU54grkvvjx1bnAWGTHzyJILzIVC7bbvJkbddtT7cLy3zNNP7X +RAPXsKx8P2AfC0lGoyo5TSqY3807EPuTrCfW0ze++3p8NfHesGR57wA5EgPNimZd +gSeb9VN8B/FR7bSEYueL2jOupbi28ewQaN2PzZWffhuohoQrlLwQznKodmpKys9a +DK1bvaS7MsTJw1pRLLYnlRV05+SQ1XH8QYYRmH/Y+a5u/gJhXR5KhZmfUfhnhpwL +61bA7tnDdn2/y8dAPcppvuxuCFlh8VqL3Fyf8tQPaCRbITHMpwuahsnTyGKIDxz6 +01tfgC0HEfCwMcfxqLvEM2komVCp8LFkzdwoLl4pCKFmhZNq04FzyImKJceihBmp +C/mnxzDKPN1OhSIN2/DnnpD77r6vS7AWOBOcmMLHsvUDpKVlJQSVgNGjuQNDTfJE +uL4wGoqZSyOYTtUzu3L7lvNvvE4huTtBRwtYQkrtzlxhWmSfrkd2PKAZbjZDn2bB +plLkRYv0yeJnlUzVmPYvm1dOdirkVhdd4g3KaiDGCxXMiBXhIH5pQM5MkwARAQAB +tCBMaW51eCBBZG1pbiA8bGludXhhZG1pbkBianUuZWR1PokCOQQTAQgAIwUCV081 +IAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheAAAoJEDhjKVZEOuZkFmAP/jo6 +vaMSXXUAUn5aZ0wa5aKsTFoYJ1NwBDVtD5CNFe0MTIEuLgUtF66Z7xvC3He/3ah2 +eJ5CAerCxWVjJx0aHq/XTvKioxZzJkOhwkuLGy7/cBrknYQbpAHD2lKPOiOs3pec +u+sQC/v46cqOKUJ7K6PEVuyl1df7C6KkbR8OEVPQXiKsD/2o6eXRx+1W8dsWDZbv +rYtY01m2lDo4vU3JQ+wiw48Mj8b+0zSRzpTkwmdU7MvaGMQqnDwQZDNz/GaEirql +Qc28r2fl3mGeQE35PuFyBiDDAuFbuSRQhg+K4y6PMATBl53XESYdLgWWSg/0bDDh +otf3CWOX1MlRZlIUqXM7Ghg43bbZYcgDfMqIyBMIp0idk6r9soKW+rdFrbkBmxMd +5oSXgz0O8i49AxkvcnBYI0jHx8BJPDOaZLa4yi3bWxkUs0An5g7l0+DnqBQOXoVV +/0149bzNznmA/JBy5/hmsfKa+YaKUqKS1g0Y1h/nFe61RRbXP7w1M+BuiUKX5seu +S7JOaQ92figjFN3hO8d+p73SGf9ge+e/lxA39Eifhtq5Kc+7kQULTdTIK9U9LT2I +YAQlrCnHlwitDq3pFTMqffGwTSodbJowTTeaVnIhG6t5eD3YuMmqsaDorPv1LiOs +p4oE11qtIkEu1nwV3MaWedCFFBuUVZCIMJTOY04IuQINBFdPNSABEACzSMmBAv0e +PCG0yx3EUUCsaglFiVpdx+KNC2gLFTdnKQ31GINhmfmtuZB/yz770TWnkauu26dn +uSVNz/bJVSb81zKvwy/dQabMA31SEl1l91vnYe2ukJmOpApLlwwGQYn//9dklkmR +5o3l2oblqghtb0lNGoGkkRmLOAfLRaUtnP2DObg2KrmrfMbNOsZtLy1dw8B0kp34 +a3ne7VB6flzmV4wr9vINsp1fY+ikf0N9idiibDM+QT1X1wudthXlOx4VpowRWd6T +UvYgNU2W606gq++JgQ7ha5xSVop7FodMlfKf5Y/TNzBdTpR7/7Yloo8RxrakUfPU +Nz2hZEPFQ+9SmelWKO48XrqpEIZnv4fmiP+GrCSQcsmnxP7cAsQOxbNf6hH5Jj2s +Q9aaMEf+XyAY9E8G1V7S77mA5/WsiW+W+020XaKHORe6xNuJrWUNqzXVHr+cZ3F5 +tzQ+dlFef9ItbS5XcMZjEGoBeFZbomikKPv4yfRDt+GIIGjHaB0wMoM5XOCxEP6H +mrbkNv1eh4Gs6YnUa6cT4sT+WkDJAo6LvNbsBi/s2RJxEwzPvuCpqAZHY+SgDsNr +UItQ62B3TlF/P0o/IoUb4zGYKxhmI5Df/suKx0z89FYT8Sxr3MuP1NoTqLNGTAAj +mOfPACEzBtGy/OB/1xuIMufnTXh4xi0MpwARAQABiQIfBBgBCAAJBQJXTzUgAhsM +AAoJEDhjKVZEOuZkZbEP/0tffWiR1QcJNDn9RBYbXmHYwprDzOELSvju5HghJ4Zc +EUJ7HXBbcbztFBsgykX0GMTiz9QKeX5HY689wlMCs/iJGU2/o+8dhqKyiNWBocSX +mDCU/76O5BMCD3GnlUSlsj4r1Wb4qmER8LpQy7JX+WHMqrcV/kUQftSwPWhnPgq7 +Slb8zuwK981PqfBJ0TAtyceqO6Ap82aKr9/JSr3vbadU5R4Qg0paBZwKpsnxsx+K +q0MP1POWYXXSPShMCk5WFh/Zln5+1uG7JC9Z9biX2FfGPXz+ArzqFtMKCzjf983c +SFCOJccGpza1yw3YFuFr8xo8e+auaGCdFQ8f/NuArtzxV0xqyPyBVCY61QvJpAWH +Z92xyyhHUGqows8dbMfG9TLSbZEMnytnZ963a3E7k/p9K4tCQ9Uj1mOFzUNyEqNO +075NJOS58N+QLrrw0/36ylqmZ+IfPipkF7ATMcIxj75GghjjstM9Rcy8Nsa/NB8l +DJ0Q0kmAgQwvupy3tbT4STl58bRowvt/BPCWyb1YZItkNfYRpYGFPp/gLxDijYLT +L6YXT3oEVqeItDlLuCns57kR7cp/Y1t83zJ8DKGck/TaUbwJBKJcYkGFoO9x+ZjF +keMeYGeTVuBsZabJJhnmH7E5c8+j328et4+/1J13xOAfixXef7Gzk4pT1m3ii7am +=ggDE +-----END PGP PUBLIC KEY BLOCK----- diff --git a/etc/mirror/examples/example-debian/example-debian.list b/etc/mirror/examples/example-debian/example-debian.list new file mode 100644 index 0000000..ad21fbb --- /dev/null +++ b/etc/mirror/examples/example-debian/example-debian.list @@ -0,0 +1 @@ +deb http://mirror.example.com/ubuntu/example-debian/ / diff --git a/etc/mirror/examples/example-debian/update-example-debian.sh b/etc/mirror/examples/example-debian/update-example-debian.sh new file mode 100755 index 0000000..0a6e7a0 --- /dev/null +++ b/etc/mirror/examples/example-debian/update-example-debian.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +# working directory +repodir=/mnt/mirror/ubuntu/example-debian/ +cd ${repodir} +chmod 0644 *deb 1>/dev/null 2>&1 + +# create the package index +dpkg-scanpackages -m . > Packages +cat Packages | gzip -9c > Packages.gz + +# create the Release file +PKGS=$(wc -c Packages) +PKGS_GZ=$(wc -c Packages.gz) +cat <<EOF > Release +Architectures: all +Date: $(date -R) +MD5Sum: + $(md5sum Packages | cut -d" " -f1) $PKGS + $(md5sum Packages.gz | cut -d" " -f1) $PKGS_GZ +SHA1: + $(sha1sum Packages | cut -d" " -f1) $PKGS + $(sha1sum Packages.gz | cut -d" " -f1) $PKGS_GZ +SHA256: + $(sha256sum Packages | cut -d" " -f1) $PKGS + $(sha256sum Packages.gz | cut -d" " -f1) $PKGS_GZ +EOF +gpg --batch --yes --passphrase-file /root/.gnupg/linuxadmin -abs -o Release.gpg Release diff --git a/etc/mirror/examples/example-rpm/FOOTER.html b/etc/mirror/examples/example-rpm/FOOTER.html new file mode 100644 index 0000000..57b6775 --- /dev/null +++ b/etc/mirror/examples/example-rpm/FOOTER.html @@ -0,0 +1,8 @@ +<html> +<head> +<title>mirror.example.com/example-rpm repository</title> +</head> +<body> +<p>Most contents are licensed under CC BY-SA 4.0.</p> +</body> +</html> diff --git a/etc/mirror/examples/example-rpm/HEADER.html b/etc/mirror/examples/example-rpm/HEADER.html new file mode 100644 index 0000000..105f0d8 --- /dev/null +++ b/etc/mirror/examples/example-rpm/HEADER.html @@ -0,0 +1,9 @@ +<html> +<body> +<h1>mirror.example.com/example-rpm</h1> +<hr> +<p>You can install the example-rpm.repo in the /etc/yum.repos.d/ directory with this command: +<pre>wget http://mirror.example.com/example-rpm/example-rpm.repo -O /etc/yum.repos.d/example-rpm.repo +yum -y install bgscripts keepalive</pre> +</body> +</html> diff --git a/etc/mirror/examples/example-rpm/example-rpm.repo b/etc/mirror/examples/example-rpm/example-rpm.repo new file mode 100644 index 0000000..12bca22 --- /dev/null +++ b/etc/mirror/examples/example-rpm/example-rpm.repo @@ -0,0 +1,8 @@ +# example-rpm.repo +# Install with: +# wget http://mirror.example.com/bgscripts/example-rpm.repo -O /etc/yum.repos.d/example-rpm.repo +[example-rpm] +name=example-rpm +baseurl=http://mirror.example.com/bgscripts/ +gpgcheck=0 +#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 diff --git a/etc/mirror/examples/example-rpm/update-example-rpm.sh b/etc/mirror/examples/example-rpm/update-example-rpm.sh new file mode 100755 index 0000000..b3003af --- /dev/null +++ b/etc/mirror/examples/example-rpm/update-example-rpm.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# working directory +repodir=/mnt/mirror/bgscripts/ +cd ${repodir} +chmod 0644 *rpm 1>/dev/null 2>&1 + +# create the package index +createrepo . diff --git a/etc/mirror/inc/deploy.conf b/etc/mirror/inc/deploy.conf new file mode 100644 index 0000000..4993ba1 --- /dev/null +++ b/etc/mirror/inc/deploy.conf @@ -0,0 +1,53 @@ +# +# File: /etc/mirror/inc/deploy.conf +# Author: bgstack15 +# Startdate: 2016-07-14 +# Title: Config file for deploy.sh +# Purpose: +# Package: mirror +# History: +# Usage: +# Reference: deploy.sh +# Improve: +# planned CLI arguments possible: +# bgscripts +# bgscripts-1.1 +# bgscripts-1.1-9 +# bgscripts 1.1 +# bgscripts 1.1-9 +# implemented CLI args possible: +# bgscripts 1.1-9 + +[input] +type=input +location=/mnt/scripts/software +packagedir=${inputlocation}/${packagename} + +[bgscripts] +# type may either be input or destination. Input is used only in the [input] zone. Everything else is a destination. +type=destination + +# flavor is what type of package to send. Options include centos, redhat, ubuntu, debian, tar, tarball +# default flavor=tarball +flavor=centos + +# base location. Required to function. +location=/mnt/mirror/bgscripts + +# packagedir is the first place the script checks before placing a destination file. The location is a backup location. The script will evaluate packagedir, so you can use variables here. +# default packagedir=<undefined> +# packagedir=/mnt/mirror/bgscripts/${packagename} + +# If link is yes, it will make a symlink to the master tarball +# default link=no +link=yes + +# If updatescript is executable, if the zone is updated this command will be called +# default updatescript=<undefined> +updatescript=/mnt/mirror/bgscripts/update-bgscripts.sh + +[exampledebian] +type=destination +flavor=debian +location=/mnt/mirror/ubuntu/example-debian +updatescript=/mnt/mirror/ubuntu/example-debian/update-example-debian.sh diff --git a/etc/mirror/inc/deploy.sh b/etc/mirror/inc/deploy.sh new file mode 100755 index 0000000..ac1cb32 --- /dev/null +++ b/etc/mirror/inc/deploy.sh @@ -0,0 +1,310 @@ +#!/bin/sh +# File: /etc/mirror/inc/deploy.sh +# Author: bgstack15@gmail.com +# Startdate: 2016-07-14 09:53:09 +# Title: Script that Deploys a Package +# Purpose: To make it easy to deploy to the mirror a new version of a package +# Package: mirror-1.0-3 +# History: +# Usage: +# Reference: ftemplate.sh 2016-07-12a; framework.sh 2016-05-25a +# mirror-master from mirror-1.0-2.noarch.rpm +# This script has a symlink: /usr/local/bin/deploy +# Improve: +# * provide better package name and version parsing +fiversion="2016-05-25a" +deployversion="2016-07-14a" + +usage() { + less -F >&2 <<ENDUSAGE +usage: deploy.sh [-duV] [-c conffile] packagename packageversion +version ${deployversion} + -d debug Show debugging info, including parsed variables. + -u usage Show this usage block. + -V version Show script version number. + -c conffile Overrides default conffile value. Default is ${conffile}. +Given a packagename and packageversion, this script will deploy the correct architecture type of package file to the specified locations. +If debug level is 3 or less, the copy will actually be performed. +See the conffile ${conffile} for examples. +Return values: +0 Normal +1 Help or version info displayed +2 Count or type of flaglessvals is incorrect +3 Incorrect OS type +4 Unable to find dependency +5 Not run as root or sudo +6 Config is invalid +Debug levels: +2 Perform file actions +3 Perform file actions and display file copy +4 Display only +5 More debug info +ENDUSAGE +} + +function fexit { + errornum="${1}"; shift + ferror "$@" + exit "${errornum}" +} + +function fileaction { + thisaction="${1}" + thisleftfile="${2}" + thisrightfile="${3}" + case "${thisaction}" in + copy) thisfacommand="/bin/cp -p";; + symlink) thisfacommand="/bin/ln -sf";; + *) thisfacommand=echo;; + esac + debuglev 2 && { ferror ${thisfacommand} "${thisleftfile}" "${thisrightfile}"; } + ! debuglev 4 && { ${thisfacommand} "${thisleftfile}" "${thisrightfile}"; } 2>&1 | grep -viE "failed to preserve ownership for" +} + +# DEFINE FUNCTIONS + +# DEFINE TRAPS + +function clean_deploy { + #rm -f $logfile >/dev/null 2>&1 + [ ] #use at end of entire script if you need to clean up tmpfiles +} + +function CTRLC { + #trap "CTRLC" 2 + [ ] #useful for controlling the ctrl+c keystroke +} + +function CTRLZ { + #trap "CTRLZ" 18 + [ ] #useful for controlling the ctrl+z keystroke +} + +function parseFlag { + flag=$1 + hasval=0 + case $flag in + # INSERT FLAGS HERE + "d" | "debug" | "DEBUG" | "dd" ) setdebug; ferror "debug level ${debug}";; + "u" | "usage" | "help" | "h" ) usage; exit 1;; + "V" | "fcheck" | "version" ) ferror "${scriptfile} version ${deployversion}"; exit 1;; + #"i" | "infile" | "inputfile" ) getval;infile1=$tempval;; + "c" | "conffile" ) getval;conffile=$tempval;; + esac + + debuglev 10 && { [[ hasval -eq 1 ]] && ferror "flag: $flag = $tempval" || ferror "flag: $flag"; } +} + +# DETERMINE LOCATION OF FRAMEWORK +while read flocation; do if [[ -x $flocation ]] && [[ $( $flocation --fcheck ) -ge 20160229 ]]; then frameworkscript=$flocation; break; fi; done <<EOFLOCATIONS +./framework.sh +${scriptdir}/framework.sh +~/bin/bgscripts/framework.sh +~/bin/framework.sh +~/bgscripts/framework.sh +~/framework.sh +/usr/local/bin/bgscripts/framework.sh +/usr/local/bin/framework.sh +/usr/bin/bgscripts/framework.sh +/usr/bin/framework.sh +/usr/bgscripts/framework.sh +/usr/framework.sh +/bin/bgscripts/framework.sh +/bin/framework.sh +/mnt/scripts/bgscripts/framework.sh +EOFLOCATIONS +[[ -z "$frameworkscript" ]] && echo "$0: framework not found. Aborted." 1>&2 && exit 4 + +# REACT TO OPERATING SYSTEM TYPE +case $( uname -s ) in + AIX) [ ];; + Linux) [ ];; + *) echo "$scriptfile: 3. Indeterminate OS: $( uname -s )" 1>&2 && exit 3;; +esac + +# INITIALIZE VARIABLES +# variables set in framework: +# today server thistty scriptdir scriptfile scripttrim +# is_cronjob stdin_piped stdout_piped stderr_piped sendsh sendopts +. ${frameworkscript} || echo "$0: framework did not run properly. Continuing..." 1>&2 +conffile=/etc/mirror/inc/deploy.conf +logfile=${scriptdir}/${scripttrim}.${today}.out +interestedparties="bgstack15@example.com" + +# REACT TO ROOT STATUS +case $is_root in + 1) # proper root + [ ] ;; + sudo|"") # sudo to root or not root at all + ferror "${scriptfile}: 5. Please run as root. Aborted." + exit 5 + ;; +esac + +# SET CUSTOM SCRIPT AND VALUES +#setval 1 sendsh sendopts<<EOFSENDSH # if $1="1" then setvalout="critical-fail" on failure +#/usr/local/bin/bgscripts/send.sh -hs # setvalout maybe be "fail" otherwise +#/usr/local/bin/send.sh -hs # on success, setvalout="valid-sendsh" +#/usr/bin/mail -s +#EOFSENDSH +#[[ "$setvalout" = "critical-fail" ]] && ferror "${scriptfile}: 4. mailer not found. Aborted." && exit 4 + +# VALIDATE PARAMETERS +# objects before the dash are options, which get filled with the optvals +# to debug flags, use option DEBUG. Variables set in framework: fallopts +validateparams packagename packageversion - "$@" + +# CONFIRM TOTAL NUMBER OF FLAGLESSVALS IS CORRECT +if [[ $thiscount -lt 2 ]]; +then + ferror "${scriptfile}: 2. Invalid packagename and version. Aborted." + exit 2 +fi + +# CONFIGURE VARIABLES AFTER PARAMETERS + +## READ CONFIG FILE TEMPLATE +zonecount=0 +[[ 1 -eq 1 ]] && { # REMOVE THIS LINE TO USE THE CONFIG FILE PARSER TEMPLATE +[[ ! -f "${conffile}" ]] && ferror "${scriptfile}: 4. Conffile not found: ${conffile}. Aborted." && exit 4 +oIFS="${IFS}"; IFS=$'\n' +infiledata=( $( sed ':loop;/^\/\*/{s/.//;:ccom;s,^.[^*]*,,;/^$/n;/^\*\//{s/..//;bloop;};bccom;}' "${conffile}") ) #the crazy sed removes c style multiline comments +IFS="${oIFS}" +for line in "${infiledata[@]}"; +do line=$( echo "${line}" | sed -e 's/^\s*//;s/\s*$//;/^[#$]/d;s/\s*[^\]#.*$//;' ); [[ -n "${line}" ]] && { + # the crazy sed removes leading and trailing whitespace, blank lines, and comments + debuglev 8 && ferror "line=\"$line\"" + if echo "${line}" | grep -qiE "\[.*\]"; + then + # new zone + zone=$( echo "${line}" | tr -d '[]' ) + (( zonecount += 1 )) + zones[${zonecount}]="${zone}" + debuglev 7 && ferror "zone=${zone}" + else + # directive + varname=$( echo "${line}" | awk -F= '{print $1}' ) + varval=$( echo "${line}" | awk -F= '{$1=""; printf "%s", $0}' | sed 's/^ //;' ) + #debuglev 7 && ferror "${zone}$( eval echo ${varname}=\\\"${varval}\\\" )" #evaluates the variables in the varval + debuglev 7 && ferror "${zone}${varname}=\"${varval}\"" + # simple define variable + eval "${zone}${varname}=\${varval}" + fi + #read -p "Please type something here:" response < $thistty + #echo "$response" +}; done +} # AND THIS LINE +#echo "zonecount=${zonecount}" +#for word in ${zones[@]}; do echo ${word}; done + +## CONFIRM the input zone exists +[[ -z "${inputtype}" ]] || [[ ! "${inputtype}" = "input" ]] && fexit 6 "Invalid inputtype: \"${inputtype}\". Must be \"input\" and in zone named \"input\". Aborted." +[[ -z "${inputlocation}" ]] || [[ ! -d "${inputlocation}" ]] && fexit 6 "Invalid inputlocation: \"${inputlocation}\". Confirm directory exists. Aborted." +[[ -n "${inputpackagedir}" ]] && eval inputpackagedir="${inputpackagedir}" + +## REACT TO BEING A CRONJOB +#if [[ $is_cronjob -eq 1 ]]; +#then +# [ ] +#else +# [ ] +#fi + +# SET TRAPS +#trap "CTRLC" 2 +#trap "CTRLZ" 18 +#trap "clean_deploy" 0 + +# MAIN LOOP +#{ + for thiszone in ${zones[@]}; + do + [[ ! "${thiszone}" = "input" ]] && { + debuglev 5 && ferror "Running zone ${thiszone}" + eval thislocation=\${${thiszone}location} + if [[ -z "${thislocation}" ]] || [[ ! -d "${thislocation}" ]]; then continue; fi + + # so the location exists + thiszoneused=0 + + # DERIVE PACKAGE SOURCE FILE + eval thisflavor=\${${thiszone}flavor} + eval thispackagedir=\${${thiszone}packagedir} + [[ -n "${thispackagedir}" ]] && eval thispackagedir="${thispackagedir}" + case "${thisflavor}" in + redhat|centos) # needs special attention to get architecture + sourcefile=$( { find "${inputpackagedir}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.rpm"; find "${inputlocation}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.rpm";} 2>/dev/null | head -n1 ) + ;; + debian|ubuntu) + sourcefile=$( { find "${inputpackagedir}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.deb"; find "${inputlocation}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.deb";} 2>/dev/null | head -n1 ) + ;; + *) # including tarball, tar + sourcefile=$( { find "${inputpackagedir}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.master.tgz"; find "${inputlocation}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.master.tgz"; } 2>/dev/null | head -n1 ) + ;; + esac + + # DERIVE TARBALL FILE + sourcetarfile=$( { find "${inputpackagedir}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.master.tgz"; find "${inputlocation}" -regex ".*${packagename}-${packageversion}.*" -regex ".*.master.tgz"; } 2>/dev/null | head -n1 ) + debuglev 5 && ferror "sourcefile=${sourcefile}" + debuglev 5 && ferror "sourcetarfile=${sourcetarfile}" + + # CALCULATE DESTINATION FILE + destinationdir=$( { find "${thispackagedir}" -maxdepth 0 -type d; find "${thislocation}" -maxdepth 0 -type d; } 2>/dev/null | grep -viE "^$" | head -n1 ) + #debuglev 5 && ferror "destinationdir=${destinationdir}" + [[ ! -d "${destinationdir}" ]] && ferror "Zone ${thiszone} file ${sourcefile} cannot be copied to the invalid directory \"${destinationdir}\". Skipped." && continue + destinationfile=$( echo "${destinationdir}/$( basename "${sourcefile}" )" | sed -e 's!\/\+!\/!g;' ) + debuglev 5 && ferror "destinationfile=${destinationfile}" + + # PERFORM FILE COPY + if [[ ! -f "${sourcefile}" ]]; + then + ferror "Sourcefile ${sourcefile} not found. Skipped the copy action." + else + fileaction copy "${sourcefile}" "${destinationfile}" + thiszoneused=1 + fi + + # IF ZONELINK + eval thislink=\${${thiszone}link} + case "${thislink}" in + "1"|"y"|"yes"|"Y"|"YES") + # have already derived tarball file + if [[ ! -f "${sourcetarfile}" ]]; + then + # link was yes, but tarball does not exist, so soft error. + ferror "Sourcetarfile ${sourcetarfile} not found. Skipped the tar file symlink." + else + # CALCULATE DESTINATION TARBALL FILE + destinationtarfile="$( echo "${destinationdir}/$( basename "${sourcetarfile}" )" | sed -e 's!\/\+!\/!g;' )" + + # PERFORM TARBALL SYMLINK + fileaction symlink "${sourcetarfile}" "${destinationtarfile}" + fi + ;; + *) # no + [ ] + ;; + esac + + # IF ZONE WAS UPDATED + eval thisupdatescript=\${${thiszone}updatescript} + if [[ thiszoneused -ne 0 ]]; + then + [[ -n "${thisupdatescript}" ]] && { + if [[ ! -x "${thisupdatescript}" ]]; + then + ferror "Cannot execute the updatescript ${thisupdatescript}. Skipped." + else + # is executable + debuglev 2 && ferror "Execute ${thisupdatescript}" + ! debuglev 4 && ${thisupdatescript} + fi + } + fi + + } # end if-not-input-zone + done +#} | tee -a $logfile + +# EMAIL LOGFILE +#$sendsh $sendopts "$server $scriptfile out" $logfile $interestedparties diff --git a/etc/mirror/inc/localize_git.sh b/etc/mirror/inc/localize_git.sh new file mode 100755 index 0000000..7423276 --- /dev/null +++ b/etc/mirror/inc/localize_git.sh @@ -0,0 +1,2 @@ +#!/bin/sh +\cp -pRf /home/work/mirror/.git /home/bgstack15/rpmbuild/SOURCES/mirror-1.0-4/ diff --git a/etc/mirror/inc/rsync b/etc/mirror/inc/rsync new file mode 100644 index 0000000..710ecb7 --- /dev/null +++ b/etc/mirror/inc/rsync @@ -0,0 +1,38 @@ +#!/bin/sh +# +# rsync: Specific functions and options for rsync use +# Original source may be from David Lowry but is uncertain. + +#RSYNCOPTIONS="--delete-during -l -p -r -t -v" +RSYNCOPTIONS="-rlptv --del" +MIRRORDIR='/mnt/mirror' + +function rmirror { + if [ -z "$2" ] ; then + echo "Usage: rmirror SOURCE DEST [EXCLUDES...]" + echo "Example: rmirror localhost:pub/project project stuff morestuff" + exit 1 + fi + + SOURCE=$1; shift + DEST=$1; shift + EXCLUDES= + + if [ ! -d "$MIRRORDIR/$DEST" ] ; then + mkdir "$MIRRORDIR/$DEST" + fi + + for OPT in $*; do + EXCLUDES="$EXCLUDES --exclude=$OPT" + done + + #date + echo rsync $RSYNCOPTIONS "$SOURCE/" "$MIRRORDIR/$DEST" $EXCLUDES + rsync $RSYNCOPTIONS "$SOURCE/" "$MIRRORDIR/$DEST" $EXCLUDES || exit 1 + #date +} + +function rmirrorn { + RSYNCOPTIONS="-n $RSYNCOPTIONS" + rmirror $* +} diff --git a/etc/mirror/inc/scrub.py b/etc/mirror/inc/scrub.py new file mode 100755 index 0000000..52652fd --- /dev/null +++ b/etc/mirror/inc/scrub.py @@ -0,0 +1,149 @@ +#!/bin/env python3 +# Filename: scrub.py +# Location: Various +# Author: bgstack15@gmail.com +# Startdate: 2016-09-28 +# Title: Script that Simultaneously Copies and Scrubs a Directory +# Purpose: Prepare projects for publication by removing private information like usernames and hostnames +# Package: Various +# History: +# 2016-10-03 working on batch rename files +# 2016-10-20 added not ".tgz" in source.name +# 2016-10-27 Fixed error when trying chmod on a symlink +# 2016-10-31 Handle symlinks by duplicating the link and stopping there. +# Also separated directory, filename renaming tasks +# Usage: +# Store this file with any package that gets published. Adjust scrub.txt in local directory. +# # First line: source directory Second line: target directory. WILL BE OVERWRITTEN! +# /etc/ansible +# /home/bjones/ansible.clean +# # Rest of the lines are "OLD WORD" "NEW WORD" +# bjones bgstack15 +# rsmith rmstack15 +# Reference: +# http://stackoverflow.com/questions/79968/split-a-string-by-spaces-preserving-quoted-substrings-in-python/524796#524796 +# http://stackoverflow.com/questions/6706953/python-using-subprocess-to-call-sed#6707003 +# http://stackoverflow.com/questions/6584871/remove-last-character-if-its-a-backslash/6584893#6584893 +# http://stackoverflow.com/questions/2212643/python-recursive-folder-read/2212728#2212728 +# parallel lists: http://stackoverflow.com/questions/1663807/how-can-i-iterate-through-two-lists-in-parallel-in-python +# file renames http://stackoverflow.com/questions/225735/batch-renaming-of-files-in-a-directory/7917798#7917798 +# Improve: +# Add option to specify scrub file +# Add exclude option to scrub file, such as .git and so on +# Accept CLI options like source, destination, even exclusions? +# Add flag for performing file renames as well, or file renames only +import re, shlex, os, sys, shutil +from pathlib import Path + +# scrubpy version +scrubpyversion = "2016-10-31a" + +# Define functions + +def removeComments(string): + #string = re.sub(re.compile("/\*.*?\*/",re.DOTALL ) ,"", string) + #string = re.sub(re.compile("//.*?\n" ) ,"" ,string) + pattern = r"(\".*?\"|\'.*?\')|(/\*.*?\*/|(//|#)[^\r\n]*$)" + regex = re.compile(pattern, re.MULTILINE|re.DOTALL) + def _replacer(match): + if match.group(2) is not None: + return "" + else: + return match.group(1) + return regex.sub(_replacer, string) + +def isValidFile(_thisstring): + # return true if not png, tgz, or other non-text file + _isValidFile=True + if re.compile('.*\.(tgz|png|gif|jpg|pyc|pyo|git|swp)').match(_thisstring): + _isValidFile=False + #print( _thisstring + ": " + str(_isValidFile) ) + return _isValidFile + +# Main code +stringfile = open('scrub.txt','r') +count=0 +thisdir="" +newdir="" +oldstrings=[] +newstrings=[] + +while True: + x = stringfile.readline().rstrip() + count += 1 + if not x: break + x = removeComments(x) + #print("x=" + x) + y = shlex.split (x) + if len(y) >= 1: + if thisdir == "": + thisdir = y[0] + elif newdir == "": + newdir = y[0] + if len(y) >= 2: + #print("y[0]=" + y[0] + "\t and y[1]=" + y[1]) + oldstrings.append(y[0]) + newstrings.append(y[1]) + +# After the file is done +stringfile.close() +#newdir = thisdir.rstrip('\/') + ".scrubbed/" + +if False: + print("\nthisdir=" + thisdir) + print("newdir=" + newdir + '\n') + print("oldstrings are:") + print(oldstrings) + print("newstrings are:") + print(newstrings) + +# Clean scrubbed directory +try: + shutil.rmtree(newdir) +except: + foo=1 + +shutil.copytree(thisdir,newdir,symlinks=True) + +# Execute substitutions +for rootfolder, subdirs, files in os.walk(thisdir): + for filename in files: + sourcepath = os.path.join(rootfolder, filename) + destdir = rootfolder.replace(thisdir.rstrip('\/'),newdir.rstrip('\/')) + destfile = os.path.join(destdir, filename) + if os.path.islink( sourcepath ): + _symlinktarget = os.readlink( sourcepath ) + try: + os.symlink( _symlinktarget, destfile ) + except Exception as e: + pass + else: + with open( sourcepath, "r" ) as source: + #if not ".swp" in source.name and not ".git" in source.name and not ".tgz" in source.name: + if isValidFile( source.name ): + #print("sourcefile=" + source.name) + #print("destfile=" + destfile + '\n') + with open( destfile, "w") as target: + data = source.read() + for oldword, newword in zip(oldstrings, newstrings): + data = data.replace(oldword,newword) + changed = data + target.write(changed) + +# Execute directory renames +for rootfolder, subdirs, files in os.walk(newdir): + for subdir in subdirs: + oldpath = os.path.join(rootfolder, subdir) + for oldword, newword in zip(oldstrings, newstrings): + if oldword in oldpath: + os.rename(oldpath, oldpath.replace(oldword,newword)) + +# Execute file renames +# Used "file renames" reference, as well as the structure of directory traversal used earlier, which was from a different source. +for rootfolder, subdirs, files in os.walk(newdir): + for filename in files: + oldpath = os.path.join(rootfolder, filename) + for oldword, newword in zip(oldstrings, newstrings): + if oldword in oldpath: + #print("oldword=" + oldword + "\toldpath=" + oldpath) + os.rename(oldpath, oldpath.replace(oldword,newword)) diff --git a/etc/mirror/inc/scrub.txt b/etc/mirror/inc/scrub.txt new file mode 100644 index 0000000..7d93f9a --- /dev/null +++ b/etc/mirror/inc/scrub.txt @@ -0,0 +1,14 @@ +# First line: source directory Second line: target directory. WILL BE OVERWRITTEN! +/home/bgstack15/rpmbuild/SOURCES/mirror-1.0-4/ +/home/work/mirror.clean +# rest of the lines are "OLD WORD" "NEW WORD" +BGSTACK15 BGSTACK15 +Bgstack15 Bgstack15 +bgstack15 bgstack15 +Alice Alice +User1 User1 +example example +EXAMPLE EXAMPLE +".com" ".com" +"203.0." "203.0." +one one diff --git a/etc/mirror/mirror-master.sh b/etc/mirror/mirror-master.sh new file mode 100755 index 0000000..fbd2ab8 --- /dev/null +++ b/etc/mirror/mirror-master.sh @@ -0,0 +1,285 @@ +#!/bin/sh +# Filename: mirror-master.sh +# Location: brass.example.com:/etc/mirror +# Author: bgstack15@gmail.com +# Startdate: 2015-12-14 08:52:20 +# Title: Master Mirror Script +# Purpose: Wraps around individual mirror scripts and logs nicely +# Package: +# History: +# 2016-01-07 Fixed logic for detecting already-running instance +# 2016-06-08 Modified to use mirror.conf config file, for mirror-1.0-2 +# Usage: Call in cron every day +# Reference: ftemplate.sh 2015-11-23a; framework.sh 2015-11-23a +# mirror.lcroce (2014-11-11) +# Improve: +fiversion="2015-11-23a" +mirrormasterversion="2016-06-08a" + +usage() { + less -F >&2 <<ENDUSAGE +usage: mirror-master.sh [-duV] [ -f | --file /etc/mirror/mirror.conf ] [ --scriptsdir /etc/mirror/scripts ] [ scriptname ] +version ${mirrormasterversion} + -d debug Show debugging info, including parsed variables. + -u usage Show this usage block. + -V version Show script version number. + -f file Use specified config file. Default is /etc/mirror/mirror.conf + --scriptsdir Use specified scripts directory. Will override anything in the called conf file +Return values: +0 Normal +1 Help or version info displayed +2 mirror-master is already running +3 Unable to modify important file +4 Unable to find dependency +5 Not run as root or sudo +Examples: + mirror-master.sh centos + This command will run only the scripts/centos file. + mirror-master.sh centos putty + This command will run only centos and putty files. + mirror-master.sh + This command will run all files with o+x perms in scriptsdir directory. +ENDUSAGE +} + +# DEFINE FUNCTIONS + +# DEFINE TRAPS + +function clean_mirrormaster { + flecho "${scriptfile}: ENDED" | tee -a ${logfile} 1>&2 2>/dev/null + rm -f $lockfile >/dev/null 2>&1 + trap '' 0 + [ ] #use at end of entire script if you need to clean up tmpfiles +} + +function CTRLC { + #trap "CTRLC" 2 + [ ] #useful for controlling the ctrl+c keystroke +} + +function CTRLZ { + #trap "CTRLZ" 18 + [ ] #useful for controlling the ctrl+z keystroke +} + +function parseFlag { + flag=$1 + hasval=0 + case $flag in + # INSERT FLAGS HERE + "d" | "debug" | "DEBUG" | "dd" ) setdebug; ferror "debug level ${debug}";; + "u" | "usage" | "help") usage; exit 1;; + "V" | "fcheck" | "version") ferror "${scriptfile} version ${mirrormasterversion}"; exit 1;; + #"i" | "infile" | "inputfile") getval;infile1=$tempval;; + "f" | "file" | "conf" | "config") getval; conffile="${tempval}";; + "scriptsdir") getval; scriptsdir="${tempval}";; + esac + + debuglev 10 && { [[ hasval -eq 1 ]] && ferror "flag: $flag = $tempval" || ferror "flag: $flag"; } +} + +# DETERMINE LOCATION OF FRAMEWORK +while read flocation; do if [[ -x $flocation ]] && [[ $( $flocation --fcheck ) -ge 20160525 ]]; then frameworkscript=$flocation; break; fi; done <<EOFLOCATIONS +./framework.sh +/usr/bgscripts/framework.sh +EOFLOCATIONS +[[ -z "$frameworkscript" ]] && echo "$0: framework not found. Aborted." 1>&2 && exit 4 + +# INITIALIZE VARIABLES +# variables set in framework: +# today server thistty scriptdir scriptfile scripttrim +# is_cronjob stdin_piped stdout_piped stderr_piped sendsh sendopts +. ${frameworkscript} || echo "$0: framework did not run properly. Continuing..." 1>&2 +infile1= +outfile1= +# DEFAULTS unless overwritten by the mirror.conf or command line +lockfile=/var/lock/mirror.lock +logdir=/var/log/mirror/; mkdir -p ${logdir} 2>/dev/null; +logfile=${logdir}/mirror.${today}.log +errorfile=${logdir}/mirror.${today}.err +scriptsdirdefault=/etc/mirror/scripts +scriptsdir="${scriptsdirdefault}" +interestedparties="root" +keeplogs=14 +options= +conffiledefault=/etc/mirror/mirror.conf +conffile="${conffiledefault}" + +# REACT TO ROOT STATUS +case $is_root in + 1) # proper root + [ ] ;; + sudo) # sudo to root + [ ] ;; + "") # not root at all + if [[ ! $(whoami) = "mirror" ]]; + then + ferror "${scriptfile}: 5. Please run as root or sudo. Aborted." + exit 5 + fi + ;; +esac + +# SET CUSTOM SCRIPT AND VALUES +#setval 1 sendsh sendopts<<EOFSENDSH # if $1="1" then setvalout="critical-fail" on failure +#/usr/local/bin/bgscripts/send.sh -hs # setvalout maybe be "fail" otherwise +#/usr/local/bin/send.sh -hs # on success, setvalout="valid-sendsh" +#/usr/bin/mail -s +#EOFSENDSH +#[[ "$setvalout" = "critical-fail" ]] && ferror "${scriptfile}: 4. mailer not found. Aborted." && exit 4 + +# VALIDATE PARAMETERS +# objects before the dash are options, which get filled with the optvals +# to debug flags, use option DEBUG. Variables set in framework: fallopts +validateparams - "$@" + +# CONFIRM TOTAL NUMBER OF FLAGLESSVALS IS CORRECT +#if [[ $thiscount -lt 2 ]]; +#then +# ferror "${scriptfile}: 2. Fewer than 2 flaglessvals. Aborted." +# exit 2 +#fi + +# CONFIGURE VARIABLES AFTER PARAMETERS + +# ENSURE conffile exists +if [[ ! -f "${conffile}" ]]; +then + if [[ "${conffile}" = "${conffiledefault}" ]]; + then + # same as default, and does not exist. Throw warning and continue. + debuglev 1 && ferror "${scriptfile}: Warning 4. Conf ${conffile} not found. Resuming with defaults." + else + # specific conf file does not exist. Abort. + ferror "${scriptfile}: 4. Conf ${conffile} not found. Aborted." + exit 4 + fi +else +# READ CONFIG FILE TEMPLATE +[[ 1 -eq 1 ]] && { # REMOVE THIS LINE TO USE THE CONFIG FILE PARSER TEMPLATE +oIFS="${IFS}"; IFS=$'\n' +infiledata=( $( sed ':loop;/^\/\*/{s/.//;:ccom;s,^.[^*]*,,;/^$/n;/^\*\//{s/..//;bloop;};bccom;}' "${conffile}") ) #the crazy sed removes c style multiline comments +IFS="${oIFS}" +for line in "${infiledata[@]}"; +do line=$( echo "${line}" | sed -e 's/^\s*//;s/\s*$//;/^[#$]/d;s/\s*[^\]#.*$//;' ); [[ -n "${line}" ]] && { + # the crazy sed removes leading and trailing whitespace, blank lines, and comments + debuglev 8 && ferror "line=\"$line\"" + if echo "${line}" | grep -qiE "\[.*\]"; + then + # new zone + zone=$( echo "${line}" | tr -d '[]' ) + debuglev 7 && ferror "zone=${zone}" + else + # directive + varname=$( echo "${line}" | awk -F\ '{print $1}' ) + varval=$( echo "${line}" | awk -F\ '{$1=""; printf "%s", $0}' | sed 's/^ //;' ) + debuglev 7 && ferror $( eval echo ${varname}=\\\"${varval}\\\" ) + # simple define variable #eval "${zone}${varname}=\${varval}" + if [[ "${zone}" = "mirror" ]]; + then + eval "${varname}"=${varval} + fi + fi + #read -p "Please type something here:" response < $thistty + #echo "$response" +}; done +} # AND THIS LINE +fi + +## REACT TO BEING A CRONJOB +#if [[ $is_cronjob -eq 1 ]]; +#then +# [ ] +#else +# [ ] +#fi + +# SET TRAPS +#trap "CTRLC" 2 +#trap "CTRLZ" 18 +trap "clean_mirrormaster" 0 + +## PRE-RUN CHECKS +# Ensure not running +if [[ -e ${lockfile} ]]; +then + psmirror=$( cat ${lockfile} ) + if ps h ${psmirror} >/dev/null; + then + # lockfile exists and points to a valid running process + ferror "${scriptfile}: 2. Mirror is already running. Aborted." + exit 2 + else + rm ${lockfile} + fi +fi + +# Lock the queue +if ! touch ${lockfile}; +then + ferror "${scriptfile}: 2. Could not create lockfile ${lockfile}. Aborted." + exit 2 +else + echo "$$" > ${lockfile} +fi + +# Ensure files can be created +for word in ${logfile} ${errorfile}; +do + if ! touch ${word}; + then + ferror "${scriptfile}: 3. Could not modify file ${word}. Aborted." + exit 3 + fi +done + +# Perform logfile cleanup +if [ -n ${keeplogs} ]; +then + find "${logdir}" -type f -mtime "+${keeplogs}" -exec rm -f {} \; 2>/dev/null +fi + +# MAIN LOOP +exec 3>&1 +{ + { + debugstring= + debuglev 1 && debugstring="debugging " + flecho "BEGIN ${debugstring}mirror" + flecho "BEGIN ${debugstring}errors" 1>&2 + + if [ -n thiscount ] && [[ thiscount -ge 1 ]]; + then + for word in ${fallopts}; + do + thisfile=$( find ${scriptsdir} -type f -perm /o=x -name "${word}" ) + if debuglev 1; + then + [[ -f "${thisfile}" ]] && \ + echo "${thisfile}" || \ + flecho "not found: ${scriptsdir}/${word}. Skipped." 1>&2 + else + [[ -f "${thisfile}" ]] && \ + find ${scriptsdir} -type f -perm /o=x -name "${word}" -exec {} $options \; || \ + flecho "not found: ${scriptsdir}/${word}. Skipped." 1>&2 + fi + done + else + if debuglev 1; + then + find "${scriptsdir}" -type f -perm /o=x + else + find "${scriptsdir}" -type f -perm /o=x -exec {} $options \; + fi + fi + + flecho "END ${debugstring}mirror" + flecho "END ${debugstring}errors" 1>&2 + } 2>&1 1>&3 | tee -a ${errorfile} 1>&2 +} 3>&1 | tee -a $logfile + +# EMAIL LOGFILE +# will only get here if not interrupted by user input +cat ${errorfile} >> ${logfile} +#$sendsh $sendopts "$server $scriptfile out" $logfile $interestedparties diff --git a/etc/mirror/mirror.conf b/etc/mirror/mirror.conf new file mode 100644 index 0000000..69f38c7 --- /dev/null +++ b/etc/mirror/mirror.conf @@ -0,0 +1,13 @@ +# File: etc/mirror/mirror.conf +# Package: mirror +[mirror] +# Shell variable manipulation is possible. For example: +# logfile ${logdir}/mirror.${today}.log +# Because of this, you must escape any spaces in the value of any variable. +lockfile /var/lock/mirror.lock +logdir /var/log/mirror +logfile ${logdir}/mirror.${today}.log +errorfile ${logdir}/mirror.${today}.err +scriptsdir /etc/mirror/scripts +keeplogs 14 +#mirrordir is actually manually configured for each executable script. See $mirrorscriptsdir diff --git a/etc/mirror/packaging.txt b/etc/mirror/packaging.txt new file mode 100644 index 0000000..18349a7 --- /dev/null +++ b/etc/mirror/packaging.txt @@ -0,0 +1,35 @@ +File: etc/mirror/packaging.txt +Package: mirror +Author: bgstack15 +Startdate: 2016-06-09 +Title: Packaging instructions for mirror +Purpose: To make it easier to build the package +Usage: +Reference: bgscripts package packaging.txt +Improve: +Document: Below this line + +### PACKAGING for rhel/centos +rpmbuilddir=~/rpmbuild/ +package=mirror +version=1.0-4 +shortversion=1.0 +packagespecfile="${package}-${version}/etc/${package}/docs/${package}.spec" +sed -i -n '1,/^\%files$/p;' ${rpmbuilddir}/SOURCES/"${packagespecfile}" +cd ${rpmbuilddir}/SOURCES/"${package}-${version}" +find * ! -type d ! -regex '.*?.swp' ! -regex '.*?DEBIAN.*?' | sed -e 's/^/\//;' -e 's/\(.*\.txt\)/%doc %attr(444, -, -) \1/;' -e 's/\(.*\.\(conf\|cron\)\|.*sudoers\.d\)/%config \1/;' -e 's/\(.*bin.*\)/%verify(link) \1/;' -e 's/\(.*\)\.py$/\1\.py\n\1\.pyc\n\1\.pyo/;' >> ${rpmbuilddir}/SOURCES/"${packagespecfile}" +rm -rf ${rpmbuilddir}/SOURCES/"${package}-${shortversion}" +cp -prf ${rpmbuilddir}/SOURCES/"${package}-${version}" ${rpmbuilddir}/SOURCES/"${package}-${shortversion}" +rm -rf ${rpmbuilddir}/SOURCES/"${package}-${shortversion}"/DEBIAN +cd ${rpmbuilddir}/SOURCES +rm -rf "${package}".tgz; tar -zc --exclude='.git' -f "${package}.tgz" "${package}-${shortversion}" && rm -rf ${rpmbuilddir}/SOURCES/"${package}-${shortversion}" +cp -p ${rpmbuilddir}/SOURCES/"${packagespecfile}" ${rpmbuilddir}/SPECS +cd ${rpmbuilddir}/RPMS/noarch +rpmbuild -bb "${rpmbuilddir}"/SPECS/"${package}".spec + +### PACKAGING in a master.tgz +cd ~/deb 2>/dev/null || cd ~/rpmbuild/SOURCES +package=mirror +version=1.0-4 +rm -rf ./"${package}-${version}".master.tgz +tar -zcf "${package}-${version}".master.tgz "${package}-${version}"/ diff --git a/etc/mirror/scripts/centos b/etc/mirror/scripts/centos new file mode 100644 index 0000000..b812bde --- /dev/null +++ b/etc/mirror/scripts/centos @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/centos +# ftp://mirror.nandomedia.com/pub/CentOS/ + +source /etc/mirror/inc/rsync + +rmirror mirrors.kernel.org::centos centos '[234]*/' 'alpha/' 's390*/' 'ia64/' diff --git a/etc/mirror/scripts/fedora-epel b/etc/mirror/scripts/fedora-epel new file mode 100644 index 0000000..88f0c77 --- /dev/null +++ b/etc/mirror/scripts/fedora-epel @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/fedora-epel +# + +source /etc/mirror/inc/rsync + +rmirror mirror.cogentco.com::fedora-epel fedora/epel 4/ testing/ diff --git a/etc/mirror/scripts/fedora-releases b/etc/mirror/scripts/fedora-releases new file mode 100644 index 0000000..8f33c46 --- /dev/null +++ b/etc/mirror/scripts/fedora-releases @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/fedora-releases +# + +source /etc/mirror/inc/rsync + +rmirror mirrors.kernel.org::fedora/releases fedora/linux/releases 21/ test/ */Live */debug */armhfp diff --git a/etc/mirror/scripts/fedora-updates b/etc/mirror/scripts/fedora-updates new file mode 100644 index 0000000..44cfe02 --- /dev/null +++ b/etc/mirror/scripts/fedora-updates @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/fedora-updates +# + +source /etc/mirror/inc/rsync + +rmirror mirror.lstn.net::fedora-enchilada/updates fedora/linux/updates '[89]/' '1[0-9]/' 20/ testing/ */ppc */ppc64 */armhfp diff --git a/etc/mirror/scripts/libreoffice b/etc/mirror/scripts/libreoffice new file mode 100644 index 0000000..cf88808 --- /dev/null +++ b/etc/mirror/scripts/libreoffice @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/libreoffice +# https://wiki.documentfoundation.org/Infra/Mirroring#Example_commands + +# + +rsync --no-motd -rlptH -hi --delete rsync://rsync.documentfoundation.org/tdf-pub/ /mnt/mirror/tdf diff --git a/etc/mirror/scripts/linuxmint-isos b/etc/mirror/scripts/linuxmint-isos new file mode 100644 index 0000000..674f96e --- /dev/null +++ b/etc/mirror/scripts/linuxmint-isos @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/linuxmint +# + +source /etc/mirror/inc/rsync + +rmirror rsync://ftp.heanet.ie/pub/linuxmint.com linuxmint/isos diff --git a/etc/mirror/scripts/linuxmint-repos b/etc/mirror/scripts/linuxmint-repos new file mode 100644 index 0000000..d73136c --- /dev/null +++ b/etc/mirror/scripts/linuxmint-repos @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/linuxmint +# + +source /etc/mirror/inc/rsync + +rmirror rsync://rsync-packages.linuxmint.com/packages linuxmint/packages diff --git a/etc/mirror/scripts/putty b/etc/mirror/scripts/putty new file mode 100644 index 0000000..82e7edd --- /dev/null +++ b/etc/mirror/scripts/putty @@ -0,0 +1,13 @@ +#!/bin/sh +# File: /etc/mirror/scripts/putty +# + +source /etc/mirror/inc/rsync + +rmirror rsync.chiark.greenend.org.uk::ftp/users/sgtatham/putty-website-mirror putty + +# make symlink for latest +rm -f /mnt/mirror/putty/latest +latest=$( find /mnt/mirror/putty/* -maxdepth 0 -type d -regex '.*[0-9]\.[0-9][0-9].*' | sort -n | tail -n1 ) +latest=${latest##*/} +ln -s "${latest}" /mnt/mirror/putty/latest diff --git a/etc/mirror/scripts/rsyslog b/etc/mirror/scripts/rsyslog new file mode 100755 index 0000000..8776f43 --- /dev/null +++ b/etc/mirror/scripts/rsyslog @@ -0,0 +1,12 @@ +#!/bin/sh +# File: /etc/mirror/scripts/rsyslog +# Reference: http://www.freeside.biz/mediawiki/index.php/Freeside:Documentation:MirroringRPMRepo + +source /etc/mirror/inc/rsync + +mkdir -p /mnt/mirror/rsyslog >/dev/null 2>&1 +cd /mnt/mirror/rsyslog +wget --mirror -nv -nH -np http://rpms.adiscon.com/v8-stable/ + +echo "Removing index.html files:" +find /mnt/mirror/rsyslog -name 'index.html*' -print -exec /bin/rm {} \; diff --git a/etc/mirror/scripts/ssh b/etc/mirror/scripts/ssh new file mode 100644 index 0000000..d08f629 --- /dev/null +++ b/etc/mirror/scripts/ssh @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/ssh +# + +source /etc/mirror/inc/rsync + +rmirror mirrors.nycbug.org::OpenBSD/OpenSSH ssh diff --git a/etc/mirror/scripts/ubuntu b/etc/mirror/scripts/ubuntu new file mode 100644 index 0000000..9571444 --- /dev/null +++ b/etc/mirror/scripts/ubuntu @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/ubuntu/ubuntu +# + +source /etc/mirror/inc/rsync + +rmirror archive.ubuntu.com::ubuntu ubuntu/ubuntu *breezy* *warty* *hoary* *dapper* *edgy* *feisty* gutsy* intrepid* jaunty* karmic* maverick* natty* oneiric* quantal* raring* saucy* *[_-]powerpc* *[_-]sparc* *[_-]ia64* pool/**/*.tar.* *.diff.gz* *.dsc source/ diff --git a/etc/mirror/scripts/ubuntu-releases b/etc/mirror/scripts/ubuntu-releases new file mode 100644 index 0000000..14a2406 --- /dev/null +++ b/etc/mirror/scripts/ubuntu-releases @@ -0,0 +1,7 @@ +#!/bin/sh +# File: /etc/mirror/scripts/ubuntu-releases +# + +source /etc/mirror/inc/rsync + +rmirror rsync://mirrors.us.kernel.org/ubuntu-releases/ ubuntu/releases edubuntu/ *powerpc* *sparc* *-4.*-* *-5.*-* *-6.*-* diff --git a/etc/sudoers.d/50_mirror-sudo b/etc/sudoers.d/50_mirror-sudo new file mode 100644 index 0000000..be019d8 --- /dev/null +++ b/etc/sudoers.d/50_mirror-sudo @@ -0,0 +1,3 @@ +# for user mirror to run mirror-master +mirror brass=(ALL) NOPASSWD: /etc/mirror/mirror-master.sh + diff --git a/usr/local/bin/deploy b/usr/local/bin/deploy new file mode 120000 index 0000000..f7b33a6 --- /dev/null +++ b/usr/local/bin/deploy @@ -0,0 +1 @@ +/etc/mirror/inc/deploy.sh
\ No newline at end of file diff --git a/usr/share/httpd/icons/repo.png b/usr/share/httpd/icons/repo.png Binary files differnew file mode 100644 index 0000000..c34fd72 --- /dev/null +++ b/usr/share/httpd/icons/repo.png diff --git a/usr/share/httpd/icons/rpm.png b/usr/share/httpd/icons/rpm.png Binary files differnew file mode 100644 index 0000000..0ff3864 --- /dev/null +++ b/usr/share/httpd/icons/rpm.png |
