path: root/scripts/prep-librewolf-dpkg.sh

#!/bin/sh
# File: prep-librewolf-dpkg.sh
# Location: https://gitlab.com/bgstack15/librewolf-linux.git
# Latest supported version: librewolf-109.0-1
# Author: bgstack15
# SPDX-License-Identifier: CC-BY-SA-4.0
# Startdate: 2020-11-29
# Title: Build Dpkg for LibreWolf
# Purpose: Prepare initial assets for running "dpkg-buildpackage -b -us -uc" for LibreWolf by adapting distro Firefox assets
# History:
#    2021-03-10 add initial Ubuntu support
#    2022-01-18 modified to use "source" repo instead of "common" for version 97.0.1
# Usage:
#    Can send these final assets up to Open Build Service
# References:
#    Script numbers from https://gitlab.com/librewolf-community/browser/linux/-/tree/master/binary_tarball/scripts
# Improve:
#    Make this idempotent. Right now it is very much not.
# Dependencies:
#    wget, git, tar, awk, sed, jq

# Aborts the script upon any failure
set -e;

#####################################
# Load settings
# basically, dot-source the conf file.
test -z "${librewolf_dpkg_conf}" && export librewolf_dpkg_conf="$( find "$( dirname "${0}" )" -maxdepth 2 -name "$( basename "${0%%.sh}.conf" )" -print 2>/dev/null | head -n1 )"
test ! -r "${librewolf_dpkg_conf}" && { echo "Unable to load config file, which should be named the same as this script but with a .conf ending. Aborted." 1>&2 ; exit 1 ; }
. "${librewolf_dpkg_conf}"

librewolf_source_url=https://gitlab.com/librewolf-community/browser/source.git
librewolf_settings_url=https://gitlab.com/librewolf-community/settings.git
librewolf_linux_url=https://gitlab.com/librewolf-community/browser/linux.git

if test "${DISTRO}" = "ubuntu" ; then
   _mozconfig='/config/mozconfig.in'
else
   _mozconfig='/browser.mozconfig.in'
fi

# user configurable
git_source_dir=${CI_PROJECT_DIR}/git  # where LibreWolf git contents are cached
debian_dir=${CI_PROJECT_DIR}/${firefox_version}/debian # where the firefox_debian.tar.xz file is extracted
source_dir=${CI_PROJECT_DIR}/${firefox_version}/librewolf_${firefox_version} # where firefox.orig.tar.xz file is extracted with --strip-components=1
work_dir=${CI_PROJECT_DIR}/prepared/

#####################################
# Download initial components
test -z "${SKIP_APT}" && apt update && apt install -y git curl wget xz-utils python3-pip jq

# Download upstream distro assets, which includes
#   1. orig tarball, which in Debian is not always the pristine contents from upstream source
#   2. debian/ directory which defines how to build a dpkg
#   3. source package control file
mkdir -p "${work_dir}" ; cd "${work_dir}"
test -z "${SKIP_DOWNLOAD}" && {
   case "${DISTRO}" in
      ubuntu)
         wget --continue --content-disposition http://archive.ubuntu.com/ubuntu/pool/main/f/firefox/firefox_"${firefox_version}".orig.tar.xz
         wget --continue --content-disposition http://archive.ubuntu.com/ubuntu/pool/main/f/firefox/firefox_"${distro_firefox_version}".debian.tar.xz
         wget --continue --content-disposition http://archive.ubuntu.com/ubuntu/pool/main/f/firefox/firefox_"${distro_firefox_version}".dsc
         ;;
      *) # catch-all, including for Debian
         wget --continue --content-disposition http://deb.debian.org/debian/pool/main/f/firefox/firefox_"${firefox_version}".orig.tar.xz
         wget --continue --content-disposition http://deb.debian.org/debian/pool/main/f/firefox/firefox_"${distro_firefox_version}".debian.tar.xz
         wget --continue --content-disposition http://deb.debian.org/debian/pool/main/f/firefox/firefox_"${distro_firefox_version}".dsc
         ;;
   esac
}

# extract these contents to where they belong
mkdir -p "${source_dir}"
test -z "${SKIP_EXTRACT}" && {
   echo "Extracting files from orig and debian tarballs. This might take a while." 1>&2
   tar -C "${source_dir}" -Jx --strip-components=1 -f firefox_"${firefox_version}".orig.tar.xz
   tar -C "$( dirname "${debian_dir}" )" -Jxf firefox_"${distro_firefox_version}".debian.tar.xz
   # dsc file is a text file and needs no extraction
}

# Download git sources
test -z "${SKIP_GIT}" && {
   git clone "${librewolf_source_url}" ${git_source_dir}/source || : ; ( cd "${git_source_dir}/source" ; git checkout master || git checkout main ; git pull ; git checkout "${librewolf_version}" ; )
   git clone "${librewolf_settings_url}" ${git_source_dir}/settings || : ; ( cd "${git_source_dir}/settings" ; git checkout master || git checkout main ; git pull ; )
   git clone "${librewolf_linux_url}" ${git_source_dir}/linux || : ; ( cd "${git_source_dir}/linux" ; git checkout master || git checkout main ; git pull ; git checkout "v${librewolf_version}" ; )
}

#####################################
# Script 1 tasks

# update debian/control file
# update fields and add libjack-dev
sed -i -r "${debian_dir}"/control \
   -e '/^[[:alpha:]]+: firefox/s/firefox/librewolf/' \
   -e '/^Package:.*-l10/,$d' \
   -e '/^Maintainer:/{s/Maintainer:/XSBC-Original-Maintainer:/;iMaintainer: B. Stack <bgstack15@gmail.com>' -e '}' \
   -e '/^Uploaders:/d' \
   -e '/libasound2-dev/s/libasound2-dev,/libasound2-dev, libjack-dev,/;' \
   -e '/^Vcs-/d' \
   -e '/Breaks:.*xul-ext-torbutton/d' \
   -e '/Description:/,+8{/Description:/,/^\s*$/d}'
cat <<'EOF' >> "${debian_dir}"/control
Description: LibreWolf variant of Mozilla Firefox web browser
 LibreWolf is a build of Firefox that seeks to protect user privacy,
 security, and freedom.
EOF

#####################################
# Script 2 tasks

# none. Dependencies are handled by the build environment by interpreting the dsc file.

#####################################
# Script 3 tasks

# overlay the orig tarball contents with LibreWolf contents
# LibreWolf branding
cp -pr "${git_source_dir}"/source/themes/browser/base \
   "${git_source_dir}"/source/themes/browser/branding \
   "${source_dir}"/browser/
# update mozconfig with needed info
sed -i -e '/with-app-name=/d' "${debian_dir}${_mozconfig}"
cat <<EOF >> "${debian_dir}${_mozconfig}"

# Start of LibreWolf effects
ac_add_options --disable-tests
ac_add_options --disable-debug
ac_add_options --disable-default-browser-agent

ac_add_options --prefix=/usr
ac_add_options --enable-hardening
ac_add_options --enable-rust-simd

# Branding
ac_add_options --with-app-name=librewolf
ac_add_options --with-app-basename=LibreWolf
ac_add_options --with-branding=browser/branding/librewolf
ac_add_options --with-distribution-id=io.gitlab.librewolf-community

export MOZ_REQUIRE_SIGNING=

# Features
ac_add_options --enable-jack
ac_add_options --disable-crashreporter

# Disables crash reporting, telemetry and other data gathering tools
mk_add_options MOZ_CRASHREPORTER=0
mk_add_options MOZ_DATA_REPORTING=0
mk_add_options MOZ_SERVICES_HEALTHREPORT=0
mk_add_options MOZ_TELEMETRY_REPORTING=0
#ac_add_options --with-wasi-sysroot=\$HOME/.mozbuild/wrlb/wasi-sysroot

EOF
test "$( uname -p )" = "x86_64" && { echo "ac_add_options --disable-elf-hack" >> "${debian_dir}${_mozconfig}" ; }
cat <<EOF >> "${debian_dir}${_mozconfig}"
# LibreWolf binary release uses clang-11 but Debian builds Firefox with gcc so this is irrelevant.
#export CC='clang-11'
#export CXX='clang++-11'
#export AR=llvm-ar-11
#export NM=llvm-nm-11
#export RANLIB=llvm-ranlib-11

ac_add_options --enable-optimize
EOF

# add patches to debian/patches
mkdir -p "${debian_dir}"/patches/librewolf

if test "${DISTRO}" = "ubuntu" ; then
   cp -pr "${git_source_dir}"/common/patches/megabar.patch \
      "${git_source_dir}"/common/patches/remove_addons.patch \
      "${git_source_dir}"/common/patches/mozilla-vpn-ad2.patch \
      "${git_source_dir}"/common/patches/sed-patches/disable-pocket.patch \
      "${git_source_dir}"/common/patches/context-menu.patch \
      "${git_source_dir}"/common/patches/browser-confvars.patch \
      "${git_source_dir}"/common/patches/urlbarprovider-interventions.patch \
      "${git_source_dir}"/common/patches/sed-patches/remove-internal-plugin-certs.patch \
      "${git_source_dir}"/common/patches/sed-patches/allow-searchengines-non-esr.patch \
      "${git_source_dir}"/common/patches/sed-patches/stop-undesired-requests.patch \
      "${git_source_dir}"/common/patches/about-dialog.patch \
      "${debian_dir}"/patches/librewolf/
   cat <<EOF >> "${debian_dir}"/patches/series
librewolf/remove_addons.patch -p1
librewolf/megabar.patch -p1
librewolf/mozilla-vpn-ad2.patch -p1
librewolf/disable-pocket.patch -p1
librewolf/context-menu.patch -p1
librewolf/browser-confvars.patch -p1
librewolf/urlbarprovider-interventions.patch -p1
librewolf/remove-internal-plugin-certs.patch -p1
librewolf/allow-searchengines-non-esr.patch -p1
librewolf/stop-undesired-requests.patch -p1
librewolf/about-dialog.patch -p1
EOF
else
   # the vpn-ad was disabled at v99.0 because it was broken
   cp -pr $( sed -r "${git_source_dir}"/source/assets/patches.txt -e "s@^@/${git_source_dir}/source/@" -e '/vpn-ad/d' ) \
      "${debian_dir}"/patches/librewolf/
   awk -F'/' '{print "librewolf/"$NF}' "${git_source_dir}"/source/assets/patches.txt | \
      sed -r -e 's@$@ -p1@' -e '/vpn-ad/d' >> "${debian_dir}/patches/series"
   #echo "librewolf/webrender_ffi.patch -p1" >> "${debian_dir}/patches/series"
   # added 109.0 to patch for debian 109.0 until defuzz-109.0 gets merged
   test -n "1" && patch -p1 "${debian_dir}/patches/librewolf/disable-data-reporting-at-compile-time.patch" <<'EOF'
diff --git a/patches/disable-data-reporting-at-compile-time.patch b/patches/disable-data-reporting-at-compile-time.patch
index 3702402..f4a4315 100644
--- a/patches/disable-data-reporting-at-compile-time.patch
+++ b/patches/disable-data-reporting-at-compile-time.patch
@@ -1,5 +1,3 @@
-diff --git a/browser/moz.configure b/browser/moz.configure
-index 8653bcb..935af40 100644
 --- a/browser/moz.configure
 +++ b/browser/moz.configure
 @@ -5,11 +5,11 @@
@@ -14,5 +12,5 @@ index 8653bcb..935af40 100644
 -imply_option("MOZ_NORMANDY", True)
 +imply_option("MOZ_NORMANDY", False)
  
- with only_when(target_is_linux & compile_environment):
+ with only_when(target_has_linux_kernel & compile_environment):
      option(env="MOZ_NO_PIE_COMPAT", help="Enable non-PIE wrapper")
EOF
   # added 101.0.1
   # ref: https://raw.githubusercontent.com/canonical/firefox-snap/5622734942524846fb0eb7108918c8cd8557fde3/patches/fix-ftbfs-newer-cbindgen.patch
   # ref: https://bugzilla.mozilla.org/show_bug.cgi?id=1773630
   # ref: https://gitlab.com/librewolf-community/browser/gentoo/-/issues/42#note_985094574
   # ref: https://forums.gentoo.org/viewtopic-p-8718512.html?sid=b090546e8d9bcd60ff775c32d04e1743
   # ref: https://www.google.com/search?hl=en&q=build%2Dbrowser%2Fdist%2Finclude%2Fmozilla%2Fwebrender%2Fwebrender_ffi.h%3A103
   # For 100.0 to fix this patch.
   test -n "1" && cat <<EOF >"${debian_dir}/patches/librewolf/remove-branding-urlbar.patch"
--- a/browser/locales/en-US/browser/browser.ftl.orig	2022-03-22 12:57:21.330016999 -0400
+++ b/browser/locales/en-US/browser/browser.ftl	2022-03-22 13:01:42.740728393 -0400
@@ -543,7 +543,7 @@
 # A label shown above the "Firefox Suggest" (bookmarks/history) group in the
 # urlbar results.
 urlbar-group-firefox-suggest =
-  .label = { -firefox-suggest-brand-name }
+  .label = Suggestions
 
 # A label shown above the search suggestions group in the urlbar results. It
 # should use sentence case.
EOF
   # remove xmas because it does the same tasks as what we accomplish in this script,
   sed -i -r -e '/(xmas)\.patch/s/^/#/;' "${debian_dir}"/patches/series

   # Fixes for Debian Firefox? 97.0
   pip3 install --user psutil==5.8.0
   # there's probably a better way to do this with pip, but I don't know it.
   # Basically, we are updating importlib_metadata and attrs to min needed because
   # somehow they're wrong in the orig.tar.xz.
   pip3 install --target "${source_dir}"/third_party/python attrs importlib_metadata
   cd "${source_dir}"/third_party/python
   for word in attrs importlib_metadata zipp ;
   do
      # yes, move importlib_metadata-4.10.1.dist-info/ to inside importlib_metadata/
      # we could not just loop the pip3 command because zipp is dragged in by importlib_metadata.
      mv -f "${word}"-* "${word}"/ || :
   done
   cd "${OLDPWD}"
fi

# observe that build-with-libstdc++-7 is disabled for this dpkg. Debian builds Firefox with gcc, not clang.

#####################################
# Script 4 tasks

if test "${DISTRO}" = "debian" ; then

sed -i -r -e '2{
   iexport DEB_BUILD_HARDENING=1
   ;iexport DEB_BUILD_HARDENING_STACKPROTECTOR=1
   ;iexport DEB_BUILD_HARDENING_FORTIFY=1
   ;iexport DEB_BUILD_HARDENING_FORMAT=1
   ;iexport DEB_BUILD_HARDENING_PIE=1
   ;iexport CPP
}
/^EXPORTS/{
   iCPPFLAGS += -D_FORTIFY_SOURCE=2
   ;iCFLAGS += -march=x86-64 -mtune=generic -O2 -pipe -fno-plt
   ;iCXXFLAGS += -march=x86-64 -mtune=generic -O2 -pipe -fno-plt
   ;iLDFLAGS += -Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now
}
2{
   iexport MOZ_NOSPAM=1
}
' "${debian_dir}"/rules
   # removed from the "2{" block at end, for version 104.0
   #iexport MACH_USE_SYSTEM_PYTHON=1

fi

#####################################
# Additional steps for dpkg implementation

# fix the binary name that gets installed in /usr/bin, and disable crash reporter by changing what variable name it looks for that will enable it

if test "${DISTRO}" = "debian" ; then

   sed -i -e '/%if browser/,+2s/firefox/librewolf/' \
      -e '/%if CRASH_REPORTER/s/CRASH_REPORTER/CRASH_REPORTER_ENABLED/' \
      "${debian_dir}"/browser.install.in

elif test "${DISTRO}" = "ubuntu" ; then

   # let's hope this is enough
   export MOZ_ENABLE_BREAKPAD=0
   export MOZ_APP_NAME=librewolf

fi

# instruct dpkg to include the librewolf settings
rm -rf "${debian_dir}"/librewolf_settings
cp -pr "${git_source_dir}"/settings "${debian_dir}"/librewolf_settings
rm -rf "${debian_dir}"/librewolf_settings/.git*

cp -p "${git_source_dir}"/source/assets/search-config.json "${debian_dir}"/
sed -i -r "${debian_dir}"/rules \
   -e '/override_dh_auto_configure/a\	cp -pf debian/search-config.json services/settings/dumps/main/'

# Fetch latest uBlock Origin webextensions
# get latest non-beta, non-Lite tag version
ubo_version="$( curl --silent https://api.github.com/repos/gorhill/uBlock/releases | jq '.[] | select(.tag_name|contains("b")|not)|select(.tag_name|contains("Lite")|not)|.tag_name' | head -n1 | tr -d '"' )"
latest_ubo_url="$( curl --silent "https://api.github.com/repos/gorhill/uBlock/releases/tags/${ubo_version}" | jq '.assets[].browser_download_url | match(".*firefox.*xpi").string' | tr -d '"' )"
curl --silent --location --output "${debian_dir}/uBlock0@raymondhill.net.xpi" "${latest_ubo_url}"

if test "${DISTRO}" = "debian" ; then

   cat <<EOF >> "${debian_dir}"/browser.install.in
debian/librewolf_settings/@browser@.cfg usr/share/@browser@
debian/librewolf_settings/defaults/pref/local-settings.js usr/share/@browser@/defaults/pref
debian/librewolf_settings/distribution/policies.json usr/share/@browser@/distribution
debian/uBlock0@raymondhill.net.xpi usr/lib/@browser@/browser/extensions
EOF
   cat <<EOF >> "${debian_dir}/source/include-binaries"
debian/uBlock0@raymondhill.net.xpi
EOF
   cat <<EOF >> "${debian_dir}"/browser.links.in
/usr/share/distribution/policies.json usr/lib/@browser@/distribution/policies.json
/usr/share/@browser@/defaults/pref/local-settings.js usr/lib/@browser@/defaults/pref/local-settings.js
/usr/share/@browser@/@browser@.cfg /usr/lib/@browser@/@browser@.cfg
EOF

# somewhat of a  brute force approach - should help having everything everywhere, though
sed -e 's#/usr/share/@browser@/distribution /usr/lib/@browser@/distribution##' -i "${debian_dir}"/browser.links.in

else

   cat <<EOF >> "${debian_dir}"/firefox.install.in

debian/librewolf_settings/librewolf.cfg @MOZ_LIBDIR@/
debian/librewolf_settings/defaults/pref/local-settings.js @MOZ_LIBDIR@/defaults/pref
debian/librewolf_settings/distribution/policies.json @MOZ_LIBDIR@/distribution
EOF

fi

# add changelog contents for LibreWolf
new_changelog="$( mktemp )"
{
   cat <<EOF
librewolf (${distro_firefox_version}) unstable; urgency=low

  * Fork to librewolf release

 -- B. Stack <bgstack15@gmail.com>  $( date "+%a, %d %b %+4Y %T %z" )

EOF
   cat "${debian_dir}"/changelog
} > "${new_changelog}"
cat "${new_changelog}" > "${debian_dir}"/changelog

rm -f "${new_changelog:-NOTHINGTODEL}"

# remove dpkg-divert items which librewolf will not use
# thankfully dpkg-divert is the only tasks in these files as of 85.0
rm -f "${debian_dir}"/browser.postrm.in "${debian_dir}"/browser.preinst.in

#####################################
# Build new assets
# dpkg-buildpackage needs the orig tarball, debian tarball, and dsc file.

test -z "${SKIP_TARBALLS}" && {
   echo "Building new tarballs. This might take a while." 1>&2
   # orig tarball
   cd "${work_dir}"
   tar -Jc -f librewolf_"${firefox_version}".orig.tar.xz -C "$( dirname "${source_dir}" )" librewolf_"${firefox_version}"
   # debian tarball
   tar -Jc -f librewolf_"${distro_firefox_version}".debian.tar.xz -C "$( dirname  "${debian_dir}" )" debian
}

# dsc file, which needs to be modified
cd "${work_dir}"
sed -r \
   -e '/^(Files|Checksums-.{0,8}):/,$d' \
   -e '1,/^Format:/{/^Format:/!{d}}' \
   -e 's/^([[:alpha:]]+:).* firefox(-l10n[^\s]*)*/\1 librewolf/' \
   -e '/firefox-l10n/d' \
   -e '/^Maintainer:/{s/Maintainer:/XSBC-Original-Maintainer:/;iMaintainer: B. Stack <bgstack15@gmail.com>' -e '}' \
   -e '/^Uploaders:/d' \
   -e '/libasound2-dev/s/libasound2-dev,/libasound2-dev, libjack-dev,/;' \
   -e '/^Vcs-/d' \
   -e '/^ firefox/s/firefox/librewolf/g' \
   firefox_"${distro_firefox_version}".dsc > librewolf_"${distro_firefox_version}".dsc
{
   echo "Files:"
   for word in librewolf*z ;
   do
      printf "%s %s\n" "$( stat -c '%s' "${word}" )" "$( md5sum "${word}" )"
   done | awk '{print " "$2,$1,$3}'
} >> librewolf_"${distro_firefox_version}".dsc

# And now you have in the ${work_dir} location three files.
# librewolf_80.3.orig.tar.xz librewolf_80.3-1.debian.tar.xz librewolf_80.3-1.dsc