add config class, and eot-ttf conversion

   eot2ttf depends on external, non-python programs but which are
   available in both target distros.

   The config class simplifies passing parameters, but requires
   initialization.

1 files changed, 135 insertions, 67 deletions

diff --git a/savewebfonts_lib.py b/savewebfonts_lib.py
index 19790df..0c5943e 100755
--- a/savewebfonts_lib.py
+++ b/savewebfonts_lib.py
@@ -10,62 +10,90 @@
 # Reference:
 #    https://github.com/fonttools/fonttools/issues/1694
 # Improve:
-#    accept a list of filetypes to save, or exclude? Such as, ['ttf','woff2']
-#    Convert woff2 fonts?
 #    Handle using tinycss old?
 # Dependencies:
 #    req-fedora: python3-beautifulsoup4, python3-tinycss2
-#    rec-fedora: python3-fonttools
-import requests, os, json, tempfile
+#    rec-fedora: python3-fonttools, libeot-tools
+#    req-devuan: python3-bs4, python3-tinycss2
+#    rec-devuan: python3-fonttools, eot2ttf
+
+import requests, os, json, tempfile, subprocess
 from sys import stderr
 from bs4 import BeautifulSoup as bs # python3-beautifulsoup4
 from urllib.parse import urljoin, urlparse
 import tinycss2 # python3-tinycss2
 
 # defaults for library
-debuglevel = 8
-MAX_STRING_PRINT_LENGTH = 180
+class swf_config:
+   def __init__(
+         self
+         , debuglevel = 8
+         , session = None
+         , MAX_STRING_PRINT_LENGTH = 180
+         , eot2ttf_binary = "eot2ttf"
+         , dryrun = False
+         , convert = False
+      ):
+      self.debuglevel = debuglevel
+      self.MAX_STRING_PRINT_LENGTH = MAX_STRING_PRINT_LENGTH
+      self.eot2ttf_binary = eot2ttf_binary
+      self.dryrun = dryrun
+      if session is None:
+         self.session = get_session()
+      else:
+         self.session = session
+      self.convert = convert
+
+   def __repr__(self):
+      response = "<swf_config"
+      for i in self.__dict__:
+         # omit printing session
+         if "session" not in i:
+            response = response + " " + (str(i)) + "=\"" + str(self.__dict__[i]) + "\","
+      response = response.rstrip(",") + ">"
+      return response
 
 # Functions
 def eprint(*args, **kwargs):
    print(*args, file=stderr, **kwargs)
 
 def ttfify_filename(filename):
-   return filename.rstrip(".woff").rstrip(".woff2").rstrip(".svg").rstrip(".eot") + ".ttf"
-
+   response = filename
+   for end in [".woff2",".woff",".eot",".svg"]:
+      if response.endswith(end):
+         response = response[:-len(end)]
+      # For python 3.9 and higher only:
+      #response = response.removesuffix(end)
+   return response + ".ttf"
 
 def get_session():
    session = requests.Session()
    session.headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36"
    return session
 
-def list_all_css_for_page(url, session = None, debuglevel = debuglevel, dryrun=False):
+def list_all_css_for_page(url, config):
    """
    Return all css links from a given page
    """
    # Reference: https://www.thepythoncode.com/article/extract-web-page-script-and-css-files-in-python
    css_files = []
-   if not session:
-      session = get_session()
-   html = session.get(url).content
+   html = config.session.get(url).content
    soup = bs(html, "html.parser")
    for css in soup.find_all("link"):
       if ".css" in css.attrs.get("href"):
          # if the link tag has the 'href' attribute
          css_url = urljoin(url, css.attrs.get("href"))
-         if debuglevel >= 8:
+         if config.debuglevel >= 8:
             eprint(f"Found css: {css_url}")
          css_files.append(css_url)
    return css_files
 
-def get_webfonts_for_one_css(url, session = None, debuglevel = debuglevel, dryrun=False):
+def get_webfonts_for_one_css(url, config):
    """
    Return a list of urls of all webfonts specified in this css file
    """
    #theseFonts = []
-   if not session:
-      session = get_session()
-   css = session.get(url).content
+   css = config.session.get(url).content
    a = tinycss2.parse_stylesheet_bytes(css)
    a = a[0]
    b = []
@@ -76,7 +104,7 @@ def get_webfonts_for_one_css(url, session = None, debuglevel = debuglevel, dryru
       try:
          if "at-rule" in i.type and "font-face" in i.at_keyword:
             b.append(i)
-            if debuglevel >= 10:
+            if config.debuglevel >= 10:
                eprint(str(x) + " " + str(i))
       except:
          pass
@@ -91,13 +119,13 @@ def get_webfonts_for_one_css(url, session = None, debuglevel = debuglevel, dryru
             # make absolute from relative
             thisurl = urljoin(url,j.value)
             if thisurl not in c:
-               if debuglevel >= 5:
+               if config.debuglevel >= 5:
                   eprint(f"get_webfonts_for_one_css: Found font url {thisurl}")
                c.append(thisurl)
    # c is a flat list of all font files, many of which are duplicates
    return c
 
-def save_font(url,destdir,session=None, debuglevel = debuglevel, dryrun=False, convert=False):
+def save_font(url,destdir,config):
    """
    Given a url, and destination dir, and optionally an existing http session, download the url and save to a file. If convert, save any woff/woff2 to ttf.
    """
@@ -108,8 +136,8 @@ def save_font(url,destdir,session=None, debuglevel = debuglevel, dryrun=False, c
    filename=""
    filename=os.path.basename(urlparse(url).path)
    ext = os.path.splitext(filename)[-1]
-   # Do not try to convert .eot
-   if convert and not filename.endswith(".ttf") and ext not in [".eot"]:
+   # Do not try to convert .svg
+   if config.convert and not filename.endswith(".ttf") and ext not in [".svg"]:
       need_convert = True
       orig_filename = filename # in case we cannot load library later
       filename = ttfify_filename(filename)
@@ -119,14 +147,11 @@ def save_font(url,destdir,session=None, debuglevel = debuglevel, dryrun=False, c
       if url.startswith("data:"):
          # not supported!
          # WORKHERE: support saving to a tempfile this datastream, probably a base64encoded woff file. Then just convert.
-         eprint(f"Warning: Url {url[:MAX_STRING_PRINT_LENGTH]} is unsupported.")
+         eprint(f"Warning: Url {url[:config.MAX_STRING_PRINT_LENGTH]} is unsupported.")
       else:
-         if not dryrun:
+         if not config.dryrun:
          # Download content
-            if session:
-               response = session.get(url)
-            else:
-               response = requests.get(url)
+            response = config.session.get(url)
 
             if 'Content-Disposition' in response.headers:
                filename=response.headers['Content-Disposition']
@@ -136,47 +161,53 @@ def save_font(url,destdir,session=None, debuglevel = debuglevel, dryrun=False, c
                   filename = ttfify_filename(filename)
                filepath = os.path.join(destdir, filename)
 
-         # Future: logic for woff2 to ttf conversion goes here, approximately
-
          try:
-            if debuglevel >= 1:
-               sstring = "Saving" if not dryrun else "Save"
+            if config.debuglevel >= 1:
+               sstring = "Saving" if not config.dryrun else "Save"
                eprint(f"{sstring} {url} to file {filepath}")
-            if not dryrun:
+            if not config.dryrun:
                if not need_convert:
                   with open(filepath,'wb') as thisfile:
                      thisfile.write(response.content)
                else:
                   # need_convert is true, and not dryrun, so call function
-                  try:
-                     from fontTools import ttLib
-                  except Exception as e:
-                     raise e
-                  convert_font(url,filepath,session=session,debuglevel=debuglevel,dryrun=dryrun)
+                  if ext in [".woff",".woff2"]:
+                     try:
+                        from fontTools import ttLib
+                     except Exception as e:
+                        raise e
+                     convert_woffwoff2_ttf(url,filepath,config=config)
+                  elif ext in [".eot"]:
+                     convert_eot_ttf(url,filepath,config=config)
+                  else:
+                     # no plan for conversion!
+                     eprint(f"Warning: no conversion plan for ext {ext} of {url}. Saving as-is.")
+                     with open(filepath,'wb') as thisfile:
+                        thisfile.write(response.content)
             return 0
          except Exception as E:
             eprint(f"Error when downloading {url}, {E}")
             return -1
    else: # filepath does exist
-      if debuglevel >= 2:
+      if config.debuglevel >= 2:
          eprint(f"File {filepath} exists for {url}. Skipping.")
       return 0
 
-def get_all_fonts_from_csslist(all_css, session=None, debuglevel=debuglevel, dryrun=False):
+def get_all_fonts_from_csslist(all_css, config):
    all_fonts = []
    for this_css in all_css:
-      webfonts = get_webfonts_for_one_css(this_css, session, debuglevel=debuglevel, dryrun=dryrun)
+      webfonts = get_webfonts_for_one_css(this_css, config)
       for webfont in webfonts:
          # filter accepted extensions here. Technically fontconfig only uses ttf.
          # Always exclude svg, because those are really big, and not usable files for fontconfig.
-         # WORKHERE: allow svg, if convert_font works on svg.
+         # WORKHERE: allow svg, if convert_woffwoff2_ttf works on svg.
          if webfont not in all_fonts and '.svg' not in webfont:
-            if debuglevel >= 2:
+            if config.debuglevel >= 2:
                eprint(f"Found font {webfont}")
             all_fonts.append(webfont)
    return all_fonts
 
-def save_all_fonts(all_fonts, destdir, session=None, debuglevel=debuglevel, dryrun=False, convert=False):
+def save_all_fonts(all_fonts, destdir, config):
    """
    Given a list of font urls, and the destdir, save all these fonts
    """
@@ -189,7 +220,7 @@ def save_all_fonts(all_fonts, destdir, session=None, debuglevel=debuglevel, dryr
          raise NotADirectoryError(20,destdir,"Please clean up this non-directory file and try again")
          return -1
    try:
-      if not dryrun:
+      if not config.dryrun:
          os.mkdir(destdir)
    except FileExistsError:
       pass # it already exists
@@ -198,48 +229,44 @@ def save_all_fonts(all_fonts, destdir, session=None, debuglevel=debuglevel, dryr
 
    # Loop through all webfont files and save them
    for font in all_fonts:
-      save_font(font, destdir, session=session, debuglevel=debuglevel, dryrun=dryrun, convert=convert)
+      save_font(font, destdir, config)
    return 0
 
-def whitelist_page(url, fontdir, session=None, debuglevel=debuglevel, dryrun=False, convert = False):
+def whitelist_page(url, fontdir, config):
    """
    For the given URL, Save all listed webfonts to a directory named
    after the domain, underneath the given fontdir. If convert, then
    convert all woff, woff2 files to ttf using woffTools
    """
    all_fonts = []
-   if not session:
-      session = get_session()
 
    # List all webfonts called by the given page
-   all_css = list_all_css_for_page(url, session, debuglevel=debuglevel, dryrun=dryrun)
-   all_fonts = get_all_fonts_from_csslist(all_css, session, debuglevel=debuglevel, dryrun=dryrun)
+   all_css = list_all_css_for_page(url, config)
+   all_fonts = get_all_fonts_from_csslist(all_css, config)
 
    # Prepare destination dir
    destdir = os.path.join(fontdir,urlparse(url).netloc)
 
    # Save all fonts to that dir
-   return save_all_fonts(all_fonts, destdir, session, debuglevel=debuglevel, dryrun=dryrun, convert=convert)
+   return save_all_fonts(all_fonts, destdir, config)
 
-def whitelist_harfile(harfile, fontdir, session=None, debuglevel=debuglevel, dryrun=False, convert=False):
+def whitelist_harfile(harfile, fontdir, config):
    """
    Given the harfile, save all fonts listed in the discovered css files
    """
    all_fonts = []
-   if not session:
-      session = get_session()
 
    # List all css in the har file
-   all_css = extract_css_urls_from_harfile(harfile)
-   all_fonts = get_all_fonts_from_csslist(all_css, session, debuglevel=debuglevel, dryrun=dryrun)
+   all_css = extract_css_urls_from_harfile(harfile, config)
+   all_fonts = get_all_fonts_from_csslist(all_css, config)
 
    # Prepare destination dir
    destdir = os.path.join(fontdir,"harfiles")
 
    # Save all fonts to that dir
-   return save_all_fonts(all_fonts, destdir, session, debuglevel=debuglevel, dryrun=dryrun, convert=convert)
+   return save_all_fonts(all_fonts, destdir, config)
 
-def extract_css_urls_from_harfile(harfile):
+def extract_css_urls_from_harfile(harfile, config):
    """
    Extract all urls that match string "css" from a har file
    """
@@ -254,21 +281,18 @@ def extract_css_urls_from_harfile(harfile):
    for d in c:
       e = c[x]['request']['url']
       if "css" in e and e not in css_files:
-         if debuglevel >= 5:
+         if config.debuglevel >= 5:
             eprint(e)
          css_files.append(e)
       x = x + 1
    return css_files
 
-def convert_font(url, filename, session=None, debuglevel=debuglevel, dryrun=False):
+def convert_woffwoff2_ttf(url, filename, config):
    """
    Save the given url to filename, with filetype ttf
    """
    # This will only be called from save_font when dryrun=False, so the dryrun flag here is useful only if called from some other usage.
-   if session:
-      response = session.get(url)
-   else:
-      response = requests.get(url)
+   response = config.session.get(url)
    try:
       from fontTools import ttLib
    except ModuleNotFoundError:
@@ -280,10 +304,54 @@ def convert_font(url, filename, session=None, debuglevel=debuglevel, dryrun=Fals
    with tempfile.TemporaryFile() as tf:
       tf.write(response.content)
       font = ttLib.TTFont(tf)
-   if debuglevel >= 3:
-      eprint(f"Converting {url[:MAX_STRING_PRINT_LENGTH]} from {font.flavor} to ttf as file {filename}")
+   if config.debuglevel >= 3:
+      eprint(f"Converting {url[:config.MAX_STRING_PRINT_LENGTH]} from {font.flavor} to ttf as file {filename}")
 
    font.flavor = None # restores default value, for non-compressed OpenType
    font.save(filename)
 
    return 0
+
+def convert_eot_ttf(uri, filename, config):
+   """
+   Save the given uri of an eot file to filename, with filetype ttf
+   """
+   # This will only be called from save_font when dryrun=False, so the dryrun flag here is useful only if called from some other usage.
+   tf = None
+   if "http://" in uri or "https://" in uri or "ftp://" in uri:
+      response = config.session.get(uri)
+      content = response.content
+      tf = tempfile.NamedTemporaryFile()
+      tf.write(content)
+      infile = tf.name # change to use this temp file
+   else:
+      # local file, or some uri scheme not planned yet
+      infile = uri
+      #with open(uri,'rb') as o:
+      #   content = o.read()
+   try:
+      r = subprocess.run(["which",config.eot2ttf_binary], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+      if r.returncode != 0:
+         #raise FileNotFoundError(18,"eot2ttf","cannot find")
+         eprint(f"Warning: Cannot convert {uri} because cannot find eot2ttf. Please set --eotbin.")
+         return -1
+      # so proceed
+   except Exception as e:
+      try: # clean up temp file
+         if tf:
+            tf.close()
+      except:
+         pass
+      raise e
+
+   r=subprocess.run([config.eot2ttf_binary,infile,filename])
+   if r.returncode != 0:
+      eprint(f"Warning: eot2ttf failed on {uri}")
+
+   # exit convert_eot_ttf
+   try: # clean up temp file
+      if tf:
+         tf.close()
+   except:
+      pass
+   return 0