Knowledge Base

Preserving for the future: Shell scripts, AoC, and more

Symlink forest generator

bgstack15

Comments

As part of my Web gallery solution, I wrote some logic that creates a YYYY/MM directory creator that places symlinks to my original files within the new directory structure. The links are added to the relevant directory based on exif metadata if possible, and file timestamp as a last resort. Check out the source files at my gitlab space:

Here is my main sample invocation. I used this a lot while testing.

./generate.py -i /mnt/bgstack15/Backups/Images/Photos/camera/2018/ -o /mnt/public/www/gallery/my2018 -n -d2 -x 'October' -s --nr

The help text is generated with ArgumentParser.

$ ./generate.py --helpusage: generate.py [-h] [-d [{0,1,2,3,4,5,6,7,8,9,10}]] [-v] [-n | -a] [-z | --nz] [-c | -s | -m] [-r | --nr] -i
                   INDIR -o OUTDIR [-x EXCLUDE]

Make symlink forest for images

optional arguments:
  -h, --help            show this help message and exit
  -d [{0,1,2,3,4,5,6,7,8,9,10}], --debug [{0,1,2,3,4,5,6,7,8,9,10}]
                        Set debug level
  -v, --version         show program's version number and exit
  -n, --dryrun          Make no changes (default)
  -a, --apply           Actually make changes
  -z, -0, --zeropad     Zero pad month directories (default)
  --nz, --nozeropad, --no-zeropad
                        Do not zero pad
  -c, --copy            Copy files instead of symlinks. Not recommended.
  -s, --symlink         Make symlinks (default)
  -m, --move            Move files. Not recommended.
  -r, --relative        Make relative symlinks if possible.
  --nr, --norelative, --no-relative
                        Make absolute symlinks (default)
  -i INDIR, --indir INDIR
  -o OUTDIR, --outdir OUTDIR
  -x EXCLUDE, --exclude EXCLUDE
                        Exclude pathname matches. Can be used multiple times.

For posterity and in case my gitlab ever dies, here are their contents at the time of this writing.

generate.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
#!/usr/bin/env python3
# Startdate: 2021-01-29 15:35
# Dependencies:
#    devuan-req: python3-exifread
# Reference:
#    https://stackoverflow.com/questions/1192978/python-get-relative-path-of-all-files-and-subfolders-in-a-directory
#    https://stackoverflow.com/questions/3207219/how-do-i-list-all-files-of-a-directory
#    https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
from sys import stderr
from exifread import process_file
from os import walk, path, stat, symlink
from pathlib import Path
from datetime import date
from shutil import copy2, move

try:
   if debuglevel is None:
      debuglevel = 0
except:
   debuglevel = 0

# Functions
def eprint(*args, **kwargs):
   print(*args, file=stderr, **kwargs)

def list_files(indir,relative=False,debuglevel=debuglevel, excludes=None):
   x = 0
   f = []
   if debuglevel >= 9:
      eprint(f"Listing all files underneath {indir}")
      if len(excludes) > 0:
         eprint("While excluding path matches:",excludes)
   for (dirpath, dirnames, filenames) in walk(indir):
      for file in filenames:
         x += 1
         fullpath=path.join(dirpath,file)
         relpath=path.join(path.relpath(dirpath,indir),file)
         use = True
         use_pattern = ""
         for e in excludes:
            # simple match, no regex
            if e in relpath:
               use = False
               use_pattern = e
               break # short-circuit
         if use:
            if debuglevel >= 9: eprint(x,fullpath,relpath)
            f.append(relpath if relative else fullpath)
         else:
            if debuglevel >= 9: eprint(x,fullpath,f"ignored per {use_pattern}")
   return f

def limit(listf=None,filters=["image","video"]):
   # available types are "image", "video"
   newlist = []
   for f in listf:
      _, ext = path.splitext(f)
      ext = ext[1:].lower()
      if ext in ["jpg","jpeg","png","svg","tif","tiff","gif"] and "image" in filters:
         newlist.append(f)
      elif ext in ["mp4","webm","avi"] and "video" in filters:
         newlist.append(f)
   return newlist

def get_file_YM(tf,debuglevel=debuglevel,zero_pad=False):
   Y = 0 ; M = 0
   #eprint(f"Debuglevel {debuglevel}")
   try:
      # read exif data
      with open(tf,'rb') as f:
         tags = process_file(f, details=False, stop_tag="Image DateTime")
      if "Image DateTime" in tags:
         #if debuglevel >= 5: eprint(tf,tags["Image DateTime"])
         Y = str(tags["Image DateTime"]).split(' ')[0].split(':')
         if Y == "['']" or Y == ['']:
            Y = ['0','0']
         M = Y[1] ; Y = Y[0];
      # Any other image timestamps could be used here
      #eprint(tags)
   except KeyboardInterrupt:
      return -1, -1
   except:
      if debuglevel >= 1: eprint(f"Unable to extract any exif data for {tf}")
   if int(Y) == 0 or int(M) == 0:
      # need to just use timestamp
      try:
         ts = date.fromtimestamp(stat(tf).st_mtime)
      except KeyboardInterrupt:
         return -1, -1
      except:
         # no timestamp available from filesystem?
         if debuglevel >= 1: eprint(f"No timestamp available for {tf}")
      if int(Y) == 0: Y = ts.year
      if int(M) == 0: M = ts.month
   if zero_pad:
      M = str(M).zfill(2)
   else:
      M = str(int(M))
   if debuglevel >= 3:
      eprint(f"{tf} {Y}/{M}")
   return Y, M

def make_YM_forest(outdir, flist, action = "symlink", dryrun=True, debuglevel=debuglevel, zero_pad=False, relative_symlinks = False):
   """
   For each file in flist, [action] it to outdir, sorting into YYYY/MM subdirs based on exif metadata or timestamp.
   Action may be one of ['symlink', 'copy', 'move'].
   If relative_symlink = True, then make relative symlinks if possible.
   """
   result = 0
   stop = False

   # validate input
   if action not in ['symlink', 'copy', 'move']:
      eprint("make_YM_forest action may be one of ['symlink', 'copy', 'move'].")
      return -1

   # Learn all directories to make
   # and also cache the files with Y/M value
   fdict = {}
   destdirs = []
   for f in flist:
      try:
         Y, M = get_file_YM(f,debuglevel=debuglevel,zero_pad=zero_pad)
         if Y == -1 and M == -1:
            #eprint("Stopping due to keyboard variant 2.")
            stop = True
            break
      except KeyboardInterrupt:
         #eprint("Stopping due to keyboard variant 1.")
         stop = True
         break
      if zero_pad:
         M = str(M).zfill(2)
      else:
         M = str(int(M))
      YM = f"{Y}/{M}"
      if YM not in destdirs: destdirs.append(YM)
      fdict[f] = YM
   # finish the for f in flist

   # short-circuit if keyboard action happened in one of the nested functions
   if stop:
      return -1

   # Make directories
   for d in destdirs:
      dd = path.join(outdir,d)
      if debuglevel >= 3:
         eprint(f"Make dir: {dd}")
      if not dryrun:
         Path(dd).mkdir(parents=True, exist_ok=True)

   # Make symlinks
   for f in fdict:
      ff = f
      basename = path.basename(ff)
      destfile = path.join(outdir,fdict[f],basename)
      if action == "symlink" and relative_symlinks:
         ff = path.relpath(ff,destfile)
      if debuglevel >= 2:
         if action == "copy":
            eprint(f"cp -p {ff} {destfile}")
         elif action == "symlink":
            eprint(f"ln -s {ff} {destfile}")
         elif action == "move":
            eprint(f"mv {ff} {destfile}")

      if not dryrun:
      #if False:
         if action == "copy":
            copy2(ff,destfile)
         elif action == "symlink":
            symlink(ff,destfile)
         elif action == "move":
            move(ff,destfile)

   return result

genlib.py

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
 
#!/usr/bin/env python3
# Startdate: 2021-01-29 15:35
# Dependencies:
#    devuan-req: python3-exifread
# Reference:
#    https://stackoverflow.com/questions/1192978/python-get-relative-path-of-all-files-and-subfolders-in-a-directory
#    https://stackoverflow.com/questions/3207219/how-do-i-list-all-files-of-a-directory
#    https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python
from sys import stderr
from exifread import process_file
from os import walk, path, stat, symlink
from pathlib import Path
from datetime import date
from shutil import copy2, move

try:
   if debuglevel is None:
      debuglevel = 0
except:
   debuglevel = 0

# Functions
def eprint(*args, **kwargs):
   print(*args, file=stderr, **kwargs)

def list_files(indir,relative=False,debuglevel=debuglevel, excludes=None):
   x = 0
   f = []
   if debuglevel >= 9:
      eprint(f"Listing all files underneath {indir}")
      if len(excludes) > 0:
         eprint("While excluding path matches:",excludes)
   for (dirpath, dirnames, filenames) in walk(indir):
      for file in filenames:
         x += 1
         fullpath=path.join(dirpath,file)
         relpath=path.join(path.relpath(dirpath,indir),file)
         use = True
         use_pattern = ""
         for e in excludes:
            # simple match, no regex
            if e in relpath:
               use = False
               use_pattern = e
               break # short-circuit
         if use:
            if debuglevel >= 9: eprint(x,fullpath,relpath)
            f.append(relpath if relative else fullpath)
         else:
            if debuglevel >= 9: eprint(x,fullpath,f"ignored per {use_pattern}")
   return f

def limit(listf=None,filters=["image","video"]):
   # available types are "image", "video"
   newlist = []
   for f in listf:
      _, ext = path.splitext(f)
      ext = ext[1:].lower()
      if ext in ["jpg","jpeg","png","svg","tif","tiff","gif"] and "image" in filters:
         newlist.append(f)
      elif ext in ["mp4","webm","avi"] and "video" in filters:
         newlist.append(f)
   return newlist

def get_file_YM(tf,debuglevel=debuglevel,zero_pad=False):
   Y = 0 ; M = 0
   #eprint(f"Debuglevel {debuglevel}")
   try:
      # read exif data
      with open(tf,'rb') as f:
         tags = process_file(f, details=False, stop_tag="Image DateTime")
      if "Image DateTime" in tags:
         #if debuglevel >= 5: eprint(tf,tags["Image DateTime"])
         Y = str(tags["Image DateTime"]).split(' ')[0].split(':')
         if Y == "['']" or Y == ['']:
            Y = ['0','0']
         M = Y[1] ; Y = Y[0];
      # Any other image timestamps could be used here
      #eprint(tags)
   except KeyboardInterrupt:
      return -1, -1
   except:
      if debuglevel >= 1: eprint(f"Unable to extract any exif data for {tf}")
   if int(Y) == 0 or int(M) == 0:
      # need to just use timestamp
      try:
         ts = date.fromtimestamp(stat(tf).st_mtime)
      except KeyboardInterrupt:
         return -1, -1
      except:
         # no timestamp available from filesystem?
         if debuglevel >= 1: eprint(f"No timestamp available for {tf}")
      if int(Y) == 0: Y = ts.year
      if int(M) == 0: M = ts.month
   if zero_pad:
      M = str(M).zfill(2)
   else:
      M = str(int(M))
   if debuglevel >= 3:
      eprint(f"{tf} {Y}/{M}")
   return Y, M

def make_YM_forest(outdir, flist, action = "symlink", dryrun=True, debuglevel=debuglevel, zero_pad=False, relative_symlinks = False):
   """
   For each file in flist, [action] it to outdir, sorting into YYYY/MM subdirs based on exif metadata or timestamp.
   Action may be one of ['symlink', 'copy', 'move'].
   If relative_symlink = True, then make relative symlinks if possible.
   """
   result = 0
   stop = False

   # validate input
   if action not in ['symlink', 'copy', 'move']:
      eprint("make_YM_forest action may be one of ['symlink', 'copy', 'move'].")
      return -1

   # Learn all directories to make
   # and also cache the files with Y/M value
   fdict = {}
   destdirs = []
   for f in flist:
      try:
         Y, M = get_file_YM(f,debuglevel=debuglevel,zero_pad=zero_pad)
         if Y == -1 and M == -1:
            #eprint("Stopping due to keyboard variant 2.")
            stop = True
            break
      except KeyboardInterrupt:
         #eprint("Stopping due to keyboard variant 1.")
         stop = True
         break
      if zero_pad:
         M = str(M).zfill(2)
      else:
         M = str(int(M))
      YM = f"{Y}/{M}"
      if YM not in destdirs: destdirs.append(YM)
      fdict[f] = YM
   # finish the for f in flist

   # short-circuit if keyboard action happened in one of the nested functions
   if stop:
      return -1

   # Make directories
   for d in destdirs:
      dd = path.join(outdir,d)
      if debuglevel >= 3:
         eprint(f"Make dir: {dd}")
      if not dryrun:
         Path(dd).mkdir(parents=True, exist_ok=True)

   # Make symlinks
   for f in fdict:
      ff = f
      basename = path.basename(ff)
      destfile = path.join(outdir,fdict[f],basename)
      if action == "symlink" and relative_symlinks:
         ff = path.relpath(ff,destfile)
      if debuglevel >= 2:
         if action == "copy":
            eprint(f"cp -p {ff} {destfile}")
         elif action == "symlink":
            eprint(f"ln -s {ff} {destfile}")
         elif action == "move":
            eprint(f"mv {ff} {destfile}")

      if not dryrun:
      #if False:
         if action == "copy":
            copy2(ff,destfile)
         elif action == "symlink":
            symlink(ff,destfile)
         elif action == "move":
            move(ff,destfile)

   return result

Comments