path: root/vooblystats.sh

#!/bin/sh
# Filename: vooblystats.sh
# License: CC-BY-SA 4.0
# Author: bgstack15@gmail.com
# Startdate: 2020-01-24 08:57:56
# Title: 
# Purpose: 
# Package: 
# History: 
# Usage: 
# Reference: ftemplate.sh 2019-05-02a ; framework.sh 2018-05-02a
# Improve:
#    write a separate, no-auth-needed tool that scrapes https://www.voobly.com/games/view/Age-of-Empires-II-The-Conquerors for lobby attendance, and put it in cronjob.
#    add debuglev ferror stuff
# Dependencies:
#    framework.sh
fiversion="2019-05-02a"
vooblystatsversion="2020-01-28a"

usage() {
   ${PAGER:-/usr/bin/less -F} >&2 <<ENDUSAGE
usage: vooblystats.sh [-duV] [-c conffile]
Demo script for pulling stats from voobly.com
version ${vooblystatsversion}
 -d debug   Show debugging info, including parsed variables.
 -u usage   Show this usage block.
 -V version Show script version number.
 -c conf    Read in this config file.
Return values:
 0 Normal
 1 Help or version info displayed
 2 Count or type of flaglessvals is incorrect
 3 Incorrect OS type
 4 Unable to find dependency
 5 Not run as root or sudo
ENDUSAGE
}

# DEFINE FUNCTIONS

make_voobly_request() {
   # call: make_voobly_request "${url}" "${referer}" "${usecookiefilebool}" "${cookiefile}" "${usedatabool}" "${data}" "{includeheadersbool}"
   ___mvr_url="${1}"
   ___mvr_referer="${2}"
   ___mvr_usecf="${3}"
   ___mvr_cf="${4}"
   ___mvr_used="${5}"
   ___mvr_data="${6}"
   ___mvr_includeh="${7}"

   ___mvr_host="$( echo "${___mvr_url}" | awk -F'/' '{print $3}' )"
   ___mvr_flags_cf=""
   fistruthy "${___mvr_usecf}" && ___mvr_flags_cf="-b ${VS_COOKIEFILE} -c ${VS_COOKIEFILE}"
   ___mvr_flags_data=""
   fistruthy "${___mvr_used}" && ___mvr_flags_data="--data ${___mvr_data}"
   ___mvr_flags_headers=""
   fistruthy "${___mvr_includeh}" && ___mvr_flags_headers="-i"
   curl "${___mvr_url}" \
      ${___mvr_flags_headers} -s ${___mvr_flags_cf} \
      -H "Host: ${___mvr_host}" \
      -H 'User-Agent: Mozilla/5.0 (Windows NT6.3; Win64; x64) ApplieWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36' \
      -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \
      -H 'Accept-Language: en-US,en;q=0.5' --compressed \
      -H "Referer: ${___mvr_referer:-${___mvr_url}}" \
      -H 'Connection: keep-alive' ${___mvr_flags_data}
   fistruthy "${___mvr_includeh}" && cat "${___mvr_cf}" 2>/dev/null

}

html_encode() {
   # call: safevar="$( html_encode "unsafe;string" )"
   # reference:
   # https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command/10797966#10797966
   ___he_input="${1}"
   echo "${___he_input}" | curl -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | sed -E 's/..(.*).../\1/'
}

auth_to_voobly() {
   # call: auth_to_voobly "${username}" "${password}" "${cookiefile}"
   ___atv_username="${1}"
   ___atv_password="${2}"
   ___atv_cookiefile="${3}"
   # transform password to html-safe
   ___atv_password_safe="$( html_encode "${___atv_password}" )"
   make_voobly_request "https://www.voobly.com/" "https://wwww.voobly.com" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response1"
   make_voobly_request "https://www.voobly.com/login" "https://wwww.voobly.com/" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response2"
   make_voobly_request "https://www.voobly.com/login/auth" "https://wwww.voobly.com/login" "true" "${___atv_cookiefile}" "true" "username=${___atv_username}&password=${___atv_password_safe}" "true" 1>"${VS_TMPDIR}/auth-response3"
   make_voobly_request "https://www.voobly.com/welcome" "https://wwww.voobly.com/login" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response4"
   make_voobly_request "https://www.voobly.com/profile/view/123989133" "https://wwww.voobly.com/welcome" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response5"
}

parse_game_page() {
   # call: parse_game_page "${inputfileORurl}" "${cookiefile}"
   # output: to stdout, a csv that originally was planned to resemble:
   # GAME,gameid,dateplayed,maptype,duration,playercount,gamemod
   # PLAYER,playerid,gameid,playercountnum,playername,playercountry,playernewrating,playerpoints,playerteam,playerciv,playerwonbool,militaryscore,economyscore,technologyscore,societyscore,totalscore,unitskilled,unitslost,buildingsrazed,buildingslost,unitsconverted,food,wood,stone,gold,tradeprofit,tributerec,tributesent,feudaltime,castletime,imptime,mapexplored,researchcount,researchpercent,totalwonders,totalcastles,reliccount,relicgold,villagerhigh
   ___pgp_infile="${1}"
   ___pgp_cf="${2}"
   if test "${___pgp_infile}" = "stdin" ;
   then
      fullpage="$( cat )"
   else
      # check if it contains a URL
      if echo "${___pgp_infile}" | grep -qiE '^https?:\/\/' ;
      then
         # need to retrieve it
         fullpage="$( make_voobly_request "${___pgp_infile}" "https://www.voobly.com/welcome" "true" "${___pgp_cf:-${VS_COOKIEFILE}}" "false" "" "false" )"
      else
         fullpage="$( cat "${___pgp_infile}" 2>/dev/null )"
      fi
   fi

   # test if valid page
   if ! echo "${fullpage}" | grep -qiE "economy score" ;
   then
      # if the page does not contain the expression "economy score" then it is invalid
      ferror "${scriptfile}: cannot parse game page ${___pgp_infile}. Skipping..."
      return
   fi

   # game info
   gameid="$( echo "${fullpage}" | _get_simple_value "Match Details" | tr -d '#' )"
   _reldate="$( echo "${fullpage}" | _get_simple_value "Date Played:" )"
   absolute_date="$( date -d "$( echo "${_reldate}" | tr -d ',-' )" -u "+%FT%TZ" )"
   map="$( echo "${fullpage}" | _get_simple_value "Map:" )"
   duration="$( echo "${fullpage}" | _get_simple_value "Duration:" )"
   playercount="$( echo "${fullpage}" | _get_simple_value "Players:" )"
   gamemod="$( echo "${fullpage}" | _get_simple_value "Game Mod:" )"
   gameladder="$( echo "${fullpage}" | awk '/Ladder:/' | grep -oE ">.*<" | tail -c +2 | head -c -2 )"

   echo "GAME,${gameid},${absolute_date},${map},${duration},${gameladder},${playercount},${gamemod}," # HEADERCSV

   # per player info
   x=0
   while test $x -lt $playercount ;
   do
      x=$(( x + 1 ))
      _playerline="$( echo "${fullpage}" | grep -E "^<table.*profile\/view\/[0-9]{1,15}.*table>$" | sed -n "${x}p" )"
      _playerline_num="$( echo "${fullpage}" | grep -n -E "^<table.*profile\/view" | awk -F':' '{print $1}' | sed -n "${x}p" )"
      _playerline_num=$(( _playerline_num + 1 ))
      _playerline2="$( echo "${fullpage}" | sed -n "${_playerline_num}p" )"
      playerid="$( echo "${_playerline}" | grep -oE "profile\/view/[0-9]{1,15}.?>" | tr -dc '[0-9]' )"
      playername="$( echo "${_playerline}" | grep -oE "profile\/view/[0-9]{1,15}.?>.*<\/a><\/" | sed -r -e 's/<\/a>.*$//;' -e 's/.*view\/[0-9]{1,15}\">//;' )"
      playercountry="$( echo "${_playerline}" | grep -oE "res\/flags/[^\.]{1,20}\.(png|jpg)" | awk -F'/' '{print $NF}' | awk -F'.' '{print $1}' )"
      playernewrating="$( echo "${_playerline2}" | awk -F'[<>]' '{print $5}' )"
      playerpoints="$( echo "${_playerline2}" | awk -F'[<>]' '{print $11}' )"
      playerteam="$( echo "${_playerline2}" | awk -F'[<>]' '{print $17}' )"
      playerwinbool="$( echo "${_playerline}" | grep -qiE 'win.PNG' && echo "true" || echo "false" )"
      playerpremiumbool="$( echo "${_playerline}" | grep -qiE 'user-premium' && echo "true" || echo "false" )"
      playergamemvpbool="$( echo "${_playerline}" | grep -qiE 'high.PNG' && echo "true" || echo "false" )"
      playerclan="$( echo "${_playerline}" | grep -oE "<a.*\.voobly\.com>.{1,50}<\/a><a.*profile\/view" | sed -r -e 's/<\/a>.*$//' -e 's/^.*\.voobly\.com>//' )"
      # if x > ( numplayers/2), then swap newrating and points
      if test "$( printf '%s;%s>(%s/2);\n' "scale=2" "${x}" "${playercount}" | bc )" = "1" ;
      then
         tempvar="${playernewrating}"
         playernewrating="${playerteam}"
         playerteam="${tempvar}"
         unset tempvar
      fi

      # in-game stats
      _playerstatslines="$( echo "${fullpage}" | grep -A4 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | grep center )"
      # works on webpage game stats sections 1, 2, and 5.
      #echo "${_playerstatslines}" | sed -r -e 's/(<[^\>]+>)+/@/g;'| tr -d ',%' | awk -F'@' 'BEGIN{OFS="\t"} {print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10}'
      _playerstatslines2="$( echo "${_playerstatslines}" | sed -r -e 's/(<[^\>]+>)+/@/g;' | tr -d ',%' | awk -F'@' '{print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10}' )"
      _playerstatslines3="$( echo "${_playerstatslines2}" | tr -d '\n' | sed -r -e 's/\s+/ /g;' )" 

      playerciv="$( echo "${fullpage}" | grep -B4 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | head -n1 | sed -r -e 's/(<[^\>]+>)+//g;' )"
      playercolorhex="$( echo "${fullpage}" | grep -B2 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | head -n1 | tr -d ';' | awk '{print $3}' )"
      # 0054A6 00A651 00FFFF 92278F C0C0C0 FF0000 FF8000 FFFF00
      playercolorname="nocolor"
      case "${playercolorhex###}" in
         0054A6) playercolorname="blue" ;;
         FF0000) playercolorname="red" ;;
         FFFF00) playercolorname="yellow" ;;
         00A651) playercolorname="green" ;;
         00FFFF) playercolorname="cyan" ;;
         92278F) playercolorname="purple" ;;
         C0C0C0) playercolorname="gray" ;;
         FF8000) playercolorname="orange" ;;
      esac

      # and because bash read is broken on devuan for an uknown reason... we have to use awk
      milscore="$( echo "${_playerstatslines3}" | awk '{print $1}' )"
      ecoscore="$( echo "${_playerstatslines3}" | awk '{print $2}' )"
      techscore="$( echo "${_playerstatslines3}" | awk '{print $3}' )"
      socscore="$( echo "${_playerstatslines3}" | awk '{print $4}' )"
      totscore="$( echo "${_playerstatslines3}" | awk '{print $5}' )"
      unitskilled="$( echo "${_playerstatslines3}" | awk '{print $6}' )"
      unitslost="$( echo "${_playerstatslines3}" | awk '{print $7}' )"
      buildingsrazed="$( echo "${_playerstatslines3}" | awk '{print $8}' )"
      buildingslost="$( echo "${_playerstatslines3}" | awk '{print $9}' )"
      unitsconverted="$( echo "${_playerstatslines3}" | awk '{print $10}' )"
      food="$( echo "${_playerstatslines3}" | awk '{print $11}' )"
      wood="$( echo "${_playerstatslines3}" | awk '{print $12}' )"
      stone="$( echo "${_playerstatslines3}" | awk '{print $13}' )"
      gold="$( echo "${_playerstatslines3}" | awk '{print $14}' )"
      tradeprofit="$( echo "${_playerstatslines3}" | awk '{print $15}' )"
      tributerec="$( echo "${_playerstatslines3}" | awk '{print $16}' )"
      tributesent="$( echo "${_playerstatslines3}" | awk '{print $17}' )"
      feudaltime="$( echo "${_playerstatslines3}" | awk '{print $18}' )"
      castletime="$( echo "${_playerstatslines3}" | awk '{print $19}' )"
      imptime="$( echo "${_playerstatslines3}" | awk '{print $20}' )"
      mapexplored="$( echo "${_playerstatslines3}" | awk '{print $21}' )"
      researchcount="$( echo "${_playerstatslines3}" | awk '{print $22}' )"
      researchpercent="$( echo "${_playerstatslines3}" | awk '{print $23}' )"
      totalwonders="$( echo "${_playerstatslines3}" | awk '{print $24}' )"
      totalcastles="$( echo "${_playerstatslines3}" | awk '{print $25}' )"
      reliccount="$( echo "${_playerstatslines3}" | awk '{print $26}' )"
      relicgold="$( echo "${_playerstatslines3}" | awk '{print $27}' )"
      villagerhigh="$( echo "${_playerstatslines3}" | awk '{print $28}' )"

      # end of player loop
      echo "PLAYER,${playerid},${gameid},${x},${playername},${playercolorhex},${playercolorname},${playercountry},${playerpremiumbool},${playerclan},${playernewrating},${playerpoints},${playerteam},${playerciv},${playerwinbool},${milscore},${ecoscore},${techscore},${socscore},${totscore},${unitskilled},${unitslost},${buildingsrazed},${buildingslost},${unitsconverted},${food},${wood},${stone},${gold},${tradeprofit},${tributerec},${tributesent},${feudaltime},${castletime},${imptime},${mapexplored},${researchcount},${researchpercent},${totalwonders},${totalcastles},${reliccount},${relicgold},${villagerhigh}," # HEADERCSV

      unset -v playerclan _playerline _playerline2 _playerline_num playerid playername playercountry playernewrating playerpoints
   done

} # end of parse_game_page

parse_profile_matchlist_page() {
   # call: parse_profile_matchlist_page "${inputfileORurl}" "${cookiefile}"
   # output: delimited list of match numbers found on the profile page.
   # reference: parse_game_page
   ___ppmp_infile="${1}"
   ___ppmp_cf="${2}"
   if test "${___ppmp_infile}" = "stdin" ;
   then
      fullpage="$( cat )"
   else
      # check if it contains a URL
      if echo "${___ppmp_infile}" | grep -qiE '^https?:\/\/' ;
      then
         # need to retrieve it
         fullpage="$( make_voobly_request "${___ppmp_infile}" "https://www.voobly.com/welcome" "true" "${___ppmp_cf:-${VS_COOKIEFILE}}" "false" "" "false" )"
      else
         fullpage="$( cat "${___ppmp_infile}" 2>/dev/null )"
      fi
   fi

   # test if valid page
   results="$( echo "${fullpage}" | grep -oE "match\/view\/[0-9]+" | awk -F'/' '!x[$3]++ {print $3}' )"
   if test "$( echo "${results}" | wc -l | tr -dc '[0-9]' )" -gt 0 ;
   then
      echo "${results}"
   else
      ferror "${scriptfile}: cannot parse matchlist page ${___ppmp_infile}. Skipping..."
      echo "invalid_matchlist_page"
   fi

} # end parse_profile_matchlist_page

get_match_url() {
   # call: get_match_url "${matchid}"
   ___gmu_gameid="${1}"
   ___gmu_gameurl="https://www.voobly.com/match/view/$( echo "${___gmu_gameid}" | grep -oE "match\/view\/[0-9]+" | awk -F'/' '{print $3}' )"
   echo "${___gmu_gameurl}"
}

get_profile_matchlist_url() {
   # call: get_profile_matchlist_url "123456789" "1" for page 1
   # call: get_profile_matchlist_url "https://www.voobly.com/profile/view/123989133/" 
   # call: get_profile_matchlist_url "https://www.voobly.com/profile/view/123989133/" "1"
   # output: url of page that lists matches, including a page number if requested.
   # we will use a cool shortcut i found, https://www.voobly.com/games/matches/user/123989133/0/0#pagebrowser1
   ___gpmu_url="${1}"
   ___gpmu_pagenum="${2}"

   # if it is already a full page with a pagenumber, just return it
   if echo "${___gpmu_url}" | grep -qiE "\/0\/[0-9]+#pagebrowser" ;
   then
      : # make no changes to it
   else
      if echo "${___gpmu_url}" | grep -qiE "voobly.com" ;
      then
         # derive userid
         ___gpmu_url="$( echo "${___gpmu_url}" | grep -oE "(view|user)\/[0-9]+" | awk -F'/' '{print $2}' )"
      fi
      ___gpmu_url="https://www.voobly.com/games/matches/user/${___gpmu_url}"
      # https://www.voobly.com/games/matches/user/123989133/0/0#pagebrowser1
      # https://www.voobly.com/games/matches/user/123989133/0/1#pagebrowser1
      if test -n "${___gpmu_pagenum}" ;
      then
         ___gpmu_url="${___gpmu_url}/0/${___gpmu_pagenum}#pagebrowser1"
      fi
   fi
   echo "${___gpmu_url}"
}

get_user_matchlist() {
   # call: get_user_matchlist "useridORuserurl" "${mingamecount}"
   # output: on stdout, a list of gameid numbers.
   ___gum_userurl="${1}"
   ___gum_min="${2}" ; test -z "${___gum_min}" && ___gum_min=10
   debuglev 9 && ferror "DEBUG9: get_user_matchlist $*"
   if echo "${___gum_userurl}" | grep -qoE "^[0-9]{1,15}$" ;
   then
      # just userid, so make url
      ___gum_userurl="https://www.voobly.com/profile/view/${___gum_userurl}/"
   fi
   results=""

   # start going through games, starting on page 1
   x=0
   ___gum_continue=0
   ___gum_previous_count=-1
   ___gum_count=0
   while test ${___gum_continue} -eq 0 ;
   do
      x=$(( x + 1 ))
      # get game page
      loopresults="$( parse_profile_matchlist_page "$( get_profile_matchlist_url "${___gum_userurl}" "${x}" )" "${VS_COOKIEFILE}" )"
      debuglev 2 && ferror "DEBUG2: get_user_matchlist: $( echo "${loopresults}" | xargs )"
      results="${results} ${loopresults}"
      ___gum_count="$( echo "${results}" | wc -w )"
      test ${___gum_count} -ge ${___gum_min} && ___gum_continue=1
      # if results have not increased, then we need to stop looping because we are not pulling any more gameids
      test ${___gum_previous_count} -ge ${___gum_count} && ___gum_continue=1
      ___gum_previous_count="${___gum_count}"
   done
   # assume we are good now, where count >= min
   echo "${results}" | xargs -n1
}

get_userid() {
   # call: get_userid "gimli" "${cookiefile}"
   # cookiefile is absolutely necessary for this function!
   # output to stdout: "123456789"
   ___gu_username="${1}"
   ___gu_cf="${2}" ; test -z "${___gu_cf}" && ___gu_cf="${VS_COOKIEFILE}"

   # if the input is already a userid, just return it
   if echo "${___gu_username}" | grep -qiE "^[0-9]+$" ;
   then
      echo "${___gu_username}"
   elif echo "${___gu_username}" | grep -qiE "profile\/view\/[0-9]+" ;
   then
      response="$( echo "${___gu_username}" | grep -oE 'profile\/view\/[0-9]+' | cut -d'/' -f3 | head -n1 )"
      echo "${response}"
   else
      # assume it is just a regular name to parse
      # we need to pass the sessionid in the post data, or else we get an error, even though it is in the cookie.
      sessionid="$( awk '/vbly_session1/{print $NF}' "${___gu_cf}" )"
      response="$( make_voobly_request "https://www.voobly.com/friends/browse/Search/Search" "https://www.voobly.com/Welcome" "true" "${VS_COOKIEFILE}" "true" "query=${___gu_username}&session1=${sessionid}" )"
      # the response will not be a real page; it will just be a redirect notice but it is good enough to get the userid
      if echo "${response}" | grep -qiE "no results found" ;
      then
         ferror "get_userid: unable to find a userid for ${___gu_username}"
         echo "invalid_userid"
      else
         response="$( echo "${response}" | grep -oE 'profile\/view\/[0-9]+' | cut -d'/' -f3 | head -n1 )"
         echo "${response}"
      fi
   fi

}

get_game_page() {
   # call: get_game_page "123456789" "${cookiefile}" | parse_game_page stdin
   # output to stdout: the html of the game page.
   ___ggp_number="${1}"
   ___ggp_cf="${2}"

   # do some validation to make sure this is a valid game number
   ___ggp_number_orig="${___ggp_number}"
   if echo "${___ggp_number}" | grep -qE "voobly\.com" ;
   then
      # somebody passed a whole URL, so let us split it
      ___ggp_number="${___ggp_number##*view/}"
   fi
   ___ggp_number="$( echo "${___ggp_number}" | grep -oE '^[0-9]+' )"

   debuglev 1 && ferror "Trying to read game ${___ggp_number}"
   
   response="$( make_voobly_request "https://www.voobly.com/match/view/${___ggp_number}" "https://www.voobly.com/" "true" "${____ggp_cf}" "no" "" "false" )"
   if echo "${response}" | grep -qiE "economy score" ;
   then
      # is valid game page
      echo "${response}"
   else
      ferror "${scriptfile}: ERROR! Invalid game page request: ${___ggp_number_orig}. Used match number ${___ggp_number}. Skipping..."
      echo "invalid_page_request"
   fi
      
}

_get_simple_value() {
   # call: echo "${fullpage}" | _get_simple_value "Match Details"
   grep -A1 "${1}<\/" | tail -n1 | awk -F'[<>]' '{print $3}'
}

display_csv_headers() {
   # output csv headers.
   #echo "GAME,gameid,absolute_date,map,duration,playercount,gamemod,"
   #echo "PLAYER,playerid,gameid,x,playername,playercolorhex,playercolorname,playercountry,playerpremiumbool,playerclan,playernewrating,playerpoints,playerteam,playerciv,playerwinbool,milscore,ecoscore,techscore,socscore,totscore,unitskilled,unitslost,buildingsrazed,buildingslost,unitsconverted,food,wood,stone,gold,tradeprofit,tributerec,tributesent,feudaltime,castletime,imptime,mapexplored,researchcount,researchpercent,totalwonders,totalcastles,reliccount,relicgold,villagerhigh,"
   sed -n -r -e '/HEADERCSV$/{ s/^.*echo "//;s/"\s*# HEADERCSV$//;s/\$\{//g;s/\}//g;p}' "${scriptfile}"
}

parse_matches_for_user() {
   # call: parse_matches_for_user "gimli" "10" "${cookiefile}"
   ___pmfu_username="${1}"
   ___pmfu_min="${2}"
   ___pmfu_cf="${3}"
   for word in $( get_user_matchlist "$( get_userid "${___pmfu_username}" )" "${___pmfu_min}" ) ;
   do
      debuglev 1 && ferror "DEBUG1: parse_matches_for_user: found gameid ${word}"
      get_game_page "${word}" "${___pmfu_cf}" | parse_game_page stdin
   done
}

cat_matchlist() {
   # call: cat_matchlist "gimli,frodo,bilbo" "${userlistdelimiter}" "${minimum}" "${cookiefile}"
   # call: echo "12345982 2985817 398282847" | cat_matchlist "stdin"
   ___cm_userlist="${1}"
   ___cm_delim="${2}"
   ___cm_min="${3}"
   ___cm_cf="${4}"
   if echo "${___cm_userlist}" | grep -qiE "^stdin$" ;
   then
      awk '!x[$0]++'
   else
      # parse list of users, and make a master list of unique entries
      results="$(
         {
            for word in $( echo "${___cm_userlist}" | tr "${___cm_delim}" '\n' ) ;
            do
               get_user_matchlist "$( get_userid "${word}" )" "${___cm_min}"
            done
         } | awk '!x[$0]++'
      )"
      # for sorting them. -n will go lowest to highest.
      results="$( echo "${results}" | sort -n )"
      echo "${results}"
   fi
}

# DEFINE TRAPS

clean_vooblystats() {
   # use at end of entire script if you need to clean up tmpfiles
   # rm -f "${tmpfile1}" "${tmpfile2}" 2>/dev/null

   # Delayed cleanup
   if test -z "${VS_NO_CLEAN}" ;
   then
      nohup /bin/bash <<EOF 1>/dev/null 2>&1 &
sleep "${VS_CLEANUP_SEC:-300}" ; /bin/rm -r "${VS_TMPDIR:-NOTHINGTODELETE}" 1>/dev/null 2>&1 ;
EOF
   fi
}

CTRLC() {
   # use with: trap "CTRLC" 2
   # useful for controlling the ctrl+c keystroke
   :
}

CTRLZ() {
   # use with: trap "CTRLZ" 18
   # useful for controlling the ctrl+z keystroke
   :
}

parseFlag() {
   flag="$1"
   hasval=0
   case ${flag} in
      # INSERT FLAGS HERE
      "d" | "debug" | "DEBUG" | "dd" ) setdebug ; ferror "debug level ${debug}" ; __debug_set_by_param=1 ;;
      "u" | "usage" | "help" | "h" ) usage ; exit 1 ;;
      "V" | "fcheck" | "version" ) ferror "${scriptfile} version ${vooblystatsversion}" ; exit 1 ;;
      #"i" | "infile" | "inputfile" ) getval ; infile1=${tempval} ;;
      "c" | "conf" | "conffile" | "config" ) getval ; conffile="${tempval}" ;;
   esac

   debuglev 10 && { test ${hasval} -eq 1 && ferror "flag: ${flag} = ${tempval}" || ferror "flag: ${flag}" ; }
}

# DETERMINE LOCATION OF FRAMEWORK
f_needed=20181030
___frameworkpath="$( find $( echo "${FRAMEWORKPATH}" | tr ':' ' ' ) -maxdepth 1 -mindepth 0 -name 'framework.sh' 2>/dev/null )"
while read flocation ; do if test -e ${flocation} ; then __thisfver="$( sh ${flocation} --fcheck 2>/dev/null )" ; if test ${__thisfver:-0} -ge ${f_needed} ; then frameworkscript="${flocation}" ; break ; elif test -n "${___thisfver}" ; then printf "Obsolete: %s %s\n" "${flocation}" "${__thisfver}" 1>&2 ; fi ; fi ; done <<EOFLOCATIONS
${FRAMEWORKBIN:-/bin/false}
${___frameworkpath:-/bin/false}
./framework.sh
${scriptdir}/framework.sh
$HOME/bin/bgscripts/framework.sh
$HOME/bin/framework.sh
$HOME/bgscripts/framework.sh
$HOME/framework.sh
$HOME/.local/share/bgscripts/framework.sh
/usr/local/bin/bgscripts/framework.sh
/usr/local/bin/framework.sh
/usr/bin/bgscripts/framework.sh
/usr/bin/framework.sh
/bin/bgscripts/framework.sh
/usr/local/share/bgscripts/framework.sh
/usr/share/bgscripts/framework.sh
EOFLOCATIONS
test -z "${frameworkscript}" && echo "$0: framework ${f_needed} not found. Try setting FRAMEWORKPATH. Aborted." 1>&2 && exit 4

# INITIALIZE VARIABLES
# variables set in framework:
# today server thistty scriptdir scriptfile scripttrim
# is_cronjob stdin_piped stdout_piped stderr_piped sendsh sendopts
. ${frameworkscript} || echo "$0: framework did not run properly. Continuing..." 1>&2
infile1=
outfile1=
logfile=${scriptdir}/${scripttrim}.${today}.out
define_if_new interestedparties "bgstack15@gmail.com"
# SIMPLECONF
define_if_new default_conffile "./vooblystats.conf"
#define_if_new defuser_conffile ~/.config/vooblystats/vooblystats.conf
define_if_new VS_TMPDIR "$( mktemp -d )"
define_if_new VS_COOKIEFILE "./cookies"
#tmpfile1="$( TMPDIR="${VS_TMPDIR}" mktemp )"
#tmpfile2="$( TMPDIR="${VS_TMPDIR}" mktemp )"

# REACT TO OPERATING SYSTEM TYPE
case $( uname -s ) in
   Linux) : ;;
   FreeBSD) : ;;
   *) echo "${scriptfile}: 3. Indeterminate OS: $( uname -s )" 1>&2 && exit 3 ;;
esac

## REACT TO ROOT STATUS
#case ${is_root} in
#   1) # proper root
#      : ;;
#   sudo) # sudo to root
#      : ;;
#   "") # not root at all
#      #ferror "${scriptfile}: 5. Please run as root or sudo. Aborted."
#      #exit 5
#      :
#      ;;
#esac

# SET CUSTOM SCRIPT AND VALUES
#setval 1 sendsh sendopts<<EOFSENDSH     # if $1="1" then setvalout="critical-fail" on failure
#/usr/local/share/bgscripts/send.sh -hs  # setvalout maybe be "fail" otherwise
#/usr/share/bgscripts/send.sh -hs        # on success, setvalout="valid-sendsh"
#/usr/local/bin/send.sh -hs
#/usr/bin/mail -s
#EOFSENDSH
#test "${setvalout}" = "critical-fail" && ferror "${scriptfile}: 4. mailer not found. Aborted." && exit 4

# VALIDATE PARAMETERS
# objects before the dash are options, which get filled with the optvals
# to debug flags, use option DEBUG. Variables set in framework: fallopts
validateparams - "$@"

# LEARN EX_DEBUG
test -z "${__debug_set_by_param}" && fisnum "${VS_DEBUG}" && debug="${VS_DEBUG}"

# CONFIRM TOTAL NUMBER OF FLAGLESSVALS IS CORRECT
#if test ${thiscount} -lt 2 ;
#then
#   ferror "${scriptfile}: 2. Fewer than 2 flaglessvals. Aborted."
#   exit 2
#fi

# LOAD CONFIG FROM SIMPLECONF
# This section follows a simple hierarchy of precedence, with first being used:
#    1. parameters and flags
#    2. environment
#    3. config file
#    4. default user config: ~/.config/script/script.conf
#    5. default config: /etc/script/script.conf
if test -f "${conffile}" ;
then
   get_conf "${conffile}"
else
   if test "${conffile}" = "${default_conffile}" || test "${conffile}" = "${defuser_conffile}" ; then : ; else test -n "${conffile}" && ferror "${scriptfile}: Ignoring conf file which is not found: ${conffile}." ; fi
fi
test -f "${defuser_conffile}" && get_conf "${defuser_conffile}"
test -f "${default_conffile}" && get_conf "${default_conffile}"

# CONFIGURE VARIABLES AFTER PARAMETERS

## START READ CONFIG FILE TEMPLATE
#oIFS="${IFS}" ; IFS="$( printf '\n' )"
#infiledata=$( ${sed} ':loop;/^\/\*/{s/.//;:ccom;s,^.[^*]*,,;/^$/n;/^\*\//{s/..//;bloop;};bccom;}' "${infile1}") #the crazy sed removes c style multiline comments
#IFS="${oIFS}" ; infilelines=$( echo "${infiledata}" | wc -l )
#{ echo "${infiledata}" ; echo "ENDOFFILE" ; } | {
#   while read line ; do
#   # the crazy sed removes leading and trailing whitespace, blank lines, and comments
#   if test ! "${line}" = "ENDOFFILE" ;
#   then
#      line=$( echo "${line}" | sed -e 's/^\s*//;s/\s*$//;/^[#$]/d;s/\s*[^\]#.*$//;' )
#      if test -n "${line}" ;
#      then
#         debuglev 8 && ferror "line=\"${line}\""
#         if echo "${line}" | grep -qiE "\[.*\]" ;
#         then
#            # new zone
#            zone=$( echo "${line}" | tr -d '[]' )
#            debuglev 7 && ferror "zone=${zone}"
#         else
#            # directive
#            varname=$( echo "${line}" | awk -F= '{print $1}' )
#            varval=$( echo "${line}" | awk -F= '{$1="" ; printf "%s", $0}' | sed 's/^ //;' )
#            debuglev 7 && ferror "${zone}${varname}=\"${varval}\""
#            # simple define variable
#            eval "${zone}${varname}=\${varval}"
#         fi
#         ## this part is untested
#         #read -p "Please type something here:" response < ${thistty}
#         #echo "${response}"
#      fi
#   else

## REACT TO BEING A CRONJOB
#if test ${is_cronjob} -eq 1 ;
#then
#   :
#else
#   :
#fi

# SET TRAPS
#trap "CTRLC" 2
#trap "CTRLZ" 18
trap '__ec=$? ; clean_vooblystats ; trap "" 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 19 20 ; exit ${__ec} ;' 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 19 20

# DEBUG SIMPLECONF
debuglev 5 && {
   ferror "Using values"
   # used values: EX_(OPT1|OPT2|VERBOSE)
   set | grep -iE "^VS_" 1>&2
}

# MAIN LOOP
#{

   # Initial authentication and download a random game page
   rm -f "${VS_COOKIEFILE}"
   
   # need to auth at least one for any other voobly function to work correctly.
   auth_to_voobly "${VS_USERNAME}" "${VS_PASSWORD}" "${VS_COOKIEFILE}"
   #get_game_page "${opt1}" "${VS_COOKIEFILE}" | parse_game_page stdin
   #parse_matches_for_user "frodo" "300" "${VS_COOKIEFILE}"
   display_csv_headers

   #get_user_matchlist "123989133" "150"

   {
      for word in $( cat_matchlist "${VS_MATCHLIST:-frodo,bilbo,merry,pippin}" "," "300" "${VS_COOKIEFILE}" )
      do
         get_game_page "${word}" | parse_game_page stdin
      done
   } | sort -k1,3

#} | tee -a ${logfile}

# EMAIL LOGFILE
#${sendsh} ${sendopts} "${server} ${scriptfile} out" ${logfile} ${interestedparties}

## STOP THE READ CONFIG FILE
#return_code 0
#fi ; done ; }