initial commit

4 files changed, 692 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..4ddf8bb
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,4 @@
+cookies
+old
+*.conf
+*.swp
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..c3d6d49
--- /dev/null
+++ b/README.md
@@ -0,0 +1,24 @@
+# Readme for vooblystats.sh
+Voobly website scraper written in POSIX shell! Of course sh is not ideal for parsing html, but I suck at understanding the python3 requests library documentation.
+
+Use the config file to set username and password. I implemented the config file so I can exclude secrets for this scm repo.
+
+This is more of a library of functions to be used for whatever you want, as long as it scrapes the webpages from voobly.com. I use it to list my friends and me, then all the games in our history, and then pull all the stats for those games.
+
+# List available functions
+View available functions
+
+    grep -E '^[^_][a-z_]+\(\)' vooblystats.sh
+
+View available functions with brief explanation of how to call.
+
+    grep -A2 -E '^[^_][a-z_]+\(\)' vooblystats.sh
+
+# License
+CC-BY-SA 4.0
+
+# Alternatives
+* I should have used python implementation https://github.com/happyleavesaoc/python-voobly or at least augmented it.
+* Haskell https://github.com/bowswung/voobly-scraper
+* C++ https://github.com/kliu31415/aoe2data
+* Another python implementation https://github.com/hepter/VooblyAPI-Parse
diff --git a/vooblystats.conf.example b/vooblystats.conf.example
new file mode 100644
index 0000000..baa38cc
--- /dev/null
+++ b/vooblystats.conf.example
@@ -0,0 +1,3 @@
+VS_USERNAME="frodo"
+VS_PASSWORD="NONE"
+VS_MATCHLIST="name1,name2,__xX_gonzo_Xx__"
diff --git a/vooblystats.sh b/vooblystats.sh
new file mode 100755
index 0000000..2c56f68
--- /dev/null
+++ b/vooblystats.sh
@@ -0,0 +1,661 @@
+#!/bin/sh
+# Filename: vooblystats.sh
+# License: CC-BY-SA 4.0
+# Author: bgstack15@gmail.com
+# Startdate: 2020-01-24 08:57:56
+# Title: 
+# Purpose: 
+# Package: 
+# History: 
+# Usage: 
+# Reference: ftemplate.sh 2019-05-02a ; framework.sh 2018-05-02a
+# Improve:
+#    write a separate, no-auth-needed tool that scrapes https://www.voobly.com/games/view/Age-of-Empires-II-The-Conquerors for lobby attendance, and put it in cronjob.
+#    add debuglev ferror stuff
+# Dependencies:
+#    framework.sh
+fiversion="2019-05-02a"
+vooblystatsversion="2020-01-28a"
+
+usage() {
+   ${PAGER:-/usr/bin/less -F} >&2 <<ENDUSAGE
+usage: vooblystats.sh [-duV] [-c conffile]
+Demo script for pulling stats from voobly.com
+version ${vooblystatsversion}
+ -d debug   Show debugging info, including parsed variables.
+ -u usage   Show this usage block.
+ -V version Show script version number.
+ -c conf    Read in this config file.
+Return values:
+ 0 Normal
+ 1 Help or version info displayed
+ 2 Count or type of flaglessvals is incorrect
+ 3 Incorrect OS type
+ 4 Unable to find dependency
+ 5 Not run as root or sudo
+ENDUSAGE
+}
+
+# DEFINE FUNCTIONS
+
+make_voobly_request() {
+   # call: make_voobly_request "${url}" "${referer}" "${usecookiefilebool}" "${cookiefile}" "${usedatabool}" "${data}" "{includeheadersbool}"
+   ___mvr_url="${1}"
+   ___mvr_referer="${2}"
+   ___mvr_usecf="${3}"
+   ___mvr_cf="${4}"
+   ___mvr_used="${5}"
+   ___mvr_data="${6}"
+   ___mvr_includeh="${7}"
+
+   ___mvr_host="$( echo "${___mvr_url}" | awk -F'/' '{print $3}' )"
+   ___mvr_flags_cf=""
+   fistruthy "${___mvr_usecf}" && ___mvr_flags_cf="-b ${VS_COOKIEFILE} -c ${VS_COOKIEFILE}"
+   ___mvr_flags_data=""
+   fistruthy "${___mvr_used}" && ___mvr_flags_data="--data ${___mvr_data}"
+   ___mvr_flags_headers=""
+   fistruthy "${___mvr_includeh}" && ___mvr_flags_headers="-i"
+   curl "${___mvr_url}" \
+      ${___mvr_flags_headers} -s ${___mvr_flags_cf} \
+      -H "Host: ${___mvr_host}" \
+      -H 'User-Agent: Mozilla/5.0 (Windows NT6.3; Win64; x64) ApplieWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36' \
+      -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' \
+      -H 'Accept-Language: en-US,en;q=0.5' --compressed \
+      -H "Referer: ${___mvr_referer:-${___mvr_url}}" \
+      -H 'Connection: keep-alive' ${___mvr_flags_data}
+   fistruthy "${___mvr_includeh}" && cat "${___mvr_cf}" 2>/dev/null
+
+}
+
+html_encode() {
+   # call: safevar="$( html_encode "unsafe;string" )"
+   # reference:
+   # https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command/10797966#10797966
+   ___he_input="${1}"
+   echo "${___he_input}" | curl -Gso /dev/null -w %{url_effective} --data-urlencode @- "" | sed -E 's/..(.*).../\1/'
+}
+
+auth_to_voobly() {
+   # call: auth_to_voobly "${username}" "${password}" "${cookiefile}"
+   ___atv_username="${1}"
+   ___atv_password="${2}"
+   ___atv_cookiefile="${3}"
+   # transform password to html-safe
+   ___atv_password_safe="$( html_encode "${___atv_password}" )"
+   make_voobly_request "https://www.voobly.com/" "https://wwww.voobly.com" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response1"
+   make_voobly_request "https://www.voobly.com/login" "https://wwww.voobly.com/" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response2"
+   make_voobly_request "https://www.voobly.com/login/auth" "https://wwww.voobly.com/login" "true" "${___atv_cookiefile}" "true" "username=${___atv_username}&password=${___atv_password_safe}" "true" 1>"${VS_TMPDIR}/auth-response3"
+   make_voobly_request "https://www.voobly.com/welcome" "https://wwww.voobly.com/login" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response4"
+   make_voobly_request "https://www.voobly.com/profile/view/123989133" "https://wwww.voobly.com/welcome" "true" "${___atv_cookiefile}" "false" "" "true" 1>"${VS_TMPDIR}/auth-response5"
+}
+
+parse_game_page() {
+   # call: parse_game_page "${inputfileORurl}" "${cookiefile}"
+   # output: to stdout, a csv that originally was planned to resemble:
+   # GAME,gameid,dateplayed,maptype,duration,playercount,gamemod
+   # PLAYER,playerid,gameid,playercountnum,playername,playercountry,playernewrating,playerpoints,playerteam,playerciv,playerwonbool,militaryscore,economyscore,technologyscore,societyscore,totalscore,unitskilled,unitslost,buildingsrazed,buildingslost,unitsconverted,food,wood,stone,gold,tradeprofit,tributerec,tributesent,feudaltime,castletime,imptime,mapexplored,researchcount,researchpercent,totalwonders,totalcastles,reliccount,relicgold,villagerhigh
+   ___pgp_infile="${1}"
+   ___pgp_cf="${2}"
+   if test "${___pgp_infile}" = "stdin" ;
+   then
+      fullpage="$( cat )"
+   else
+      # check if it contains a URL
+      if echo "${___pgp_infile}" | grep -qiE '^https?:\/\/' ;
+      then
+         # need to retrieve it
+         fullpage="$( make_voobly_request "${___pgp_infile}" "https://www.voobly.com/welcome" "true" "${___pgp_cf:-${VS_COOKIEFILE}}" "false" "" "false" )"
+      else
+         fullpage="$( cat "${___pgp_infile}" 2>/dev/null )"
+      fi
+   fi
+
+   # test if valid page
+   if ! echo "${fullpage}" | grep -qiE "economy score" ;
+   then
+      # if the page does not contain the expression "economy score" then it is invalid
+      ferror "${scriptfile}: cannot parse game page ${___pgp_infile}. Skipping..."
+      return
+   fi
+
+   # game info
+   gameid="$( echo "${fullpage}" | _get_simple_value "Match Details" | tr -d '#' )"
+   _reldate="$( echo "${fullpage}" | _get_simple_value "Date Played:" )"
+   absolute_date="$( date -d "$( echo "${_reldate}" | tr -d ',-' )" -u "+%FT%TZ" )"
+   map="$( echo "${fullpage}" | _get_simple_value "Map:" )"
+   duration="$( echo "${fullpage}" | _get_simple_value "Duration:" )"
+   playercount="$( echo "${fullpage}" | _get_simple_value "Players:" )"
+   gamemod="$( echo "${fullpage}" | _get_simple_value "Game Mod:" )"
+   gameladder="$( echo "${fullpage}" | awk '/Ladder:/' | grep -oE ">.*<" | tail -c +2 | head -c -2 )"
+
+   echo "GAME,${gameid},${absolute_date},${map},${duration},${gameladder},${playercount},${gamemod}," # HEADERCSV
+
+   # per player info
+   x=0
+   while test $x -lt $playercount ;
+   do
+      x=$(( x + 1 ))
+      _playerline="$( echo "${fullpage}" | grep -E "^<table.*profile\/view\/[0-9]{1,15}.*table>$" | sed -n "${x}p" )"
+      _playerline_num="$( echo "${fullpage}" | grep -n -E "^<table.*profile\/view" | awk -F':' '{print $1}' | sed -n "${x}p" )"
+      _playerline_num=$(( _playerline_num + 1 ))
+      _playerline2="$( echo "${fullpage}" | sed -n "${_playerline_num}p" )"
+      playerid="$( echo "${_playerline}" | grep -oE "profile\/view/[0-9]{1,15}.?>" | tr -dc '[0-9]' )"
+      playername="$( echo "${_playerline}" | grep -oE "profile\/view/[0-9]{1,15}.?>.*<\/a><\/" | sed -r -e 's/<\/a>.*$//;' -e 's/.*view\/[0-9]{1,15}\">//;' )"
+      playercountry="$( echo "${_playerline}" | grep -oE "res\/flags/[^\.]{1,20}\.(png|jpg)" | awk -F'/' '{print $NF}' | awk -F'.' '{print $1}' )"
+      playernewrating="$( echo "${_playerline2}" | awk -F'[<>]' '{print $5}' )"
+      playerpoints="$( echo "${_playerline2}" | awk -F'[<>]' '{print $11}' )"
+      playerteam="$( echo "${_playerline2}" | awk -F'[<>]' '{print $17}' )"
+      playerwinbool="$( echo "${_playerline}" | grep -qiE 'win.PNG' && echo "true" || echo "false" )"
+      playerpremiumbool="$( echo "${_playerline}" | grep -qiE 'user-premium' && echo "true" || echo "false" )"
+      playergamemvpbool="$( echo "${_playerline}" | grep -qiE 'high.PNG' && echo "true" || echo "false" )"
+      playerclan="$( echo "${_playerline}" | grep -oE "<a.*\.voobly\.com>.{1,50}<\/a><a.*profile\/view" | sed -r -e 's/<\/a>.*$//' -e 's/^.*\.voobly\.com>//' )"
+      # if x > ( numplayers/2), then swap newrating and points
+      if test "$( printf '%s;%s>(%s/2);\n' "scale=2" "${x}" "${playercount}" | bc )" = "1" ;
+      then
+         tempvar="${playernewrating}"
+         playernewrating="${playerteam}"
+         playerteam="${tempvar}"
+         unset tempvar
+      fi
+
+      # in-game stats
+      _playerstatslines="$( echo "${fullpage}" | grep -A4 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | grep center )"
+      # works on webpage game stats sections 1, 2, and 5.
+      #echo "${_playerstatslines}" | sed -r -e 's/(<[^\>]+>)+/@/g;'| tr -d ',%' | awk -F'@' 'BEGIN{OFS="\t"} {print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10}'
+      _playerstatslines2="$( echo "${_playerstatslines}" | sed -r -e 's/(<[^\>]+>)+/@/g;' | tr -d ',%' | awk -F'@' '{print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10}' )"
+      _playerstatslines3="$( echo "${_playerstatslines2}" | tr -d '\n' | sed -r -e 's/\s+/ /g;' )" 
+
+      playerciv="$( echo "${fullpage}" | grep -B4 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | head -n1 | sed -r -e 's/(<[^\>]+>)+//g;' )"
+      playercolorhex="$( echo "${fullpage}" | grep -B2 -E "${playername}.*<\/a>(<\/b>|<\/span>)*$" | head -n1 | tr -d ';' | awk '{print $3}' )"
+      # 0054A6 00A651 00FFFF 92278F C0C0C0 FF0000 FF8000 FFFF00
+      playercolorname="nocolor"
+      case "${playercolorhex###}" in
+         0054A6) playercolorname="blue" ;;
+         FF0000) playercolorname="red" ;;
+         FFFF00) playercolorname="yellow" ;;
+         00A651) playercolorname="green" ;;
+         00FFFF) playercolorname="cyan" ;;
+         92278F) playercolorname="purple" ;;
+         C0C0C0) playercolorname="gray" ;;
+         FF8000) playercolorname="orange" ;;
+      esac
+
+      # and because bash read is broken on devuan for an uknown reason... we have to use awk
+      milscore="$( echo "${_playerstatslines3}" | awk '{print $1}' )"
+      ecoscore="$( echo "${_playerstatslines3}" | awk '{print $2}' )"
+      techscore="$( echo "${_playerstatslines3}" | awk '{print $3}' )"
+      socscore="$( echo "${_playerstatslines3}" | awk '{print $4}' )"
+      totscore="$( echo "${_playerstatslines3}" | awk '{print $5}' )"
+      unitskilled="$( echo "${_playerstatslines3}" | awk '{print $6}' )"
+      unitslost="$( echo "${_playerstatslines3}" | awk '{print $7}' )"
+      buildingsrazed="$( echo "${_playerstatslines3}" | awk '{print $8}' )"
+      buildingslost="$( echo "${_playerstatslines3}" | awk '{print $9}' )"
+      unitsconverted="$( echo "${_playerstatslines3}" | awk '{print $10}' )"
+      food="$( echo "${_playerstatslines3}" | awk '{print $11}' )"
+      wood="$( echo "${_playerstatslines3}" | awk '{print $12}' )"
+      stone="$( echo "${_playerstatslines3}" | awk '{print $13}' )"
+      gold="$( echo "${_playerstatslines3}" | awk '{print $14}' )"
+      tradeprofit="$( echo "${_playerstatslines3}" | awk '{print $15}' )"
+      tributerec="$( echo "${_playerstatslines3}" | awk '{print $16}' )"
+      tributesent="$( echo "${_playerstatslines3}" | awk '{print $17}' )"
+      feudaltime="$( echo "${_playerstatslines3}" | awk '{print $18}' )"
+      castletime="$( echo "${_playerstatslines3}" | awk '{print $19}' )"
+      imptime="$( echo "${_playerstatslines3}" | awk '{print $20}' )"
+      mapexplored="$( echo "${_playerstatslines3}" | awk '{print $21}' )"
+      researchcount="$( echo "${_playerstatslines3}" | awk '{print $22}' )"
+      researchpercent="$( echo "${_playerstatslines3}" | awk '{print $23}' )"
+      totalwonders="$( echo "${_playerstatslines3}" | awk '{print $24}' )"
+      totalcastles="$( echo "${_playerstatslines3}" | awk '{print $25}' )"
+      reliccount="$( echo "${_playerstatslines3}" | awk '{print $26}' )"
+      relicgold="$( echo "${_playerstatslines3}" | awk '{print $27}' )"
+      villagerhigh="$( echo "${_playerstatslines3}" | awk '{print $28}' )"
+
+      # end of player loop
+      echo "PLAYER,${playerid},${gameid},${x},${playername},${playercolorhex},${playercolorname},${playercountry},${playerpremiumbool},${playerclan},${playernewrating},${playerpoints},${playerteam},${playerciv},${playerwinbool},${milscore},${ecoscore},${techscore},${socscore},${totscore},${unitskilled},${unitslost},${buildingsrazed},${buildingslost},${unitsconverted},${food},${wood},${stone},${gold},${tradeprofit},${tributerec},${tributesent},${feudaltime},${castletime},${imptime},${mapexplored},${researchcount},${researchpercent},${totalwonders},${totalcastles},${reliccount},${relicgold},${villagerhigh}," # HEADERCSV
+
+      unset -v playerclan _playerline _playerline2 _playerline_num playerid playername playercountry playernewrating playerpoints
+   done
+
+} # end of parse_game_page
+
+parse_profile_matchlist_page() {
+   # call: parse_profile_matchlist_page "${inputfileORurl}" "${cookiefile}"
+   # output: delimited list of match numbers found on the profile page.
+   # reference: parse_game_page
+   ___ppmp_infile="${1}"
+   ___ppmp_cf="${2}"
+   if test "${___ppmp_infile}" = "stdin" ;
+   then
+      fullpage="$( cat )"
+   else
+      # check if it contains a URL
+      if echo "${___ppmp_infile}" | grep -qiE '^https?:\/\/' ;
+      then
+         # need to retrieve it
+         fullpage="$( make_voobly_request "${___ppmp_infile}" "https://www.voobly.com/welcome" "true" "${___ppmp_cf:-${VS_COOKIEFILE}}" "false" "" "false" )"
+      else
+         fullpage="$( cat "${___ppmp_infile}" 2>/dev/null )"
+      fi
+   fi
+
+   # test if valid page
+   results="$( echo "${fullpage}" | grep -oE "match\/view\/[0-9]+" | awk -F'/' '!x[$3]++ {print $3}' )"
+   if test "$( echo "${results}" | wc -l | tr -dc '[0-9]' )" -gt 0 ;
+   then
+      echo "${results}"
+   else
+      ferror "${scriptfile}: cannot parse matchlist page ${___ppmp_infile}. Skipping..."
+      echo "invalid_matchlist_page"
+   fi
+
+} # end parse_profile_matchlist_page
+
+get_match_url() {
+   # call: get_match_url "${matchid}"
+   ___gmu_gameid="${1}"
+   ___gmu_gameurl="https://www.voobly.com/match/view/$( echo "${___gmu_gameid}" | grep -oE "match\/view\/[0-9]+" | awk -F'/' '{print $3}' )"
+   echo "${___gmu_gameurl}"
+}
+
+get_profile_matchlist_url() {
+   # call: get_profile_matchlist_url "123456789" "1" for page 1
+   # call: get_profile_matchlist_url "https://www.voobly.com/profile/view/123989133/" 
+   # call: get_profile_matchlist_url "https://www.voobly.com/profile/view/123989133/" "1"
+   # output: url of page that lists matches, including a page number if requested.
+   # we will use a cool shortcut i found, https://www.voobly.com/games/matches/user/123989133/0/0#pagebrowser1
+   ___gpmu_url="${1}"
+   ___gpmu_pagenum="${2}"
+
+   # if it is already a full page with a pagenumber, just return it
+   if echo "${___gpmu_url}" | grep -qiE "\/0\/[0-9]+#pagebrowser" ;
+   then
+      : # make no changes to it
+   else
+      if echo "${___gpmu_url}" | grep -qiE "voobly.com" ;
+      then
+         # derive userid
+         ___gpmu_url="$( echo "${___gpmu_url}" | grep -oE "(view|user)\/[0-9]+" | awk -F'/' '{print $2}' )"
+      fi
+      ___gpmu_url="https://www.voobly.com/games/matches/user/${___gpmu_url}"
+      # https://www.voobly.com/games/matches/user/123989133/0/0#pagebrowser1
+      # https://www.voobly.com/games/matches/user/123989133/0/1#pagebrowser1
+      if test -n "${___gpmu_pagenum}" ;
+      then
+         ___gpmu_url="${___gpmu_url}/0/${___gpmu_pagenum}#pagebrowser1"
+      fi
+   fi
+   echo "${___gpmu_url}"
+}
+
+get_user_matchlist() {
+   # call: get_user_matchlist "useridORuserurl" "${mingamecount}"
+   # output: on stdout, a list of gameid numbers.
+   ___gum_userurl="${1}"
+   ___gum_min="${2}" ; test -z "${___gum_min}" && ___gum_min=10
+   debuglev 9 && ferror "DEBUG9: get_user_matchlist $*"
+   if echo "${___gum_userurl}" | grep -qoE "^[0-9]{1,15}$" ;
+   then
+      # just userid, so make url
+      ___gum_userurl="https://www.voobly.com/profile/view/${___gum_userurl}/"
+   fi
+   results=""
+
+   # start going through games, starting on page 1
+   x=0
+   ___gum_continue=0
+   ___gum_previous_count=-1
+   ___gum_count=0
+   while test ${___gum_continue} -eq 0 ;
+   do
+      x=$(( x + 1 ))
+      # get game page
+      loopresults="$( parse_profile_matchlist_page "$( get_profile_matchlist_url "${___gum_userurl}" "${x}" )" "${VS_COOKIEFILE}" )"
+      debuglev 2 && ferror "DEBUG2: get_user_matchlist: $( echo "${loopresults}" | xargs )"
+      results="${results} ${loopresults}"
+      ___gum_count="$( echo "${results}" | wc -w )"
+      test ${___gum_count} -ge ${___gum_min} && ___gum_continue=1
+      # if results have not increased, then we need to stop looping because we are not pulling any more gameids
+      test ${___gum_previous_count} -ge ${___gum_count} && ___gum_continue=1
+      ___gum_previous_count="${___gum_count}"
+   done
+   # assume we are good now, where count >= min
+   echo "${results}" | xargs -n1
+}
+
+get_userid() {
+   # call: get_userid "gimli" "${cookiefile}"
+   # cookiefile is absolutely necessary for this function!
+   # output to stdout: "123456789"
+   ___gu_username="${1}"
+   ___gu_cf="${2}" ; test -z "${___gu_cf}" && ___gu_cf="${VS_COOKIEFILE}"
+
+   # if the input is already a userid, just return it
+   if echo "${___gu_username}" | grep -qiE "^[0-9]+$" ;
+   then
+      echo "${___gu_username}"
+   elif echo "${___gu_username}" | grep -qiE "profile\/view\/[0-9]+" ;
+   then
+      response="$( echo "${___gu_username}" | grep -oE 'profile\/view\/[0-9]+' | cut -d'/' -f3 | head -n1 )"
+      echo "${response}"
+   else
+      # assume it is just a regular name to parse
+      # we need to pass the sessionid in the post data, or else we get an error, even though it is in the cookie.
+      sessionid="$( awk '/vbly_session1/{print $NF}' "${___gu_cf}" )"
+      response="$( make_voobly_request "https://www.voobly.com/friends/browse/Search/Search" "https://www.voobly.com/Welcome" "true" "${VS_COOKIEFILE}" "true" "query=${___gu_username}&session1=${sessionid}" )"
+      # the response will not be a real page; it will just be a redirect notice but it is good enough to get the userid
+      if echo "${response}" | grep -qiE "no results found" ;
+      then
+         ferror "get_userid: unable to find a userid for ${___gu_username}"
+         echo "invalid_userid"
+      else
+         response="$( echo "${response}" | grep -oE 'profile\/view\/[0-9]+' | cut -d'/' -f3 | head -n1 )"
+         echo "${response}"
+      fi
+   fi
+
+}
+
+get_game_page() {
+   # call: get_game_page "123456789" "${cookiefile}" | parse_game_page stdin
+   # output to stdout: the html of the game page.
+   ___ggp_number="${1}"
+   ___ggp_cf="${2}"
+
+   # do some validation to make sure this is a valid game number
+   ___ggp_number_orig="${___ggp_number}"
+   if echo "${___ggp_number}" | grep -qE "voobly\.com" ;
+   then
+      # somebody passed a whole URL, so let us split it
+      ___ggp_number="${___ggp_number##*view/}"
+   fi
+   ___ggp_number="$( echo "${___ggp_number}" | grep -oE '^[0-9]+' )"
+
+   debuglev 1 && ferror "Trying to read game ${___ggp_number}"
+   
+   response="$( make_voobly_request "https://www.voobly.com/match/view/${___ggp_number}" "https://www.voobly.com/" "true" "${____ggp_cf}" "no" "" "false" )"
+   if echo "${response}" | grep -qiE "economy score" ;
+   then
+      # is valid game page
+      echo "${response}"
+   else
+      ferror "${scriptfile}: ERROR! Invalid game page request: ${___ggp_number_orig}. Used match number ${___ggp_number}. Skipping..."
+      echo "invalid_page_request"
+   fi
+      
+}
+
+_get_simple_value() {
+   # call: echo "${fullpage}" | _get_simple_value "Match Details"
+   grep -A1 "${1}<\/" | tail -n1 | awk -F'[<>]' '{print $3}'
+}
+
+display_csv_headers() {
+   # output csv headers.
+   #echo "GAME,gameid,absolute_date,map,duration,playercount,gamemod,"
+   #echo "PLAYER,playerid,gameid,x,playername,playercolorhex,playercolorname,playercountry,playerpremiumbool,playerclan,playernewrating,playerpoints,playerteam,playerciv,playerwinbool,milscore,ecoscore,techscore,socscore,totscore,unitskilled,unitslost,buildingsrazed,buildingslost,unitsconverted,food,wood,stone,gold,tradeprofit,tributerec,tributesent,feudaltime,castletime,imptime,mapexplored,researchcount,researchpercent,totalwonders,totalcastles,reliccount,relicgold,villagerhigh,"
+   sed -n -r -e '/HEADERCSV$/{ s/^.*echo "//;s/"\s*# HEADERCSV$//;s/\$\{//g;s/\}//g;p}' "${scriptfile}"
+}
+
+parse_matches_for_user() {
+   # call: parse_matches_for_user "gimli" "10" "${cookiefile}"
+   ___pmfu_username="${1}"
+   ___pmfu_min="${2}"
+   ___pmfu_cf="${3}"
+   for word in $( get_user_matchlist "$( get_userid "${___pmfu_username}" )" "${___pmfu_min}" ) ;
+   do
+      debuglev 1 && ferror "DEBUG1: parse_matches_for_user: found gameid ${word}"
+      get_game_page "${word}" "${___pmfu_cf}" | parse_game_page stdin
+   done
+}
+
+cat_matchlist() {
+   # call: cat_matchlist "gimli,frodo,bilbo" "${userlistdelimiter}" "${minimum}" "${cookiefile}"
+   # call: echo "12345982 2985817 398282847" | cat_matchlist "stdin"
+   ___cm_userlist="${1}"
+   ___cm_delim="${2}"
+   ___cm_min="${3}"
+   ___cm_cf="${4}"
+   if echo "${___cm_userlist}" | grep -qiE "^stdin$" ;
+   then
+      awk '!x[$0]++'
+   else
+      # parse list of users, and make a master list of unique entries
+      results="$(
+         {
+            for word in $( echo "${___cm_userlist}" | tr "${___cm_delim}" '\n' ) ;
+            do
+               get_user_matchlist "$( get_userid "${word}" )" "${___cm_min}"
+            done
+         } | awk '!x[$0]++'
+      )"
+      # for sorting them. -n will go lowest to highest.
+      results="$( echo "${results}" | sort -n )"
+      echo "${results}"
+   fi
+}
+
+# DEFINE TRAPS
+
+clean_vooblystats() {
+   # use at end of entire script if you need to clean up tmpfiles
+   # rm -f "${tmpfile1}" "${tmpfile2}" 2>/dev/null
+
+   # Delayed cleanup
+   if test -z "${VS_NO_CLEAN}" ;
+   then
+      nohup /bin/bash <<EOF 1>/dev/null 2>&1 &
+sleep "${VS_CLEANUP_SEC:-300}" ; /bin/rm -r "${VS_TMPDIR:-NOTHINGTODELETE}" 1>/dev/null 2>&1 ;
+EOF
+   fi
+}
+
+CTRLC() {
+   # use with: trap "CTRLC" 2
+   # useful for controlling the ctrl+c keystroke
+   :
+}
+
+CTRLZ() {
+   # use with: trap "CTRLZ" 18
+   # useful for controlling the ctrl+z keystroke
+   :
+}
+
+parseFlag() {
+   flag="$1"
+   hasval=0
+   case ${flag} in
+      # INSERT FLAGS HERE
+      "d" | "debug" | "DEBUG" | "dd" ) setdebug ; ferror "debug level ${debug}" ; __debug_set_by_param=1 ;;
+      "u" | "usage" | "help" | "h" ) usage ; exit 1 ;;
+      "V" | "fcheck" | "version" ) ferror "${scriptfile} version ${vooblystatsversion}" ; exit 1 ;;
+      #"i" | "infile" | "inputfile" ) getval ; infile1=${tempval} ;;
+      "c" | "conf" | "conffile" | "config" ) getval ; conffile="${tempval}" ;;
+   esac
+
+   debuglev 10 && { test ${hasval} -eq 1 && ferror "flag: ${flag} = ${tempval}" || ferror "flag: ${flag}" ; }
+}
+
+# DETERMINE LOCATION OF FRAMEWORK
+f_needed=20181030
+___frameworkpath="$( find $( echo "${FRAMEWORKPATH}" | tr ':' ' ' ) -maxdepth 1 -mindepth 0 -name 'framework.sh' 2>/dev/null )"
+while read flocation ; do if test -e ${flocation} ; then __thisfver="$( sh ${flocation} --fcheck 2>/dev/null )" ; if test ${__thisfver:-0} -ge ${f_needed} ; then frameworkscript="${flocation}" ; break ; elif test -n "${___thisfver}" ; then printf "Obsolete: %s %s\n" "${flocation}" "${__thisfver}" 1>&2 ; fi ; fi ; done <<EOFLOCATIONS
+${FRAMEWORKBIN:-/bin/false}
+${___frameworkpath:-/bin/false}
+./framework.sh
+${scriptdir}/framework.sh
+$HOME/bin/bgscripts/framework.sh
+$HOME/bin/framework.sh
+$HOME/bgscripts/framework.sh
+$HOME/framework.sh
+$HOME/.local/share/bgscripts/framework.sh
+/usr/local/bin/bgscripts/framework.sh
+/usr/local/bin/framework.sh
+/usr/bin/bgscripts/framework.sh
+/usr/bin/framework.sh
+/bin/bgscripts/framework.sh
+/usr/local/share/bgscripts/framework.sh
+/usr/share/bgscripts/framework.sh
+EOFLOCATIONS
+test -z "${frameworkscript}" && echo "$0: framework ${f_needed} not found. Try setting FRAMEWORKPATH. Aborted." 1>&2 && exit 4
+
+# INITIALIZE VARIABLES
+# variables set in framework:
+# today server thistty scriptdir scriptfile scripttrim
+# is_cronjob stdin_piped stdout_piped stderr_piped sendsh sendopts
+. ${frameworkscript} || echo "$0: framework did not run properly. Continuing..." 1>&2
+infile1=
+outfile1=
+logfile=${scriptdir}/${scripttrim}.${today}.out
+define_if_new interestedparties "bgstack15@gmail.com"
+# SIMPLECONF
+define_if_new default_conffile "./vooblystats.conf"
+#define_if_new defuser_conffile ~/.config/vooblystats/vooblystats.conf
+define_if_new VS_TMPDIR "$( mktemp -d )"
+define_if_new VS_COOKIEFILE "./cookies"
+#tmpfile1="$( TMPDIR="${VS_TMPDIR}" mktemp )"
+#tmpfile2="$( TMPDIR="${VS_TMPDIR}" mktemp )"
+
+# REACT TO OPERATING SYSTEM TYPE
+case $( uname -s ) in
+   Linux) : ;;
+   FreeBSD) : ;;
+   *) echo "${scriptfile}: 3. Indeterminate OS: $( uname -s )" 1>&2 && exit 3 ;;
+esac
+
+## REACT TO ROOT STATUS
+#case ${is_root} in
+#   1) # proper root
+#      : ;;
+#   sudo) # sudo to root
+#      : ;;
+#   "") # not root at all
+#      #ferror "${scriptfile}: 5. Please run as root or sudo. Aborted."
+#      #exit 5
+#      :
+#      ;;
+#esac
+
+# SET CUSTOM SCRIPT AND VALUES
+#setval 1 sendsh sendopts<<EOFSENDSH     # if $1="1" then setvalout="critical-fail" on failure
+#/usr/local/share/bgscripts/send.sh -hs  # setvalout maybe be "fail" otherwise
+#/usr/share/bgscripts/send.sh -hs        # on success, setvalout="valid-sendsh"
+#/usr/local/bin/send.sh -hs
+#/usr/bin/mail -s
+#EOFSENDSH
+#test "${setvalout}" = "critical-fail" && ferror "${scriptfile}: 4. mailer not found. Aborted." && exit 4
+
+# VALIDATE PARAMETERS
+# objects before the dash are options, which get filled with the optvals
+# to debug flags, use option DEBUG. Variables set in framework: fallopts
+validateparams - "$@"
+
+# LEARN EX_DEBUG
+test -z "${__debug_set_by_param}" && fisnum "${VS_DEBUG}" && debug="${VS_DEBUG}"
+
+# CONFIRM TOTAL NUMBER OF FLAGLESSVALS IS CORRECT
+#if test ${thiscount} -lt 2 ;
+#then
+#   ferror "${scriptfile}: 2. Fewer than 2 flaglessvals. Aborted."
+#   exit 2
+#fi
+
+# LOAD CONFIG FROM SIMPLECONF
+# This section follows a simple hierarchy of precedence, with first being used:
+#    1. parameters and flags
+#    2. environment
+#    3. config file
+#    4. default user config: ~/.config/script/script.conf
+#    5. default config: /etc/script/script.conf
+if test -f "${conffile}" ;
+then
+   get_conf "${conffile}"
+else
+   if test "${conffile}" = "${default_conffile}" || test "${conffile}" = "${defuser_conffile}" ; then : ; else test -n "${conffile}" && ferror "${scriptfile}: Ignoring conf file which is not found: ${conffile}." ; fi
+fi
+test -f "${defuser_conffile}" && get_conf "${defuser_conffile}"
+test -f "${default_conffile}" && get_conf "${default_conffile}"
+
+# CONFIGURE VARIABLES AFTER PARAMETERS
+
+## START READ CONFIG FILE TEMPLATE
+#oIFS="${IFS}" ; IFS="$( printf '\n' )"
+#infiledata=$( ${sed} ':loop;/^\/\*/{s/.//;:ccom;s,^.[^*]*,,;/^$/n;/^\*\//{s/..//;bloop;};bccom;}' "${infile1}") #the crazy sed removes c style multiline comments
+#IFS="${oIFS}" ; infilelines=$( echo "${infiledata}" | wc -l )
+#{ echo "${infiledata}" ; echo "ENDOFFILE" ; } | {
+#   while read line ; do
+#   # the crazy sed removes leading and trailing whitespace, blank lines, and comments
+#   if test ! "${line}" = "ENDOFFILE" ;
+#   then
+#      line=$( echo "${line}" | sed -e 's/^\s*//;s/\s*$//;/^[#$]/d;s/\s*[^\]#.*$//;' )
+#      if test -n "${line}" ;
+#      then
+#         debuglev 8 && ferror "line=\"${line}\""
+#         if echo "${line}" | grep -qiE "\[.*\]" ;
+#         then
+#            # new zone
+#            zone=$( echo "${line}" | tr -d '[]' )
+#            debuglev 7 && ferror "zone=${zone}"
+#         else
+#            # directive
+#            varname=$( echo "${line}" | awk -F= '{print $1}' )
+#            varval=$( echo "${line}" | awk -F= '{$1="" ; printf "%s", $0}' | sed 's/^ //;' )
+#            debuglev 7 && ferror "${zone}${varname}=\"${varval}\""
+#            # simple define variable
+#            eval "${zone}${varname}=\${varval}"
+#         fi
+#         ## this part is untested
+#         #read -p "Please type something here:" response < ${thistty}
+#         #echo "${response}"
+#      fi
+#   else
+
+## REACT TO BEING A CRONJOB
+#if test ${is_cronjob} -eq 1 ;
+#then
+#   :
+#else
+#   :
+#fi
+
+# SET TRAPS
+#trap "CTRLC" 2
+#trap "CTRLZ" 18
+trap '__ec=$? ; clean_vooblystats ; trap "" 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 19 20 ; exit ${__ec} ;' 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 18 19 20
+
+# DEBUG SIMPLECONF
+debuglev 5 && {
+   ferror "Using values"
+   # used values: EX_(OPT1|OPT2|VERBOSE)
+   set | grep -iE "^VS_" 1>&2
+}
+
+# MAIN LOOP
+#{
+
+   # Initial authentication and download a random game page
+   rm -f "${VS_COOKIEFILE}"
+   
+   # need to auth at least one for any other voobly function to work correctly.
+   auth_to_voobly "${VS_USERNAME}" "${VS_PASSWORD}" "${VS_COOKIEFILE}"
+   #get_game_page "${opt1}" "${VS_COOKIEFILE}" | parse_game_page stdin
+   #parse_matches_for_user "frodo" "300" "${VS_COOKIEFILE}"
+   display_csv_headers
+
+   #get_user_matchlist "123989133" "150"
+
+   {
+      for word in $( cat_matchlist "${VS_MATCHLIST:-frodo,bilbo,merry,pippin}" "," "300" "${VS_COOKIEFILE}" )
+      do
+         get_game_page "${word}" | parse_game_page stdin
+      done
+   } | sort -k1,3
+
+#} | tee -a ${logfile}
+
+# EMAIL LOGFILE
+#${sendsh} ${sendopts} "${server} ${scriptfile} out" ${logfile} ${interestedparties}
+
+## STOP THE READ CONFIG FILE
+#return_code 0
+#fi ; done ; }