URL argument cleanup; calculate finishers and category/sex positions.

aacstats.py

@@ -10,25 +10,10 @@ app = Flask(__name__)
 PAGE_SIZE=20
 MIN_MONTHS_FOR_LISTINGS=3
 
-def now():
-    return dt.datetime.now()
-
-def getstart():
-    start = request.args.get('start', '0')
-    if not isinstance(start, (int, float)):
-        return 0
-    return start
-
-def getshow():
-    show = request.args.get('show', PAGE_SIZE)
-    if show == 'all':
-        return -1
-    if not isinstance(show, (int, float)):
-        return PAGE_SIZE
-    return show
-
 @app.template_filter('urlescape')
 def urlescape(string):
+    if string is None:
+        return ''
     return urllib.parse.quote_plus(string)
 
 @app.template_filter('pace')
@@ -40,7 +25,7 @@ def year(time):
     return time.strftime('%Y')
 
 @app.template_filter('cleandate')
-def clean_date(time):
+def cleandate(time):
     if time.month == 1 and time.day == 1:
         return time.strftime('%Y')
     return time.strftime('%Y-%m-%d')
@@ -51,7 +36,40 @@ def ordinal(n):
         return
     return "%d%s" % (n,"tsnrhtdd"[(math.floor(n/10)%10!=1)*(n%10<4)*n%10::4])
 
-def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None):
+@app.template_filter('cleandict')
+def cleandict(dict):
+    ''' Prevent duplication of existing query strings when calling url_for(..., **request.args) '''
+    newdict = {}
+    for key, value in dict.items():
+        if key not in ( 'title', 'year', 'start', 'show' ) and value not in ( None, '', ):
+             newdict[key] = value
+    return newdict
+
+def now():
+    return dt.datetime.now()
+
+def getstart():
+    start = request.args.get('start', '0')
+    if not isinstance(start, (int, float)):
+        try:
+            return int(start)
+        except:
+            return 0
+    return start
+
+def getshow():
+    show = request.args.get('show', PAGE_SIZE)
+    if show == 'all':
+        return -1
+    if not isinstance(show, (int, float)):
+        try:
+            return int(show)
+        except:
+            return PAGE_SIZE
+    return show
+
+
+def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None, finishers=False):
     db = MySQLdb.connect(user='aac', passwd='saOAcCWHg4LaoSSA', db='AAC',
                          use_unicode=True, charset="utf8", cursorclass=MySQLdb.cursors.DictCursor)
     c =  db.cursor()
@@ -88,6 +106,10 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
         firstdate = firstdate.replace(year=int(year))
         lastdate  = lastdate.replace(year=int(year))
         where += ' AND date > "{}" AND date < "{}"'.format(firstdate, lastdate)
+    ''' This statement is expensive but doesn't increase the count, so don't change the count statement '''
+    if finishers: 
+        select = 'total.finishers, query.* FROM( SELECT *'
+        close =   ') AS query INNER JOIN (SELECT event, date, distance, COUNT(event) as finishers FROM `results` GROUP BY event, distance, date) AS total ON total.event=query.event AND total.date=query.date AND total.distance=query.distance'
 
     for column in search.keys():
         if isinstance(column, str):
@@ -147,6 +169,7 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
     queryresults = c.fetchall()
 
     select = 'COUNT(*)'
+    close  = ''
     if listing:
         if listing == 'race':
             select = 'COUNT(*) FROM ( SELECT COUNT(event)'
@@ -227,7 +250,7 @@ def race(year=None, title=None):
 def person(title=None, year=None):
     if title is not None:
         title = urllib.parse.unquote_plus(title)
-    results = read_db(person=title, year=year)
+    results = read_db(person=title, year=year, finishers=True)
     return render_template('index.html', ltype='person', title=title,
                             results=results, year=year, 
                             request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)
@@ -236,7 +259,7 @@ def person(title=None, year=None):
 def licence(year=now().year, title=None):
     if title is not None:
         title = urllib.parse.unquote_plus(title)
-    results = read_db(licence=title, year=year)
+    results = read_db(licence=title, year=year, finishers=True)
     return render_template('index.html', ltype='licence', title=title,
                             results=results, year=year, 
                             request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)

load_spreadsheet.py

@@ -19,7 +19,6 @@ import argparse
 import datetime as dt
 import dateutil.parser as dtp
 import logging
-import uuid
 import os
 import re
 import sys
@@ -61,7 +60,7 @@ def main():
         soup = bs4.BeautifulSoup(page, 'html.parser')
         for event in soup.find_all('tr'):
           raceinfo = dict()
-          link = event.find('a', href=re.compile('.xls$'))
+          link = event.find('a', href=re.compile('.xlsx?$'))
           name = event.find('td', class_=re.compile('EventHeadline'))
           date = event.find('td', class_=re.compile('EventDate'))
           dist = event.find('td', class_=re.compile('EventDist'), string=re.compile('^\s*[\d+\.]\s*(KM)?\s*$'))
@@ -75,6 +74,8 @@ def main():
               if dist is not None:
                 raceinfo['distance'] = dist.string
               spreadsheets.append(raceinfo)
+    #pp.pprint(spreadsheets)
+    #sys.exit(1)
     for race in spreadsheets:
       url = race['url']
       with urllib.request.urlopen(url) as response, tempfile.TemporaryDirectory() as tmpdir:
@@ -92,6 +93,8 @@ def main():
             raise
           else:
             load_into_db(rows)
+      log.debug("\n")
+
 
       
   elif args.input_file:
@@ -135,8 +138,8 @@ def load_into_db(rows):
       `date` datetime DEFAULT NULL,
       `distance` float(10) DEFAULT NULL,
       `event` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
-      `eventuuid` varchar(36) COLLATE utf8_unicode_ci NOT NULL,
       `position` int(5) NOT NULL,
+      `finishers` int(5) DEFAULT NULL,
       `time` time NOT NULL,
       `name` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
       `surname` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
@@ -194,8 +197,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
   rows = []
   filename = os.path.basename(spreadsheet)
   if re.search('.xlsx?$', spreadsheet, flags=re.IGNORECASE) is not None:
-    ''' The eventuuid should be unique for this event, but are not derived from the name, and cannot be used to detect duplicate events '''
-    eventuuid = uuid.uuid4()
     book = xlrd.open_workbook(spreadsheet)
     for sheetname in book.sheet_names():
       sheet = book.sheet_by_name(sheetname)
@@ -313,7 +314,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
               continue
           item['date']      = eventdate
           item['event']     = eventname
-          item['eventuuid'] = eventuuid
           item['distance']  = distance
           item['source']    = src
           rows.append(item)
@@ -357,10 +357,10 @@ def clean_data(input_rows):
       r['distance'] = length.group(1)
 
     ''' Fix sex '''
-    if 'sex' in ir and re.search('^\sF', str(ir.get('sex')), flags=re.IGNORECASE) is not None:
+    if 'sex' in ir and re.search('^\s*F', str(ir.get('sex')), flags=re.IGNORECASE) is not None:
-      r['sex'] = 'F'
+      r['sex'] = 'female'
     else:
-      r['sex'] = 'M'
+      r['sex'] = 'male'
 
     ''' Fix club '''
     if re.search('^\s*(AAC\b|Atlantic\s*Athletic)', str(ir.get('club')), flags=re.IGNORECASE) is not None:
@@ -389,14 +389,16 @@ def clean_data(input_rows):
     ''' Should be a string '''
     for key in ( 'event', 'name', 'surname', 'licence', 'club', 'category', 'sex', ):
       val = ir.get(key)
+      if isinstance(val, float):
+        val = int(val)
       if val is not None:
         try:
-          r[key] = str(val)
+          r[key] = re.sub('(^\s*|\s*$)', '', str(val))
         except:
           pass
 
     ''' Leave alone '''
-    for key in ( 'event', 'eventuuid', 'source', ):
+    for key in ( 'event', 'source', ):
       r[key] = ir.get(key)
 
     rows.append(r)

templates/head.html

@@ -11,6 +11,7 @@
 {%- set ns.show = getshow -%}
 
 {# Reset arguments, so as not to display standard arguments in the query part of the URL #}
+{%- set ns.query = request.args | cleandict -%}
 {%- if ns.start == 0 -%}
   {%- set ns.start = None -%}
 {%- endif -%}
@@ -20,4 +21,4 @@
 {%- endif -%}
 
 <body>
-{% include 'tabs.html' with context %}
+{% include 'tabs.html' with context -%}

templates/index.html

@@ -16,10 +16,8 @@
     <thead>
       <tr>
         <th>Position</th>
-       {% if ltype != 'person' %}
         <th>Name</th>
         <th>Licence</th>
-       {% endif %}
         <th>Time</th>
         <th>Average Pace</th>
        {% if ltype != 'event' %}
@@ -31,15 +29,15 @@
     </thead>
     <tbody>
   {%- for row in results['rows'] -%}
-     {%- set person='{} {}'.format(row.name|e, row.surname|e) -%}
+     {%- set person = '{} {}'.format(row.name or '', row.surname or '') -%}
       <tr>
-         <td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}</span></td>
+         <td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}{% if row.finishers %} / {{ row.finishers }}{% endif %}</span></td>
-         <td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|urlescape, start=None) }}">{{ person }}</a></span></td>
+         <td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|trim|urlescape, start=None) }}">{{ person|trim|e }}</a></span></td>
-         <td class="nowrap"><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td>
+         <td class="nowrap"><span class="label">Licence</span> <span>{% if row.licence %}<a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a>{% endif %}</span></td>
          <td><span class="label">Time</span> <span>{{ row.time|e }}</span></td>
          <td class="nowrap"><span class="label">Average Pace</span> <span>{% if row.distance is number and row.distance|int != 0 %}{{ (row.time / row.distance) | pace }} min/KM{% endif %}</span></td>
        {%- if ltype != 'event' -%}
-         <td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }} ({{ row.distance|e }} KM)</a></span></td>
+         <td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }} ({{ row.distance|trim|e }} KM)</a></span></td>
        {%- endif -%}
          <td class="nowrap"><span class="label">Date</span> <span>{{ row.date|cleandate|e }}</span></td>
          <td class="long"><span class="label">Notes</span> <span>

templates/list-licence.html

@@ -19,8 +19,8 @@
     <tbody>
   {%- for row in results['rows'] -%}
       <tr>
-         <td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td>
+         <td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a></span></td>
-         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
+         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
          <td><span class="label">Year</span> <span>{{ row.date | year }}</a></span></td>
       </tr>
   {%- endfor -%}

templates/list-race.html

@@ -18,7 +18,7 @@
     <tbody>
   {%- for row in results['rows'] -%}
       <tr>
-         <td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }}</a></span></td>
+         <td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }}</a></span></td>
          <td><span class="label">Date</span> <span>{{ row.date | cleandate }}</span></td>
       </tr>
   {%- endfor -%}

templates/list-rankings.html

@@ -20,7 +20,7 @@
     <tbody>
   {%- for row in results['rows'] -%}
       <tr>
-         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
+         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
          <td><span class="label">Average Position</span> <span>{{ row.score|e }}</span></td>
          <!--td><span class="label">Sum of Race Positions</span> <span>{{ row.positions|e }}</span></td-->
          <td><span class="label">Number of Races</span> <span>{{ row.races|e }}</span></td>

templates/list-runners.html

@@ -18,7 +18,7 @@
     <tbody>
   {%- for row in results['rows'] -%}
       <tr>
-         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
+         <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
          <td><span class="label">Distance</span> <span>{{ row.total|e }}</span></td>
       </tr>
   {%- endfor -%}

templates/prevnext.html

@@ -14,16 +14,16 @@
   {%- set ns.ellipsis = False -%}
 
   {% if thispage > 1 %}
-    <span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=show) }}">&laquo; First</a></span>
+    <span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=ns.show, **ns.query) }}">&laquo; First</a></span>
     {% if (getstart - getshow) > 1 %}
-      <span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=show) }}">&lsaquo; Prev</a></span>
+      <span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=ns.show, **ns.query) }}">&lsaquo; Prev</a></span>
     {% endif %}
   {% endif %}
 
   {% for page in range(1, totalpages+1) %}
     {% if page < 4 or page > (totalpages+1-4) or (page > (thispage-3) and page < (thispage+3)) %}
       {%- if page != thispage -%}
-        <span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=show) }}">{{ page }}</a></span>
+        <span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=ns.show, **ns.query) }}">{{ page }}</a></span>
       {%- else -%}
         <span class="nav plain"><strong>{{ page }}</strong></span>
       {%- endif %}
@@ -38,8 +38,8 @@
 
   {% if thispage < totalpages %}
     {% if (getstart + getshow) != (totalpages - 1) * getshow %}
-      <span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=show) }}">Next &rsaquo;</a></span>
+      <span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=ns.show, **ns.query) }}">Next &rsaquo;</a></span>
     {% endif %}
-    <span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=show) }}">Last &raquo;</a></span>
+    <span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=ns.show, **ns.query) }}">Last &raquo;</a></span>
   {% endif %}
 </nav>

templates/search.html

@@ -24,6 +24,6 @@
       <label for="searchposition">Position</label>
       <input type="text" id="searchposition" name="position" value="{{ request.args.get('position', '') }}" />
     </div-->
-    <div><input type="submit" id="searchsubmit" name="submit" value="Search" /></div>
+    <div><input type="submit" id="searchsubmit" value="Search" /></div>
   </div>
 </form>