AtlanticAthleticClub/AACResults
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

URL argument cleanup; calculate finishers and category/sex positions.

Browse Source
This commit is contained in:
Timothy Allen 2018-08-14 11:01:26 +02:00
parent 711151b828
commit 6c1e9d82a0
10 changed files with 80 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
aacstats.py
View File

@ -10,25 +10,10 @@ app = Flask(__name__)
PAGE_SIZE=20 PAGE_SIZE=20
MIN_MONTHS_FOR_LISTINGS=3 MIN_MONTHS_FOR_LISTINGS=3
def now():
return dt.datetime.now()
def getstart():
start = request.args.get('start', '0')
if not isinstance(start, (int, float)):
return 0
return start
def getshow():
show = request.args.get('show', PAGE_SIZE)
if show == 'all':
return -1
if not isinstance(show, (int, float)):
return PAGE_SIZE
return show
@app.template_filter('urlescape') @app.template_filter('urlescape')
def urlescape(string): def urlescape(string):
if string is None:
return ''
return urllib.parse.quote_plus(string) return urllib.parse.quote_plus(string)
@app.template_filter('pace') @app.template_filter('pace')
@ -40,7 +25,7 @@ def year(time):
return time.strftime('%Y') return time.strftime('%Y')
@app.template_filter('cleandate') @app.template_filter('cleandate')
def clean_date(time): def cleandate(time):
if time.month == 1 and time.day == 1: if time.month == 1 and time.day == 1:
return time.strftime('%Y') return time.strftime('%Y')
return time.strftime('%Y-%m-%d') return time.strftime('%Y-%m-%d')
@ -51,7 +36,40 @@ def ordinal(n):
return return
return "%d%s" % (n,"tsnrhtdd"[(math.floor(n/10)%10!=1)*(n%10<4)*n%10::4]) return "%d%s" % (n,"tsnrhtdd"[(math.floor(n/10)%10!=1)*(n%10<4)*n%10::4])
def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None): @app.template_filter('cleandict')
def cleandict(dict):
''' Prevent duplication of existing query strings when calling url_for(..., **request.args) '''
newdict = {}
for key, value in dict.items():
if key not in ( 'title', 'year', 'start', 'show' ) and value not in ( None, '', ):
newdict[key] = value
return newdict
def now():
return dt.datetime.now()
def getstart():
start = request.args.get('start', '0')
if not isinstance(start, (int, float)):
try:
return int(start)
except:
return 0
return start
def getshow():
show = request.args.get('show', PAGE_SIZE)
if show == 'all':
return -1
if not isinstance(show, (int, float)):
try:
return int(show)
except:
return PAGE_SIZE
return show
def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None, finishers=False):
db = MySQLdb.connect(user='aac', passwd='saOAcCWHg4LaoSSA', db='AAC', db = MySQLdb.connect(user='aac', passwd='saOAcCWHg4LaoSSA', db='AAC',
use_unicode=True, charset="utf8", cursorclass=MySQLdb.cursors.DictCursor) use_unicode=True, charset="utf8", cursorclass=MySQLdb.cursors.DictCursor)
c = db.cursor() c = db.cursor()
@ -60,11 +78,11 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
show = getshow() show = getshow()
select = '*' select = '*'
close = '' close = ''
where = 'WHERE club LIKE "AAC"' where = 'WHERE club LIKE "AAC"'
group = '' group = ''
order = 'date DESC, event, position' order = 'date DESC, event, position'
limit = 'LIMIT {},{}'.format(start, show) limit = 'LIMIT {},{}'.format(start, show)
if show == -1: if show == -1:
limit = '' limit = ''
@ -88,6 +106,10 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
firstdate = firstdate.replace(year=int(year)) firstdate = firstdate.replace(year=int(year))
lastdate = lastdate.replace(year=int(year)) lastdate = lastdate.replace(year=int(year))
where += ' AND date > "{}" AND date < "{}"'.format(firstdate, lastdate) where += ' AND date > "{}" AND date < "{}"'.format(firstdate, lastdate)
''' This statement is expensive but doesn't increase the count, so don't change the count statement '''
if finishers:
select = 'total.finishers, query.* FROM( SELECT *'
close = ') AS query INNER JOIN (SELECT event, date, distance, COUNT(event) as finishers FROM `results` GROUP BY event, distance, date) AS total ON total.event=query.event AND total.date=query.date AND total.distance=query.distance'
for column in search.keys(): for column in search.keys():
if isinstance(column, str): if isinstance(column, str):
@ -147,6 +169,7 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
queryresults = c.fetchall() queryresults = c.fetchall()
select = 'COUNT(*)' select = 'COUNT(*)'
close = ''
if listing: if listing:
if listing == 'race': if listing == 'race':
select = 'COUNT(*) FROM ( SELECT COUNT(event)' select = 'COUNT(*) FROM ( SELECT COUNT(event)'
@ -227,7 +250,7 @@ def race(year=None, title=None):
def person(title=None, year=None): def person(title=None, year=None):
if title is not None: if title is not None:
title = urllib.parse.unquote_plus(title) title = urllib.parse.unquote_plus(title)
results = read_db(person=title, year=year) results = read_db(person=title, year=year, finishers=True)
return render_template('index.html', ltype='person', title=title, return render_template('index.html', ltype='person', title=title,
results=results, year=year, results=results, year=year,
request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE) request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)
@ -236,7 +259,7 @@ def person(title=None, year=None):
def licence(year=now().year, title=None): def licence(year=now().year, title=None):
if title is not None: if title is not None:
title = urllib.parse.unquote_plus(title) title = urllib.parse.unquote_plus(title)
results = read_db(licence=title, year=year) results = read_db(licence=title, year=year, finishers=True)
return render_template('index.html', ltype='licence', title=title, return render_template('index.html', ltype='licence', title=title,
results=results, year=year, results=results, year=year,
request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE) request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)

24
load_spreadsheet.py
View File

@ -19,7 +19,6 @@ import argparse
import datetime as dt import datetime as dt
import dateutil.parser as dtp import dateutil.parser as dtp
import logging import logging
import uuid
import os import os
import re import re
import sys import sys
@ -61,7 +60,7 @@ def main():
soup = bs4.BeautifulSoup(page, 'html.parser') soup = bs4.BeautifulSoup(page, 'html.parser')
for event in soup.find_all('tr'): for event in soup.find_all('tr'):
raceinfo = dict() raceinfo = dict()
link = event.find('a', href=re.compile('.xls$')) link = event.find('a', href=re.compile('.xlsx?$'))
name = event.find('td', class_=re.compile('EventHeadline')) name = event.find('td', class_=re.compile('EventHeadline'))
date = event.find('td', class_=re.compile('EventDate')) date = event.find('td', class_=re.compile('EventDate'))
dist = event.find('td', class_=re.compile('EventDist'), string=re.compile('^\s*[\d+\.]\s*(KM)?\s*$')) dist = event.find('td', class_=re.compile('EventDist'), string=re.compile('^\s*[\d+\.]\s*(KM)?\s*$'))
@ -75,6 +74,8 @@ def main():
if dist is not None: if dist is not None:
raceinfo['distance'] = dist.string raceinfo['distance'] = dist.string
spreadsheets.append(raceinfo) spreadsheets.append(raceinfo)
#pp.pprint(spreadsheets)
#sys.exit(1)
for race in spreadsheets: for race in spreadsheets:
url = race['url'] url = race['url']
with urllib.request.urlopen(url) as response, tempfile.TemporaryDirectory() as tmpdir: with urllib.request.urlopen(url) as response, tempfile.TemporaryDirectory() as tmpdir:
@ -92,6 +93,8 @@ def main():
raise raise
else: else:
load_into_db(rows) load_into_db(rows)
log.debug("\n")
elif args.input_file: elif args.input_file:
@ -135,8 +138,8 @@ def load_into_db(rows):
`date` datetime DEFAULT NULL, `date` datetime DEFAULT NULL,
`distance` float(10) DEFAULT NULL, `distance` float(10) DEFAULT NULL,
`event` varchar(100) COLLATE utf8_unicode_ci NOT NULL, `event` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`eventuuid` varchar(36) COLLATE utf8_unicode_ci NOT NULL,
`position` int(5) NOT NULL, `position` int(5) NOT NULL,
`finishers` int(5) DEFAULT NULL,
`time` time NOT NULL, `time` time NOT NULL,
`name` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL, `name` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
`surname` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL, `surname` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
@ -194,8 +197,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
rows = [] rows = []
filename = os.path.basename(spreadsheet) filename = os.path.basename(spreadsheet)
if re.search('.xlsx?$', spreadsheet, flags=re.IGNORECASE) is not None: if re.search('.xlsx?$', spreadsheet, flags=re.IGNORECASE) is not None:
''' The eventuuid should be unique for this event, but are not derived from the name, and cannot be used to detect duplicate events '''
eventuuid = uuid.uuid4()
book = xlrd.open_workbook(spreadsheet) book = xlrd.open_workbook(spreadsheet)
for sheetname in book.sheet_names(): for sheetname in book.sheet_names():
sheet = book.sheet_by_name(sheetname) sheet = book.sheet_by_name(sheetname)
@ -313,7 +314,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
continue continue
item['date'] = eventdate item['date'] = eventdate
item['event'] = eventname item['event'] = eventname
item['eventuuid'] = eventuuid
item['distance'] = distance item['distance'] = distance
item['source'] = src item['source'] = src
rows.append(item) rows.append(item)
@ -357,10 +357,10 @@ def clean_data(input_rows):
r['distance'] = length.group(1) r['distance'] = length.group(1)
''' Fix sex ''' ''' Fix sex '''
if 'sex' in ir and re.search('^\sF', str(ir.get('sex')), flags=re.IGNORECASE) is not None: if 'sex' in ir and re.search('^\s*F', str(ir.get('sex')), flags=re.IGNORECASE) is not None:
r['sex'] = 'F' r['sex'] = 'female'
else: else:
r['sex'] = 'M' r['sex'] = 'male'
''' Fix club ''' ''' Fix club '''
if re.search('^\s*(AAC\b|Atlantic\s*Athletic)', str(ir.get('club')), flags=re.IGNORECASE) is not None: if re.search('^\s*(AAC\b|Atlantic\s*Athletic)', str(ir.get('club')), flags=re.IGNORECASE) is not None:
@ -389,14 +389,16 @@ def clean_data(input_rows):
''' Should be a string ''' ''' Should be a string '''
for key in ( 'event', 'name', 'surname', 'licence', 'club', 'category', 'sex', ): for key in ( 'event', 'name', 'surname', 'licence', 'club', 'category', 'sex', ):
val = ir.get(key) val = ir.get(key)
if isinstance(val, float):
val = int(val)
if val is not None: if val is not None:
try: try:
r[key] = str(val) r[key] = re.sub('(^\s*|\s*$)', '', str(val))
except: except:
pass pass
''' Leave alone ''' ''' Leave alone '''
for key in ( 'event', 'eventuuid', 'source', ): for key in ( 'event', 'source', ):
r[key] = ir.get(key) r[key] = ir.get(key)
rows.append(r) rows.append(r)

3
templates/head.html
View File

@ -11,6 +11,7 @@
{%- set ns.show = getshow -%} {%- set ns.show = getshow -%}
{# Reset arguments, so as not to display standard arguments in the query part of the URL #} {# Reset arguments, so as not to display standard arguments in the query part of the URL #}
{%- set ns.query = request.args | cleandict -%}
{%- if ns.start == 0 -%} {%- if ns.start == 0 -%}
{%- set ns.start = None -%} {%- set ns.start = None -%}
{%- endif -%} {%- endif -%}
@ -20,4 +21,4 @@
{%- endif -%} {%- endif -%}
<body> <body>
{% include 'tabs.html' with context %} {% include 'tabs.html' with context -%}

12
templates/index.html
View File

@ -16,10 +16,8 @@
<thead> <thead>
<tr> <tr>
<th>Position</th> <th>Position</th>
{% if ltype != 'person' %}
<th>Name</th> <th>Name</th>
<th>Licence</th> <th>Licence</th>
{% endif %}
<th>Time</th> <th>Time</th>
<th>Average Pace</th> <th>Average Pace</th>
{% if ltype != 'event' %} {% if ltype != 'event' %}
@ -31,15 +29,15 @@
</thead> </thead>
<tbody> <tbody>
{%- for row in results['rows'] -%} {%- for row in results['rows'] -%}
{%- set person='{} {}'.format(row.name|e, row.surname|e) -%} {%- set person = '{} {}'.format(row.name or '', row.surname or '') -%}
<tr> <tr>
<td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}</span></td> <td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}{% if row.finishers %} / {{ row.finishers }}{% endif %}</span></td>
<td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|urlescape, start=None) }}">{{ person }}</a></span></td> <td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|trim|urlescape, start=None) }}">{{ person|trim|e }}</a></span></td>
<td class="nowrap"><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td> <td class="nowrap"><span class="label">Licence</span> <span>{% if row.licence %}<a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a>{% endif %}</span></td>
<td><span class="label">Time</span> <span>{{ row.time|e }}</span></td> <td><span class="label">Time</span> <span>{{ row.time|e }}</span></td>
<td class="nowrap"><span class="label">Average Pace</span> <span>{% if row.distance is number and row.distance|int != 0 %}{{ (row.time / row.distance) | pace }} min/KM{% endif %}</span></td> <td class="nowrap"><span class="label">Average Pace</span> <span>{% if row.distance is number and row.distance|int != 0 %}{{ (row.time / row.distance) | pace }} min/KM{% endif %}</span></td>
{%- if ltype != 'event' -%} {%- if ltype != 'event' -%}
<td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }} ({{ row.distance|e }} KM)</a></span></td> <td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }} ({{ row.distance|trim|e }} KM)</a></span></td>
{%- endif -%} {%- endif -%}
<td class="nowrap"><span class="label">Date</span> <span>{{ row.date|cleandate|e }}</span></td> <td class="nowrap"><span class="label">Date</span> <span>{{ row.date|cleandate|e }}</span></td>
<td class="long"><span class="label">Notes</span> <span> <td class="long"><span class="label">Notes</span> <span>

4
templates/list-licence.html
View File

@ -19,8 +19,8 @@
<tbody> <tbody>
{%- for row in results['rows'] -%} {%- for row in results['rows'] -%}
<tr> <tr>
<td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td> <td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a></span></td>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td> <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Year</span> <span>{{ row.date | year }}</a></span></td> <td><span class="label">Year</span> <span>{{ row.date | year }}</a></span></td>
</tr> </tr>
{%- endfor -%} {%- endfor -%}

2
templates/list-race.html
View File

@ -18,7 +18,7 @@
<tbody> <tbody>
{%- for row in results['rows'] -%} {%- for row in results['rows'] -%}
<tr> <tr>
<td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }}</a></span></td> <td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }}</a></span></td>
<td><span class="label">Date</span> <span>{{ row.date | cleandate }}</span></td> <td><span class="label">Date</span> <span>{{ row.date | cleandate }}</span></td>
</tr> </tr>
{%- endfor -%} {%- endfor -%}

2
templates/list-rankings.html
View File

@ -20,7 +20,7 @@
<tbody> <tbody>
{%- for row in results['rows'] -%} {%- for row in results['rows'] -%}
<tr> <tr>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td> <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Average Position</span> <span>{{ row.score|e }}</span></td> <td><span class="label">Average Position</span> <span>{{ row.score|e }}</span></td>
<!--td><span class="label">Sum of Race Positions</span> <span>{{ row.positions|e }}</span></td--> <!--td><span class="label">Sum of Race Positions</span> <span>{{ row.positions|e }}</span></td-->
<td><span class="label">Number of Races</span> <span>{{ row.races|e }}</span></td> <td><span class="label">Number of Races</span> <span>{{ row.races|e }}</span></td>

2
templates/list-runners.html
View File

@ -18,7 +18,7 @@
<tbody> <tbody>
{%- for row in results['rows'] -%} {%- for row in results['rows'] -%}
<tr> <tr>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td> <td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Distance</span> <span>{{ row.total|e }}</span></td> <td><span class="label">Distance</span> <span>{{ row.total|e }}</span></td>
</tr> </tr>
{%- endfor -%} {%- endfor -%}

10
templates/prevnext.html
View File

@ -14,16 +14,16 @@
{%- set ns.ellipsis = False -%} {%- set ns.ellipsis = False -%}
{% if thispage > 1 %} {% if thispage > 1 %}
<span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=show) }}">&laquo; First</a></span> <span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=ns.show, **ns.query) }}">&laquo; First</a></span>
{% if (getstart - getshow) > 1 %} {% if (getstart - getshow) > 1 %}
<span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=show) }}">&lsaquo; Prev</a></span> <span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=ns.show, **ns.query) }}">&lsaquo; Prev</a></span>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% for page in range(1, totalpages+1) %} {% for page in range(1, totalpages+1) %}
{% if page < 4 or page > (totalpages+1-4) or (page > (thispage-3) and page < (thispage+3)) %} {% if page < 4 or page > (totalpages+1-4) or (page > (thispage-3) and page < (thispage+3)) %}
{%- if page != thispage -%} {%- if page != thispage -%}
<span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=show) }}">{{ page }}</a></span> <span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=ns.show, **ns.query) }}">{{ page }}</a></span>
{%- else -%} {%- else -%}
<span class="nav plain"><strong>{{ page }}</strong></span> <span class="nav plain"><strong>{{ page }}</strong></span>
{%- endif %} {%- endif %}
@ -38,8 +38,8 @@
{% if thispage < totalpages %} {% if thispage < totalpages %}
{% if (getstart + getshow) != (totalpages - 1) * getshow %} {% if (getstart + getshow) != (totalpages - 1) * getshow %}
<span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=show) }}">Next &rsaquo;</a></span> <span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=ns.show, **ns.query) }}">Next &rsaquo;</a></span>
{% endif %} {% endif %}
<span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=show) }}">Last &raquo;</a></span> <span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=ns.show, **ns.query) }}">Last &raquo;</a></span>
{% endif %} {% endif %}
</nav> </nav>

2
templates/search.html
View File

@ -24,6 +24,6 @@
<label for="searchposition">Position</label> <label for="searchposition">Position</label>
<input type="text" id="searchposition" name="position" value="{{ request.args.get('position', '') }}" /> <input type="text" id="searchposition" name="position" value="{{ request.args.get('position', '') }}" />
</div--> </div-->
<div><input type="submit" id="searchsubmit" name="submit" value="Search" /></div> <div><input type="submit" id="searchsubmit" value="Search" /></div>
</div> </div>
</form> </form>