AtlanticAthleticClub/AACResults
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

URL argument cleanup; calculate finishers and category/sex positions.

Browse Source
This commit is contained in:
Timothy Allen 2018-08-14 11:01:26 +02:00
parent 711151b828
commit 6c1e9d82a0
10 changed files with 80 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
aacstats.py
View File

@ -10,25 +10,10 @@ app = Flask(__name__)
PAGE_SIZE=20
MIN_MONTHS_FOR_LISTINGS=3
def now():
return dt.datetime.now()
def getstart():
start = request.args.get('start', '0')
if not isinstance(start, (int, float)):
return 0
return start
def getshow():
show = request.args.get('show', PAGE_SIZE)
if show == 'all':
return -1
if not isinstance(show, (int, float)):
return PAGE_SIZE
return show
@app.template_filter('urlescape')
def urlescape(string):
if string is None:
return ''
return urllib.parse.quote_plus(string)
@app.template_filter('pace')
@ -40,7 +25,7 @@ def year(time):
return time.strftime('%Y')
@app.template_filter('cleandate')
def clean_date(time):
def cleandate(time):
if time.month == 1 and time.day == 1:
return time.strftime('%Y')
return time.strftime('%Y-%m-%d')
@ -51,7 +36,40 @@ def ordinal(n):
return
return "%d%s" % (n,"tsnrhtdd"[(math.floor(n/10)%10!=1)*(n%10<4)*n%10::4])
def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None):
@app.template_filter('cleandict')
def cleandict(dict):
''' Prevent duplication of existing query strings when calling url_for(..., **request.args) '''
newdict = {}
for key, value in dict.items():
if key not in ( 'title', 'year', 'start', 'show' ) and value not in ( None, '', ):
newdict[key] = value
return newdict
def now():
return dt.datetime.now()
def getstart():
start = request.args.get('start', '0')
if not isinstance(start, (int, float)):
try:
return int(start)
except:
return 0
return start
def getshow():
show = request.args.get('show', PAGE_SIZE)
if show == 'all':
return -1
if not isinstance(show, (int, float)):
try:
return int(show)
except:
return PAGE_SIZE
return show
def read_db(listing=None, event=None, person=None, licence=None, search=dict(), year=None, finishers=False):
db = MySQLdb.connect(user='aac', passwd='saOAcCWHg4LaoSSA', db='AAC',
use_unicode=True, charset="utf8", cursorclass=MySQLdb.cursors.DictCursor)
c = db.cursor()
@ -88,6 +106,10 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
firstdate = firstdate.replace(year=int(year))
lastdate = lastdate.replace(year=int(year))
where += ' AND date > "{}" AND date < "{}"'.format(firstdate, lastdate)
''' This statement is expensive but doesn't increase the count, so don't change the count statement '''
if finishers:
select = 'total.finishers, query.* FROM( SELECT *'
close = ') AS query INNER JOIN (SELECT event, date, distance, COUNT(event) as finishers FROM `results` GROUP BY event, distance, date) AS total ON total.event=query.event AND total.date=query.date AND total.distance=query.distance'
for column in search.keys():
if isinstance(column, str):
@ -147,6 +169,7 @@ def read_db(listing=None, event=None, person=None, licence=None, search=dict(),
queryresults = c.fetchall()
select = 'COUNT(*)'
close = ''
if listing:
if listing == 'race':
select = 'COUNT(*) FROM ( SELECT COUNT(event)'
@ -227,7 +250,7 @@ def race(year=None, title=None):
def person(title=None, year=None):
if title is not None:
title = urllib.parse.unquote_plus(title)
results = read_db(person=title, year=year)
results = read_db(person=title, year=year, finishers=True)
return render_template('index.html', ltype='person', title=title,
results=results, year=year,
request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)
@ -236,7 +259,7 @@ def person(title=None, year=None):
def licence(year=now().year, title=None):
if title is not None:
title = urllib.parse.unquote_plus(title)
results = read_db(licence=title, year=year)
results = read_db(licence=title, year=year, finishers=True)
return render_template('index.html', ltype='licence', title=title,
results=results, year=year,
request=request, getstart=getstart(), getshow=getshow(), now=now(), PAGE_SIZE=PAGE_SIZE)

24
load_spreadsheet.py
View File

@ -19,7 +19,6 @@ import argparse
import datetime as dt
import dateutil.parser as dtp
import logging
import uuid
import os
import re
import sys
@ -61,7 +60,7 @@ def main():
soup = bs4.BeautifulSoup(page, 'html.parser')
for event in soup.find_all('tr'):
raceinfo = dict()
link = event.find('a', href=re.compile('.xls$'))
link = event.find('a', href=re.compile('.xlsx?$'))
name = event.find('td', class_=re.compile('EventHeadline'))
date = event.find('td', class_=re.compile('EventDate'))
dist = event.find('td', class_=re.compile('EventDist'), string=re.compile('^\s*[\d+\.]\s*(KM)?\s*$'))
@ -75,6 +74,8 @@ def main():
if dist is not None:
raceinfo['distance'] = dist.string
spreadsheets.append(raceinfo)
#pp.pprint(spreadsheets)
#sys.exit(1)
for race in spreadsheets:
url = race['url']
with urllib.request.urlopen(url) as response, tempfile.TemporaryDirectory() as tmpdir:
@ -92,6 +93,8 @@ def main():
raise
else:
load_into_db(rows)
log.debug("\n")
elif args.input_file:
@ -135,8 +138,8 @@ def load_into_db(rows):
`date` datetime DEFAULT NULL,
`distance` float(10) DEFAULT NULL,
`event` varchar(100) COLLATE utf8_unicode_ci NOT NULL,
`eventuuid` varchar(36) COLLATE utf8_unicode_ci NOT NULL,
`position` int(5) NOT NULL,
`finishers` int(5) DEFAULT NULL,
`time` time NOT NULL,
`name` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
`surname` varchar(75) COLLATE utf8_unicode_ci DEFAULT NULL,
@ -194,8 +197,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
rows = []
filename = os.path.basename(spreadsheet)
if re.search('.xlsx?$', spreadsheet, flags=re.IGNORECASE) is not None:
''' The eventuuid should be unique for this event, but are not derived from the name, and cannot be used to detect duplicate events '''
eventuuid = uuid.uuid4()
book = xlrd.open_workbook(spreadsheet)
for sheetname in book.sheet_names():
sheet = book.sheet_by_name(sheetname)
@ -313,7 +314,6 @@ def read_spreadsheet(spreadsheet, src=None, eventname=None, eventdate=None, even
continue
item['date'] = eventdate
item['event'] = eventname
item['eventuuid'] = eventuuid
item['distance'] = distance
item['source'] = src
rows.append(item)
@ -357,10 +357,10 @@ def clean_data(input_rows):
r['distance'] = length.group(1)
''' Fix sex '''
if 'sex' in ir and re.search('^\sF', str(ir.get('sex')), flags=re.IGNORECASE) is not None:
r['sex'] = 'F'
if 'sex' in ir and re.search('^\s*F', str(ir.get('sex')), flags=re.IGNORECASE) is not None:
r['sex'] = 'female'
else:
r['sex'] = 'M'
r['sex'] = 'male'
''' Fix club '''
if re.search('^\s*(AAC\b|Atlantic\s*Athletic)', str(ir.get('club')), flags=re.IGNORECASE) is not None:
@ -389,14 +389,16 @@ def clean_data(input_rows):
''' Should be a string '''
for key in ( 'event', 'name', 'surname', 'licence', 'club', 'category', 'sex', ):
val = ir.get(key)
if isinstance(val, float):
val = int(val)
if val is not None:
try:
r[key] = str(val)
r[key] = re.sub('(^\s*|\s*$)', '', str(val))
except:
pass
''' Leave alone '''
for key in ( 'event', 'eventuuid', 'source', ):
for key in ( 'event', 'source', ):
r[key] = ir.get(key)
rows.append(r)

3
templates/head.html
View File

@ -11,6 +11,7 @@
{%- set ns.show = getshow -%}
{# Reset arguments, so as not to display standard arguments in the query part of the URL #}
{%- set ns.query = request.args | cleandict -%}
{%- if ns.start == 0 -%}
{%- set ns.start = None -%}
{%- endif -%}
@ -20,4 +21,4 @@
{%- endif -%}
<body>
{% include 'tabs.html' with context %}
{% include 'tabs.html' with context -%}

12
templates/index.html
View File

@ -16,10 +16,8 @@
<thead>
<tr>
<th>Position</th>
{% if ltype != 'person' %}
<th>Name</th>
<th>Licence</th>
{% endif %}
<th>Time</th>
<th>Average Pace</th>
{% if ltype != 'event' %}
@ -31,15 +29,15 @@
</thead>
<tbody>
{%- for row in results['rows'] -%}
{%- set person='{} {}'.format(row.name|e, row.surname|e) -%}
{%- set person = '{} {}'.format(row.name or '', row.surname or '') -%}
<tr>
<td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}</span></td>
<td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|urlescape, start=None) }}">{{ person }}</a></span></td>
<td class="nowrap"><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td>
<td class="nowrap"><span class="label">Position</span> <span>{{ row.position|e }}{% if row.finishers %} / {{ row.finishers }}{% endif %}</span></td>
<td class="nowrap"><span class="label">Name</span> <span><a href="{{ url_for('person', title=person|trim|urlescape, start=None) }}">{{ person|trim|e }}</a></span></td>
<td class="nowrap"><span class="label">Licence</span> <span>{% if row.licence %}<a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a>{% endif %}</span></td>
<td><span class="label">Time</span> <span>{{ row.time|e }}</span></td>
<td class="nowrap"><span class="label">Average Pace</span> <span>{% if row.distance is number and row.distance|int != 0 %}{{ (row.time / row.distance) | pace }} min/KM{% endif %}</span></td>
{%- if ltype != 'event' -%}
<td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }} ({{ row.distance|e }} KM)</a></span></td>
<td class="long"><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }} ({{ row.distance|trim|e }} KM)</a></span></td>
{%- endif -%}
<td class="nowrap"><span class="label">Date</span> <span>{{ row.date|cleandate|e }}</span></td>
<td class="long"><span class="label">Notes</span> <span>

4
templates/list-licence.html
View File

@ -19,8 +19,8 @@
<tbody>
{%- for row in results['rows'] -%}
<tr>
<td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|e }}</a></span></td>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
<td><span class="label">Licence</span> <span><a href="{{ url_for('licence', title=row.licence|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.licence|trim|e }}</a></span></td>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Year</span> <span>{{ row.date | year }}</a></span></td>
</tr>
{%- endfor -%}

2
templates/list-race.html
View File

@ -18,7 +18,7 @@
<tbody>
{%- for row in results['rows'] -%}
<tr>
<td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|e }}</a></span></td>
<td><span class="label">Race</span> <span><a href="{{ url_for('race', title=row.event|trim|urlescape, year=row.date|year, start=None, show=ns.show) }}">{{ row.event|trim|e }}</a></span></td>
<td><span class="label">Date</span> <span>{{ row.date | cleandate }}</span></td>
</tr>
{%- endfor -%}

2
templates/list-rankings.html
View File

@ -20,7 +20,7 @@
<tbody>
{%- for row in results['rows'] -%}
<tr>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Average Position</span> <span>{{ row.score|e }}</span></td>
<!--td><span class="label">Sum of Race Positions</span> <span>{{ row.positions|e }}</span></td-->
<td><span class="label">Number of Races</span> <span>{{ row.races|e }}</span></td>

2
templates/list-runners.html
View File

@ -18,7 +18,7 @@
<tbody>
{%- for row in results['rows'] -%}
<tr>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|e }}</a></span></td>
<td><span class="label">Name</span> <span><a href="{{ url_for('person', title=row.person|trim|urlescape, year=year, start=None, show=ns.show) }}">{{ row.person|trim|e }}</a></span></td>
<td><span class="label">Distance</span> <span>{{ row.total|e }}</span></td>
</tr>
{%- endfor -%}

10
templates/prevnext.html
View File

@ -14,16 +14,16 @@
{%- set ns.ellipsis = False -%}
{% if thispage > 1 %}
<span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=show) }}">&laquo; First</a></span>
<span class="first"><a href="{{ url_for(request.endpoint, title=title, year=year, start=None, show=ns.show, **ns.query) }}">&laquo; First</a></span>
{% if (getstart - getshow) > 1 %}
<span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=show) }}">&lsaquo; Prev</a></span>
<span class="prev"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.prev, show=ns.show, **ns.query) }}">&lsaquo; Prev</a></span>
{% endif %}
{% endif %}
{% for page in range(1, totalpages+1) %}
{% if page < 4 or page > (totalpages+1-4) or (page > (thispage-3) and page < (thispage+3)) %}
{%- if page != thispage -%}
<span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=show) }}">{{ page }}</a></span>
<span class="nav link"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(page - 1) * getshow, show=ns.show, **ns.query) }}">{{ page }}</a></span>
{%- else -%}
<span class="nav plain"><strong>{{ page }}</strong></span>
{%- endif %}
@ -38,8 +38,8 @@
{% if thispage < totalpages %}
{% if (getstart + getshow) != (totalpages - 1) * getshow %}
<span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=show) }}">Next &rsaquo;</a></span>
<span class="next"><a href="{{ url_for(request.endpoint, title=title, year=year, start=ns.next, show=ns.show, **ns.query) }}">Next &rsaquo;</a></span>
{% endif %}
<span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=show) }}">Last &raquo;</a></span>
<span class="last"><a href="{{ url_for(request.endpoint, title=title, year=year, start=(totalpages - 1) * getshow, show=ns.show, **ns.query) }}">Last &raquo;</a></span>
{% endif %}
</nav>

2
templates/search.html
View File

@ -24,6 +24,6 @@
<label for="searchposition">Position</label>
<input type="text" id="searchposition" name="position" value="{{ request.args.get('position', '') }}" />
</div-->
<div><input type="submit" id="searchsubmit" name="submit" value="Search" /></div>
<div><input type="submit" id="searchsubmit" value="Search" /></div>
</div>
</form>