Remove defaults, move settings to configuration file

This commit is contained in:
Christopher Teutsch 2019-05-18 22:54:37 +02:00
parent aaaf0e748b
commit 507ca82a1e
Signed by: iwonder
GPG Key ID: 0EE33D788D50130D
2 changed files with 63 additions and 28 deletions

View File

@ -2,11 +2,12 @@
import requests import requests
import json import json
from typing import List from typing import List
from pprint import pprint, pformat # from pprint import pprint, pformat
import datetime import datetime
import pause import pause
import sys import sys
import mysql.connector import mysql.connector
import configparser
TABLE = """ TABLE = """
CREATE TABLE IF NOT EXISTS vrr ( CREATE TABLE IF NOT EXISTS vrr (
@ -15,15 +16,10 @@ direction_code varchar(1) not null,
station_id int not null, station_id int not null,
orig_datetime datetime not null, orig_datetime datetime not null,
status enum('on_time', 'early', 'late', 'cancelled', 'no_data') not null, status enum('on_time', 'early', 'late', 'cancelled', 'no_data') not null,
delay_value int default null, delay_value int,
primary key (line_code, direction_code, station_id, orig_datetime)); primary key (line_code, direction_code, station_id, orig_datetime));
""" """
# CONFIGURATION
USE_MODES = []
USE_STATION_ID = 20021002
USE_LINES = []
class MOT: class MOT:
LONG_DISTANCE_TRAIN = 0 LONG_DISTANCE_TRAIN = 0
@ -36,16 +32,41 @@ class MOT:
ALL_MODES = [LONG_DISTANCE_TRAIN, REGIONAL_TRAIN, COMMUTER_TRAIN, UNDERGROUND_TRAIN, TRAM, BUS, ELEVATED_TRAIN] ALL_MODES = [LONG_DISTANCE_TRAIN, REGIONAL_TRAIN, COMMUTER_TRAIN, UNDERGROUND_TRAIN, TRAM, BUS, ELEVATED_TRAIN]
# Parse the configuration file:
cfg = configparser.ConfigParser()
cfg.read('vrr.ini')
db_config = {
'user': cfg['db']['user'],
'password': cfg['db']['pass'],
'host': cfg['db']['host'],
'database': cfg['db']['database'],
}
USE_MODES = []
if cfg['crawl']['use_long_distance']:
USE_MODES.append(MOT.LONG_DISTANCE_TRAIN)
if cfg['crawl']['use_regional_trains']:
USE_MODES.append(MOT.REGIONAL_TRAIN)
if cfg['crawl']['use_commuter_trains']:
USE_MODES.append(MOT.COMMUTER_TRAIN)
if cfg['crawl']['use_trams']:
USE_MODES.append(MOT.TRAM)
if cfg['crawl']['use_buses']:
USE_MODES.append(MOT.BUS)
if cfg['crawl']['use_elevated_trains']:
USE_MODES.append(MOT.ELEVATED_TRAIN)
if cfg['crawl']['station_id'] is not None:
USE_STATION_ID = cfg['crawl']['station_id']
else:
sys.exit("Please specify a station_id in the [crawl] section of vrr.ini")
USE_LINES = cfg['crawl']['use_lines'].split(',')
ALL_LINES = [] ALL_LINES = []
TRIP_CANCELLED = -9999 TRIP_CANCELLED = -9999
lines_filter = [
'rbg:70070: :H', # U70 -> Düsseldorf Hbf
'rbg:70070: :R', # U70 -> Krefeld Rheinstr
'rbg:70076: :H', # U76 -> Düsseldorf Hbf
'rbg:70076: :R', # U76 -> Krefeld Rheinstr
]
def make_request_data(station_id: int, result_count: int = 8, modes: List = MOT.ALL_MODES, def make_request_data(station_id: int, result_count: int = 8, modes: List = MOT.ALL_MODES,
lines: List[str] = ALL_LINES) -> dict: lines: List[str] = ALL_LINES) -> dict:
@ -170,7 +191,7 @@ def get_next_refresh(data: dict):
if (datetime.datetime.fromtimestamp(time) - datetime.datetime.now()) > datetime.timedelta(minutes=5): if (datetime.datetime.fromtimestamp(time) - datetime.datetime.now()) > datetime.timedelta(minutes=5):
return (datetime.datetime.now() + datetime.timedelta(minutes=5)).timestamp() return (datetime.datetime.now() + datetime.timedelta(minutes=5)).timestamp()
return time return time
return (datetime.datetime.now() + datetime.timedelta(seconds=60)).timestamp() return (datetime.datetime.now() + datetime.timedelta(seconds=90)).timestamp()
def _make_delay_value_for_sql(value: str or int or None) -> int or None: def _make_delay_value_for_sql(value: str or int or None) -> int or None:
@ -198,17 +219,17 @@ def _make_status_value_for_sql(trip: dict) -> str:
return status return status
def update(station_id: int): def update(station_id: int, lines: List[str]) -> dict or None:
try: try:
reply_data = get_data( reply_data = get_data(
make_request_data( make_request_data(
station_id, station_id,
8, 8,
lines=lines_filter lines=lines
) )
) )
except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e: except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
print("Could not get the data: " + str(e)) print("Could not get the data: " + str(e), file=sys.stderr)
return None return None
for trip in reply_data['departureData']: for trip in reply_data['departureData']:
@ -217,15 +238,9 @@ def update(station_id: int):
def wait(cxn: mysql.connector.MySQLConnection, station_id: int): def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
lines_filter = [
'rbg:70070: :H', # U70 -> Düsseldorf Hbf
'rbg:70070: :R', # U70 -> Krefeld Rheinstr
'rbg:70076: :H', # U76 -> Düsseldorf Hbf
'rbg:70076: :R', # U76 -> Krefeld Rheinstr
]
cur = cxn.cursor() cur = cxn.cursor()
while True: while True:
data = update(station_id) data = update(station_id, USE_LINES)
if data is not None: if data is not None:
for t in data['departureData']: for t in data['departureData']:
cur.execute('REPLACE INTO vrr ' cur.execute('REPLACE INTO vrr '
@ -233,8 +248,7 @@ def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
' VALUES (%s, %s, %s, ' ' VALUES (%s, %s, %s, '
'from_unixtime(%s), %s, %s)', 'from_unixtime(%s), %s, %s)',
(t['lineCode'], t['directionCode'], station_id, (t['lineCode'], t['directionCode'], station_id,
t['orgFullTime'], _make_status_value_for_sql(t), _make_delay_value_for_sql(t['delay']) t['orgFullTime'], _make_status_value_for_sql(t), _make_delay_value_for_sql(t['delay'])))
))
cxn.commit() cxn.commit()
next_refresh = get_next_refresh(data) next_refresh = get_next_refresh(data)
print("Sleeping until " + datetime.datetime.fromtimestamp(next_refresh).isoformat(), file=sys.stderr) print("Sleeping until " + datetime.datetime.fromtimestamp(next_refresh).isoformat(), file=sys.stderr)
@ -242,7 +256,7 @@ def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
def main(): def main():
cxn = mysql.connector.connect(user='vrr', password='vrr', host='127.0.0.1', database='vrr') cxn = mysql.connector.connect(**db_config)
cursor = cxn.cursor() cursor = cxn.cursor()
cursor.execute(TABLE) cursor.execute(TABLE)

21
vrr.ini Normal file
View File

@ -0,0 +1,21 @@
[crawl]
; A valid EFA station ID. This can be obtained in the Öffi "departures" view or by inspecting
; the regular POST requests on abfahrtsmonitor.vrr.de.
station_id=20021002
; The server allows filtering by transport mode. Set those preferences here.
use_long_distance=yes
use_regional_trains=yes
use_commuter_trains=yes
use_underground_trains=yes
use_trams=yes
use_buses=yes
use_elevated_trains=yes
; The server allows filtering by line and direction. These line names can be
; obtained by inspecting the POST requests on abfahrtsmonitor.vrr.de.
use_lines=rbg:70070:H,rbg:70070:R,rbg:70076:H,rbg:70076:H
[db]
user=vrr
pass=vrr
database=vrr
host=localhost