Remove defaults, move settings to configuration file

monitor.py

@@ -2,11 +2,12 @@
 import requests
 import json
 from typing import List
-from pprint import pprint, pformat
+# from pprint import pprint, pformat
 import datetime
 import pause
 import sys
 import mysql.connector
+import configparser
 
 TABLE = """
 CREATE TABLE IF NOT EXISTS vrr (
@@ -15,15 +16,10 @@ direction_code varchar(1) not null,
 station_id int not null,
 orig_datetime datetime not null,
 status enum('on_time', 'early', 'late', 'cancelled', 'no_data') not null,
-delay_value int default null,
+delay_value int,
 primary key (line_code, direction_code, station_id, orig_datetime));
 """
 
-# CONFIGURATION
-USE_MODES = []
-USE_STATION_ID = 20021002
-USE_LINES = []
-
 
 class MOT:
     LONG_DISTANCE_TRAIN = 0
@@ -36,16 +32,41 @@ class MOT:
     ALL_MODES = [LONG_DISTANCE_TRAIN, REGIONAL_TRAIN, COMMUTER_TRAIN, UNDERGROUND_TRAIN, TRAM, BUS, ELEVATED_TRAIN]
 
 
+# Parse the configuration file:
+
+cfg = configparser.ConfigParser()
+cfg.read('vrr.ini')
+
+db_config = {
+    'user': cfg['db']['user'],
+    'password': cfg['db']['pass'],
+    'host': cfg['db']['host'],
+    'database': cfg['db']['database'],
+}
+
+USE_MODES = []
+if cfg['crawl']['use_long_distance']:
+    USE_MODES.append(MOT.LONG_DISTANCE_TRAIN)
+if cfg['crawl']['use_regional_trains']:
+    USE_MODES.append(MOT.REGIONAL_TRAIN)
+if cfg['crawl']['use_commuter_trains']:
+    USE_MODES.append(MOT.COMMUTER_TRAIN)
+if cfg['crawl']['use_trams']:
+    USE_MODES.append(MOT.TRAM)
+if cfg['crawl']['use_buses']:
+    USE_MODES.append(MOT.BUS)
+if cfg['crawl']['use_elevated_trains']:
+    USE_MODES.append(MOT.ELEVATED_TRAIN)
+
+if cfg['crawl']['station_id'] is not None:
+    USE_STATION_ID = cfg['crawl']['station_id']
+else:
+    sys.exit("Please specify a station_id in the [crawl] section of vrr.ini")
+USE_LINES = cfg['crawl']['use_lines'].split(',')
+
 ALL_LINES = []
 TRIP_CANCELLED = -9999
 
-lines_filter = [
-    'rbg:70070: :H',  # U70 -> Düsseldorf Hbf
-    'rbg:70070: :R',  # U70 -> Krefeld Rheinstr
-    'rbg:70076: :H',  # U76 -> Düsseldorf Hbf
-    'rbg:70076: :R',  # U76 -> Krefeld Rheinstr
-]
-
 
 def make_request_data(station_id: int, result_count: int = 8, modes: List = MOT.ALL_MODES,
                       lines: List[str] = ALL_LINES) -> dict:
@@ -170,7 +191,7 @@ def get_next_refresh(data: dict):
                 if (datetime.datetime.fromtimestamp(time) - datetime.datetime.now()) > datetime.timedelta(minutes=5):
                     return (datetime.datetime.now() + datetime.timedelta(minutes=5)).timestamp()
                 return time
-    return (datetime.datetime.now() + datetime.timedelta(seconds=60)).timestamp()
+    return (datetime.datetime.now() + datetime.timedelta(seconds=90)).timestamp()
 
 
 def _make_delay_value_for_sql(value: str or int or None) -> int or None:
@@ -198,17 +219,17 @@ def _make_status_value_for_sql(trip: dict) -> str:
     return status
 
 
-def update(station_id: int):
+def update(station_id: int, lines: List[str]) -> dict or None:
     try:
         reply_data = get_data(
             make_request_data(
                 station_id,
                 8,
-                lines=lines_filter
+                lines=lines
             )
         )
     except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError) as e:
-        print("Could not get the data: " + str(e))
+        print("Could not get the data: " + str(e), file=sys.stderr)
         return None
 
     for trip in reply_data['departureData']:
@@ -217,15 +238,9 @@ def update(station_id: int):
 
 
 def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
-    lines_filter = [
-        'rbg:70070: :H',  # U70 -> Düsseldorf Hbf
-        'rbg:70070: :R',  # U70 -> Krefeld Rheinstr
-        'rbg:70076: :H',  # U76 -> Düsseldorf Hbf
-        'rbg:70076: :R',  # U76 -> Krefeld Rheinstr
-    ]
     cur = cxn.cursor()
     while True:
-        data = update(station_id)
+        data = update(station_id, USE_LINES)
         if data is not None:
             for t in data['departureData']:
                 cur.execute('REPLACE INTO vrr '
@@ -233,8 +248,7 @@ def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
                             ' VALUES (%s, %s, %s, '
                             'from_unixtime(%s), %s, %s)',
                             (t['lineCode'], t['directionCode'], station_id,
-                             t['orgFullTime'], _make_status_value_for_sql(t), _make_delay_value_for_sql(t['delay'])
+                             t['orgFullTime'], _make_status_value_for_sql(t), _make_delay_value_for_sql(t['delay'])))
-                            ))
             cxn.commit()
         next_refresh = get_next_refresh(data)
         print("Sleeping until " + datetime.datetime.fromtimestamp(next_refresh).isoformat(), file=sys.stderr)
@@ -242,7 +256,7 @@ def wait(cxn: mysql.connector.MySQLConnection, station_id: int):
 
 
 def main():
-    cxn = mysql.connector.connect(user='vrr', password='vrr', host='127.0.0.1', database='vrr')
+    cxn = mysql.connector.connect(**db_config)
     cursor = cxn.cursor()
     cursor.execute(TABLE)
 

vrr.ini

@@ -0,0 +1,21 @@
+[crawl]
+; A valid EFA station ID. This can be obtained in the Öffi "departures" view or by inspecting
+; the regular POST requests on abfahrtsmonitor.vrr.de.
+station_id=20021002
+; The server allows filtering by transport mode. Set those preferences here.
+use_long_distance=yes
+use_regional_trains=yes
+use_commuter_trains=yes
+use_underground_trains=yes
+use_trams=yes
+use_buses=yes
+use_elevated_trains=yes
+; The server allows filtering by line and direction. These line names can be
+; obtained by inspecting the POST requests on abfahrtsmonitor.vrr.de.
+use_lines=rbg:70070:H,rbg:70070:R,rbg:70076:H,rbg:70076:H
+
+[db]
+user=vrr
+pass=vrr
+database=vrr
+host=localhost