[refactor] move some functions to utils.py

crawl.py

@@ -3,25 +3,16 @@
 # pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see
 # import logging
 import argparse
-import random
 import sys
 import os
 import pathlib
 
-from typing import BinaryIO, List
-
 from datetime import date as DTDate
 from datetime import datetime as DTDateTime
-from datetime import timedelta as DTTimeDelta
-from dateutil.relativedelta import FR, relativedelta
 import appdirs
 
 import utils
 
-import mechanize as m
-
-CARD_MASTERCARD = ['0']
-CARD_VISA = ['1']
 
 DIRECTION_TO_EUR = 0
 DIRECTION_FROM_EUR = 1
@@ -77,7 +68,6 @@ vals_group.add_argument(
     nargs='?'
     )
 
-
 def _process_stdin(argv: str, res: utils.CurrencyResult) -> None:
     argv = argv.split()
     try:
@@ -116,42 +106,9 @@ def is_float(string: str) -> bool:
     except ValueError:
         return False
 
-def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
-    print('Downloading newest rates...', end='')
-    b = m.Browser()
-    # Firefox 64 User-Agent
-    # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
-    # b.set_header('User-Agent', ua)
-    # Ignore robots.txt
-    # b.set_handle_robots(False)
-    # Debugging flags
-    # b.set_debug_http(True)
-    # b.set_debug_redirects(True)
-    # b.set_debug_responses(True)
-    # PDF URL
-    b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf')
-
-    fm = b.forms()
-    fm_i = fm[0]
-    fm_i.set_all_readonly(False)
-    # Configure form
-    fm_i['creditCardsRadio'] = card_type # VISA
-    fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',')
-    # Retrieve file using button click
-    rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20)))
-    rq.add_header('Accept', '*/*')
-    rp = b.retrieve(rq)
-    print(' Done.')
-    return open(rp[0], 'rb')
-
-def _get_date() -> DTDate:
-    if DTDate.today().weekday() in [6, 0]:
-        date = DTDate.today() + relativedelta(weekday=FR(-1))
-    else:
-        date = DTDate.today() - DTTimeDelta(1)
-    return date
 def _parse_date_from_args(date_str: str) -> DTDate:
     return DTDateTime.strptime(date_str).date()
+
 def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -> float:
     if direction == DIRECTION_FROM_EUR:
         result = amt * rate.ask / 1+duty
@@ -160,6 +117,7 @@ def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -
     else:
         raise ValueError('direction must be DIRECTION_FROM_EUR or DIRECTION_TO_EUR')
     return result
+
 def fmt_and_calc(amt: float, cur: str, res: utils.CurrencyResult, direction: str) -> str:
     cur = cur.upper()
     if cur in res.rates:
@@ -180,11 +138,11 @@ args = parser.parse_args()
 
 # determine card type
 if args.card_type == 'VISA':
-    use_card_type = CARD_VISA
+    use_card_type = utils.CARD_VISA
 elif args.card_type == 'MC':
-    use_card_type = CARD_MASTERCARD
+    use_card_type = utils.CARD_MASTERCARD
 elif args.card_type is None:
-    use_card_type = CARD_VISA
+    use_card_type = utils.CARD_VISA
 else:
     sys.exit('Unsupported card type ' + args.card_type)
 if args.reverse:
@@ -194,7 +152,7 @@ else:
 if args.date:
     retrieve_date = _parse_date_from_args(args.date)
 else:
-    retrieve_date = _get_date()
+    retrieve_date = utils.get_date()
 
 if args.cache_dir is not None:
     filepath = pathlib.Path(args.cache_dir).resolve()
@@ -207,7 +165,7 @@ if os.path.exists(filename):
     with open(filename, 'rb') as f:
         results = utils.get_results_from_pdf(f)
 else:
-    buf = _retrieve_file(retrieve_date, card_type=use_card_type)
+    buf = utils.get_fileio(retrieve_date, card_type=use_card_type)
     with open(filename, 'wb') as f:
         f.write(buf.read())
         buf.seek(0)

utils.py

@@ -6,10 +6,19 @@ from collections import namedtuple
 import datetime
 import re
 from typing import List, BinaryIO
+from datetime import date as DTDate
+from datetime import timedelta as DTTimeDelta
+
 import PyPDF3
+import mechanize as m
+import random
 
 Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid'])
 
+# Constants
+CARD_MASTERCARD = ['0']
+CARD_VISA = ['1']
+
 class CurrencyResult():
     def __init__(self):
         self.rates = list()
@@ -98,3 +107,36 @@ def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> Currency
     for page in pages:
         text += page.extractText()
     return get_results_from_text(text, currency=currency)
+def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
+    print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='')
+    b = m.Browser()
+    # Firefox 64 User-Agent
+    # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
+    # b.set_header('User-Agent', ua)
+    # Ignore robots.txt
+    # b.set_handle_robots(False)
+    # Debugging flags
+    # b.set_debug_http(True)
+    # b.set_debug_redirects(True)
+    # b.set_debug_responses(True)
+    # PDF URL
+    b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf')
+    fm = b.forms()
+    fm_i = fm[0]
+    fm_i.set_all_readonly(False)
+    # Configure form
+    fm_i['creditCardsRadio'] = card_type # VISA
+    fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',')
+    # Retrieve file using button click
+    rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20)))
+    rq.add_header('Accept', '*/*')
+    rp = b.retrieve(rq)
+    print(' Done.')
+    return open(rp[0], 'rb')
+def get_date() -> DTDate:
+    if DTDate.today().weekday() in [6, 0]:
+        date = DTDate.today() + relativedelta(weekday=FR(-1))
+    else:
+        date = DTDate.today() - DTTimeDelta(1)
+    return date
+