From 4a3565c3ec7d85715e252710bdd827097900d939 Mon Sep 17 00:00:00 2001 From: Christopher Teutsch Date: Sat, 4 May 2019 20:55:02 +0200 Subject: [PATCH] [refactor] move some functions to utils.py --- crawl.py | 56 +++++++------------------------------------------------- utils.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/crawl.py b/crawl.py index 64cb332..aeb351a 100755 --- a/crawl.py +++ b/crawl.py @@ -3,25 +3,16 @@ # pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see # import logging import argparse -import random import sys import os import pathlib -from typing import BinaryIO, List - from datetime import date as DTDate from datetime import datetime as DTDateTime -from datetime import timedelta as DTTimeDelta -from dateutil.relativedelta import FR, relativedelta import appdirs import utils -import mechanize as m - -CARD_MASTERCARD = ['0'] -CARD_VISA = ['1'] DIRECTION_TO_EUR = 0 DIRECTION_FROM_EUR = 1 @@ -77,7 +68,6 @@ vals_group.add_argument( nargs='?' ) - def _process_stdin(argv: str, res: utils.CurrencyResult) -> None: argv = argv.split() try: @@ -116,42 +106,9 @@ def is_float(string: str) -> bool: except ValueError: return False -def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value - print('Downloading newest rates...', end='') - b = m.Browser() - # Firefox 64 User-Agent - # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0' - # b.set_header('User-Agent', ua) - # Ignore robots.txt - # b.set_handle_robots(False) - # Debugging flags - # b.set_debug_http(True) - # b.set_debug_redirects(True) - # b.set_debug_responses(True) - # PDF URL - b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf') - - fm = b.forms() - fm_i = fm[0] - fm_i.set_all_readonly(False) - # Configure form - fm_i['creditCardsRadio'] = card_type # VISA - fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',') - # Retrieve file using button click - rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20))) - rq.add_header('Accept', '*/*') - rp = b.retrieve(rq) - print(' Done.') - return open(rp[0], 'rb') - -def _get_date() -> DTDate: - if DTDate.today().weekday() in [6, 0]: - date = DTDate.today() + relativedelta(weekday=FR(-1)) - else: - date = DTDate.today() - DTTimeDelta(1) - return date def _parse_date_from_args(date_str: str) -> DTDate: return DTDateTime.strptime(date_str).date() + def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -> float: if direction == DIRECTION_FROM_EUR: result = amt * rate.ask / 1+duty @@ -160,6 +117,7 @@ def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) - else: raise ValueError('direction must be DIRECTION_FROM_EUR or DIRECTION_TO_EUR') return result + def fmt_and_calc(amt: float, cur: str, res: utils.CurrencyResult, direction: str) -> str: cur = cur.upper() if cur in res.rates: @@ -180,11 +138,11 @@ args = parser.parse_args() # determine card type if args.card_type == 'VISA': - use_card_type = CARD_VISA + use_card_type = utils.CARD_VISA elif args.card_type == 'MC': - use_card_type = CARD_MASTERCARD + use_card_type = utils.CARD_MASTERCARD elif args.card_type is None: - use_card_type = CARD_VISA + use_card_type = utils.CARD_VISA else: sys.exit('Unsupported card type ' + args.card_type) if args.reverse: @@ -194,7 +152,7 @@ else: if args.date: retrieve_date = _parse_date_from_args(args.date) else: - retrieve_date = _get_date() + retrieve_date = utils.get_date() if args.cache_dir is not None: filepath = pathlib.Path(args.cache_dir).resolve() @@ -207,7 +165,7 @@ if os.path.exists(filename): with open(filename, 'rb') as f: results = utils.get_results_from_pdf(f) else: - buf = _retrieve_file(retrieve_date, card_type=use_card_type) + buf = utils.get_fileio(retrieve_date, card_type=use_card_type) with open(filename, 'wb') as f: f.write(buf.read()) buf.seek(0) diff --git a/utils.py b/utils.py index 376ad8b..8fd6f49 100644 --- a/utils.py +++ b/utils.py @@ -6,10 +6,19 @@ from collections import namedtuple import datetime import re from typing import List, BinaryIO +from datetime import date as DTDate +from datetime import timedelta as DTTimeDelta + import PyPDF3 +import mechanize as m +import random Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid']) +# Constants +CARD_MASTERCARD = ['0'] +CARD_VISA = ['1'] + class CurrencyResult(): def __init__(self): self.rates = list() @@ -98,3 +107,36 @@ def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> Currency for page in pages: text += page.extractText() return get_results_from_text(text, currency=currency) +def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value + print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='') + b = m.Browser() + # Firefox 64 User-Agent + # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0' + # b.set_header('User-Agent', ua) + # Ignore robots.txt + # b.set_handle_robots(False) + # Debugging flags + # b.set_debug_http(True) + # b.set_debug_redirects(True) + # b.set_debug_responses(True) + # PDF URL + b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf') + fm = b.forms() + fm_i = fm[0] + fm_i.set_all_readonly(False) + # Configure form + fm_i['creditCardsRadio'] = card_type # VISA + fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',') + # Retrieve file using button click + rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20))) + rq.add_header('Accept', '*/*') + rp = b.retrieve(rq) + print(' Done.') + return open(rp[0], 'rb') +def get_date() -> DTDate: + if DTDate.today().weekday() in [6, 0]: + date = DTDate.today() + relativedelta(weekday=FR(-1)) + else: + date = DTDate.today() - DTTimeDelta(1) + return date +