[refactor] move some functions to utils.py
This commit is contained in:
parent
8ff527b88a
commit
4a3565c3ec
GPG Key ID:
0EE33D788D50130D
56
crawl.py
56
crawl.py
@ -3,25 +3,16 @@
|
||||
# pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see
|
||||
# import logging
|
||||
import argparse
|
||||
import random
|
||||
import sys
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
from typing import BinaryIO, List
|
||||
|
||||
from datetime import date as DTDate
|
||||
from datetime import datetime as DTDateTime
|
||||
from datetime import timedelta as DTTimeDelta
|
||||
from dateutil.relativedelta import FR, relativedelta
|
||||
import appdirs
|
||||
|
||||
import utils
|
||||
|
||||
import mechanize as m
|
||||
|
||||
CARD_MASTERCARD = ['0']
|
||||
CARD_VISA = ['1']
|
||||
|
||||
DIRECTION_TO_EUR = 0
|
||||
DIRECTION_FROM_EUR = 1
|
||||
@ -77,7 +68,6 @@ vals_group.add_argument(
|
||||
nargs='?'
|
||||
)
|
||||
|
||||
|
||||
def _process_stdin(argv: str, res: utils.CurrencyResult) -> None:
|
||||
argv = argv.split()
|
||||
try:
|
||||
@ -116,42 +106,9 @@ def is_float(string: str) -> bool:
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
|
||||
print('Downloading newest rates...', end='')
|
||||
b = m.Browser()
|
||||
# Firefox 64 User-Agent
|
||||
# ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
|
||||
# b.set_header('User-Agent', ua)
|
||||
# Ignore robots.txt
|
||||
# b.set_handle_robots(False)
|
||||
# Debugging flags
|
||||
# b.set_debug_http(True)
|
||||
# b.set_debug_redirects(True)
|
||||
# b.set_debug_responses(True)
|
||||
# PDF URL
|
||||
b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf')
|
||||
|
||||
fm = b.forms()
|
||||
fm_i = fm[0]
|
||||
fm_i.set_all_readonly(False)
|
||||
# Configure form
|
||||
fm_i['creditCardsRadio'] = card_type # VISA
|
||||
fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',')
|
||||
# Retrieve file using button click
|
||||
rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20)))
|
||||
rq.add_header('Accept', '*/*')
|
||||
rp = b.retrieve(rq)
|
||||
print(' Done.')
|
||||
return open(rp[0], 'rb')
|
||||
|
||||
def _get_date() -> DTDate:
|
||||
if DTDate.today().weekday() in [6, 0]:
|
||||
date = DTDate.today() + relativedelta(weekday=FR(-1))
|
||||
else:
|
||||
date = DTDate.today() - DTTimeDelta(1)
|
||||
return date
|
||||
def _parse_date_from_args(date_str: str) -> DTDate:
|
||||
return DTDateTime.strptime(date_str).date()
|
||||
|
||||
def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -> float:
|
||||
if direction == DIRECTION_FROM_EUR:
|
||||
result = amt * rate.ask / 1+duty
|
||||
@ -160,6 +117,7 @@ def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -
|
||||
else:
|
||||
raise ValueError('direction must be DIRECTION_FROM_EUR or DIRECTION_TO_EUR')
|
||||
return result
|
||||
|
||||
def fmt_and_calc(amt: float, cur: str, res: utils.CurrencyResult, direction: str) -> str:
|
||||
cur = cur.upper()
|
||||
if cur in res.rates:
|
||||
@ -180,11 +138,11 @@ args = parser.parse_args()
|
||||
|
||||
# determine card type
|
||||
if args.card_type == 'VISA':
|
||||
use_card_type = CARD_VISA
|
||||
use_card_type = utils.CARD_VISA
|
||||
elif args.card_type == 'MC':
|
||||
use_card_type = CARD_MASTERCARD
|
||||
use_card_type = utils.CARD_MASTERCARD
|
||||
elif args.card_type is None:
|
||||
use_card_type = CARD_VISA
|
||||
use_card_type = utils.CARD_VISA
|
||||
else:
|
||||
sys.exit('Unsupported card type ' + args.card_type)
|
||||
if args.reverse:
|
||||
@ -194,7 +152,7 @@ else:
|
||||
if args.date:
|
||||
retrieve_date = _parse_date_from_args(args.date)
|
||||
else:
|
||||
retrieve_date = _get_date()
|
||||
retrieve_date = utils.get_date()
|
||||
|
||||
if args.cache_dir is not None:
|
||||
filepath = pathlib.Path(args.cache_dir).resolve()
|
||||
@ -207,7 +165,7 @@ if os.path.exists(filename):
|
||||
with open(filename, 'rb') as f:
|
||||
results = utils.get_results_from_pdf(f)
|
||||
else:
|
||||
buf = _retrieve_file(retrieve_date, card_type=use_card_type)
|
||||
buf = utils.get_fileio(retrieve_date, card_type=use_card_type)
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(buf.read())
|
||||
buf.seek(0)
|
||||
|
||||
42
utils.py
42
utils.py
@ -6,10 +6,19 @@ from collections import namedtuple
|
||||
import datetime
|
||||
import re
|
||||
from typing import List, BinaryIO
|
||||
from datetime import date as DTDate
|
||||
from datetime import timedelta as DTTimeDelta
|
||||
|
||||
import PyPDF3
|
||||
import mechanize as m
|
||||
import random
|
||||
|
||||
Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid'])
|
||||
|
||||
# Constants
|
||||
CARD_MASTERCARD = ['0']
|
||||
CARD_VISA = ['1']
|
||||
|
||||
class CurrencyResult():
|
||||
def __init__(self):
|
||||
self.rates = list()
|
||||
@ -98,3 +107,36 @@ def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> Currency
|
||||
for page in pages:
|
||||
text += page.extractText()
|
||||
return get_results_from_text(text, currency=currency)
|
||||
def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
|
||||
print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='')
|
||||
b = m.Browser()
|
||||
# Firefox 64 User-Agent
|
||||
# ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
|
||||
# b.set_header('User-Agent', ua)
|
||||
# Ignore robots.txt
|
||||
# b.set_handle_robots(False)
|
||||
# Debugging flags
|
||||
# b.set_debug_http(True)
|
||||
# b.set_debug_redirects(True)
|
||||
# b.set_debug_responses(True)
|
||||
# PDF URL
|
||||
b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf')
|
||||
fm = b.forms()
|
||||
fm_i = fm[0]
|
||||
fm_i.set_all_readonly(False)
|
||||
# Configure form
|
||||
fm_i['creditCardsRadio'] = card_type # VISA
|
||||
fm_i['selectedDatesString'] = str(date.strftime('%Y%m%d') + ',')
|
||||
# Retrieve file using button click
|
||||
rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20)))
|
||||
rq.add_header('Accept', '*/*')
|
||||
rp = b.retrieve(rq)
|
||||
print(' Done.')
|
||||
return open(rp[0], 'rb')
|
||||
def get_date() -> DTDate:
|
||||
if DTDate.today().weekday() in [6, 0]:
|
||||
date = DTDate.today() + relativedelta(weekday=FR(-1))
|
||||
else:
|
||||
date = DTDate.today() - DTTimeDelta(1)
|
||||
return date
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user