Add CSV writing feature

This commit is contained in:
Christopher Teutsch 2020-07-18 20:22:33 +02:00
parent add06ef20d
commit 486c86766a
4 changed files with 51 additions and 20 deletions

View File

@ -1,7 +1,7 @@
# CLI for First Data conversion rates
## Requirements
* python3 with modules `PyPDF3`, `appdirs`, `mechanize`
* python3 with modules `PyPDF3`, `appdirs`, `mechanize`, `dateutil`
## Usage:
`python3 crawl.py [-t {VISA,MC}] [-g ISO_DATE] [-r] {-i | CURRENCY AMOUNT}`
@ -10,6 +10,7 @@
#### `AMOUNT`
This must be a number.
#### `CURRENCY`
This must be the three-letter currency abbreviation, case is irrelevant.
@ -22,16 +23,26 @@ Format: ISO date
#### `-r`, `--direction`
Reverse conversion direction (EUR to specified currency, instead of specified currency to EUR)
#### `-c`, `--csv`
Write the currency results to standard output, formatted as CSV:
|ISO4217 abbreviation|Full German name|Asking rate|Bidding rate|Date the rate was valid on|
|:---|:---|:---|:---|:---|
#### `-i`, `--interactive`
Calculate interactively on stdin
##### `q`, `exit`, `quit`
Quit the program.
##### `AMOUNT CURRENCY`
Convert AMOUNT euros to CURRENCY.
##### `CURRENCY AMOUNT`
Convert AMOUNT CURRENCY to euros.
##### `d`, `date`
Print the date the data is from.

View File

@ -8,6 +8,7 @@ import pathlib
from datetime import date as DTDate
from datetime import datetime as DTDateTime
import appdirs
import csv
import utils
@ -15,7 +16,7 @@ import utils
DIRECTION_TO_EUR = 0
DIRECTION_FROM_EUR = 1
## Argument parsing
# Argument parsing
parser = argparse.ArgumentParser(
description='Currency conversion using First Data cards.')
parser.add_argument(
@ -38,6 +39,12 @@ parser.add_argument(
action='store_true',
help='Reverse direction (EUR -> currency)'
)
parser.add_argument(
'-c', '--csv',
dest='csv',
action='store_true',
help='Write the results to stdout as CSV'
)
parser.add_argument(
'--cache-dir',
dest='cache_dir',
@ -58,13 +65,13 @@ vals_group.add_argument(
type=str,
help='Currency abbreviation to convert from/to (e.g. EUR)',
nargs='?'
)
)
vals_group.add_argument(
'amt',
type=float,
help='Amount',
nargs='?'
)
)
def _process_stdin(argv: str, res: utils.CurrencyResult) -> None:
@ -103,7 +110,7 @@ d | date: Print the date which the data is from.
dir_use = DIRECTION_TO_EUR
else:
raise ValueError
# check that the currency exists
if cur_use not in res.rates:
raise ValueError
@ -143,7 +150,8 @@ def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -
elif direction == DIRECTION_TO_EUR:
result = amt / rate.bid * 1+duty
else:
raise ValueError('direction must be DIRECTION_FROM_EUR or DIRECTION_TO_EUR')
raise ValueError(
'direction must be DIRECTION_FROM_EUR or DIRECTION_TO_EUR')
return result
@ -159,11 +167,12 @@ def fmt_and_calc(amt: float, cur: str, res: utils.CurrencyResult, direction: str
else:
return 'Currency %s could not be found' % cur
# args = parser.parse_args('USD 1000'.split())
args = parser.parse_args()
#logger = logging.getLogger('mechanize')
#logger.addHandler(logging.StreamHandler(sys.stdout))
#logger.setLevel(logging.DEBUG)
# logger.addHandler(logging.StreamHandler(sys.stdout))
# logger.setLevel(logging.DEBUG)
# determine card type
if args.card_type == 'VISA':
@ -186,7 +195,8 @@ else:
if args.cache_dir is not None:
filepath = pathlib.Path(args.cache_dir).resolve()
else:
filepath = pathlib.Path(appdirs.user_cache_dir('FirstDataCrawler', 'iwonder'))
filepath = pathlib.Path(appdirs.user_cache_dir(
'FirstDataCrawler', 'iwonder'))
if not filepath.exists():
filepath.mkdir(parents=True)
filename = filepath / utils.mk_filename(retrieve_date, use_card_type)
@ -209,5 +219,9 @@ if args.interactive:
_process_stdin(input('> '), results)
except (KeyboardInterrupt, EOFError):
sys.exit()
elif args.csv:
w = csv.writer(sys.stdout)
for rate in results.rates.values():
w.writerow([rate.abbr, rate.full_name, rate.ask, rate.bid, rate.date])
else:
print(fmt_and_calc(args.amt, args.currency, results, direction))

View File

@ -1,3 +1,4 @@
PyPDF3
appdirs
mechanize
mechanize
python-dateutil

View File

@ -7,13 +7,15 @@ from collections import namedtuple
from datetime import date as DTDate
from datetime import timedelta as DTTimeDelta
from datetime import datetime as DTDateTime
from typing import BinaryIO, List
from typing import BinaryIO, List, Dict
from collections import OrderedDict
from sys import stderr
import mechanize as m
import PyPDF3
from dateutil.relativedelta import FR, relativedelta
Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid'])
Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid', 'date'])
# Constants
CARD_MASTERCARD = ['0']
@ -22,7 +24,7 @@ CARD_VISA = ['1']
class CurrencyResult:
def __init__(self):
self.rates = list()
self.rates = Dict[str, Rate]
self.card_type = str()
self.date = None
@ -63,7 +65,7 @@ def _array_remove_empty(obj: list) -> List[str]:
return obj
def _parse_line(line: str) -> Rate or None:
def _parse_line(line: str, ctx: CurrencyResult) -> Rate or None:
arr = line.split(" ") # 3 spaces = minimum separation in PDF
arr = _array_remove_empty(arr)
# process currency name
@ -72,13 +74,14 @@ def _parse_line(line: str) -> Rate or None:
abbr=names[0],
full_name=names[1].strip("()"),
ask=_parse_rate(arr[1]),
bid=_parse_rate(arr[2])
bid=_parse_rate(arr[2]),
date=ctx.date
)
return rate
def get_results_from_text(text: str, currency: str = None) -> CurrencyResult:
rates = {}
rates = OrderedDict()
result = CurrencyResult()
lines = text.splitlines()
# skip intro lines
@ -92,25 +95,25 @@ def get_results_from_text(text: str, currency: str = None) -> CurrencyResult:
# now the rates begin
if currency is None:
for line in lines:
line_result = _parse_line(line)
line_result = _parse_line(line, result)
rates[line_result.abbr] = line_result
else:
pattern = re.compile("^"+currency)
for line in lines:
if pattern.match(line):
line_result = _parse_line(line)
line_result = _parse_line(line, result)
rates[line_result.abbr] = line_result
result.rates = rates
return result
def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> CurrencyResult:
print('Parsing data... ', end='')
print('Parsing data... ', end='', file=stderr)
reader = PyPDF3.PdfFileReader(buf)
text = str()
for num in range(0, reader.getNumPages()-1):
text += reader.getPage(num).extractText()
print('Done.')
print('Done.', file=stderr)
return get_results_from_text(text, currency=currency)
@ -164,3 +167,5 @@ def mk_filename(date: DTDate, card_type: List[str]) -> str:
else:
raise TypeError("not a valid card type")
return fn