iwonder/first-data-crawler-pdf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add quiet flag for usage in cronjobs

Browse Source
This commit is contained in:
Christopher Teutsch 2020-08-24 17:20:36 +02:00
parent 1d5283d382
commit cf1b6dade9
3 changed files with 26 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -30,6 +30,9 @@ Write the currency results to standard output, formatted as CSV:
|ISO4217 abbreviation|Full German name|Asking rate|Bidding rate|Date the rate was valid on| |ISO4217 abbreviation|Full German name|Asking rate|Bidding rate|Date the rate was valid on|
|:---|:---|:---|:---|:---| |:---|:---|:---|:---|:---|
#### `-q`, `--quiet`
Do not output informational messages such as "Parsing..." or "Downloading..."
#### `-i`, `--interactive` #### `-i`, `--interactive`
Calculate interactively on stdin Calculate interactively on stdin

16
crawl.py
View File

@ -45,6 +45,12 @@ parser.add_argument(
action='store_true', action='store_true',
help='Write the results to stdout as CSV' help='Write the results to stdout as CSV'
) )
parser.add_argument(
'-q', '--quiet',
dest='quiet',
action='store_true',
help='Do not output the \'Downloading...\' and \'Parsing...\' messages.'
)
parser.add_argument( parser.add_argument(
'--cache-dir', '--cache-dir',
dest='cache_dir', dest='cache_dir',
@ -141,7 +147,8 @@ def is_float(string: str) -> bool:
def _parse_date_from_args(date_str: str) -> DTDate: def _parse_date_from_args(date_str: str) -> DTDate:
return DTDateTime.strptime(date_str).date() from dateutil.parser import isoparse
return isoparse(date_str).date()
def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -> float: def calc_result(amt: float, rate: utils.Rate, direction: int, duty: float = 0) -> float:
@ -202,17 +209,16 @@ if not filepath.exists():
filename = filepath / utils.mk_filename(retrieve_date, use_card_type) filename = filepath / utils.mk_filename(retrieve_date, use_card_type)
if os.path.exists(filename): if os.path.exists(filename):
with open(filename, 'rb') as f: with open(filename, 'rb') as f:
results = utils.get_results_from_pdf(f) results = utils.get_results_from_pdf(f, quiet=args.quiet)
else: else:
buf = utils.get_fileio(retrieve_date, card_type=use_card_type) buf = utils.get_fileio(retrieve_date, card_type=use_card_type, quiet=args.quiet)
with open(filename, 'wb') as f: with open(filename, 'wb') as f:
f.write(buf.read()) f.write(buf.read())
buf.seek(0) buf.seek(0)
results = utils.get_results_from_pdf(buf) results = utils.get_results_from_pdf(buf, quiet=args.quiet)
# #
# processing # processing
# #
if args.interactive: if args.interactive:
try: try:
while True: while True:

21
utils.py
View File

@ -21,7 +21,6 @@ Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid', 'date'])
CARD_MASTERCARD = ['0'] CARD_MASTERCARD = ['0']
CARD_VISA = ['1'] CARD_VISA = ['1']
class CurrencyResult: class CurrencyResult:
def __init__(self): def __init__(self):
self.rates = Dict[str, Rate] self.rates = Dict[str, Rate]
@ -80,7 +79,7 @@ def _parse_line(line: str, ctx: CurrencyResult) -> Rate or None:
return rate return rate
def get_results_from_text(text: str, currency: str = None) -> CurrencyResult: def get_results_from_text(text: str, currency: str = None, quiet: bool = False) -> CurrencyResult:
rates = OrderedDict() rates = OrderedDict()
result = CurrencyResult() result = CurrencyResult()
lines = text.splitlines() lines = text.splitlines()
@ -107,19 +106,22 @@ def get_results_from_text(text: str, currency: str = None) -> CurrencyResult:
return result return result
def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> CurrencyResult: def get_results_from_pdf(buf: BinaryIO or str, currency: str = None, quiet: bool = False) -> CurrencyResult:
print('Parsing data... ', end='', file=stderr) if not quiet:
print('Parsing data... ', end='', file=stderr)
reader = PyPDF3.PdfFileReader(buf) reader = PyPDF3.PdfFileReader(buf)
text = str() text = str()
for num in range(0, reader.getNumPages()-1): for num in range(0, reader.getNumPages()-1):
text += reader.getPage(num).extractText() text += reader.getPage(num).extractText()
print('Done.', file=stderr) if not quiet:
return get_results_from_text(text, currency=currency) print('Done.', file=stderr)
return get_results_from_text(text, currency=currency, quiet=quiet)
def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA, quiet: bool = False) -> BinaryIO: # pylint: disable=dangerous-default-value
# pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see # pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see
print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='', file=stderr) if not quiet:
print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='', file=stderr)
b = m.Browser() b = m.Browser()
# Firefox 64 User-Agent # Firefox 64 User-Agent
# ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0' # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
@ -143,7 +145,8 @@ def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # py
rq = fm.click(name='submitButton', coord=(random.randint(1, 114), random.randint(1, 20))) rq = fm.click(name='submitButton', coord=(random.randint(1, 114), random.randint(1, 20)))
rq.add_header('Accept', '*/*') rq.add_header('Accept', '*/*')
rp = b.retrieve(rq) rp = b.retrieve(rq)
print(' Done.', file=stderr) if not quiet:
print(' Done.', file=stderr)
# Returns an open file-like object with the PDF as contents # Returns an open file-like object with the PDF as contents
return open(rp[0], 'rb') return open(rp[0], 'rb')