iwonder/first-data-crawler-pdf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Proper prompts

Browse Source
This commit is contained in:
Christopher Teutsch 2019-05-03 23:40:14 +02:00
parent e2f67fa552
commit 68a0e30b3f
Signed by: iwonder
GPG Key ID: 0EE33D788D50130D
1 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
crawl.py
View File

@ -49,6 +49,11 @@ parser.add_argument(
action='store_true', action='store_true',
help='Reverse direction (EUR -> currency)' help='Reverse direction (EUR -> currency)'
) )
parser.add_argument(
'--cache-dir',
dest='cache_dir',
help='Override the default cache directory with your own path'
)
exc_group = parser.add_mutually_exclusive_group() exc_group = parser.add_mutually_exclusive_group()
exc_group.add_argument( exc_group.add_argument(
'-i', '--interactive', '-i', '--interactive',
@ -111,7 +116,7 @@ def is_float(string: str) -> bool:
return False return False
def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
print('Downloading newest rates...') print('Downloading newest rates...', end='')
b = m.Browser() b = m.Browser()
# Firefox 64 User-Agent # Firefox 64 User-Agent
# ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0' # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
@ -135,7 +140,7 @@ def _retrieve_file(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO:
rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20))) rq = fm_i.click(name='submitButton', coord=(random.randint(1, 119), random.randint(1, 20)))
rq.add_header('Accept', '*/*') rq.add_header('Accept', '*/*')
rp = b.retrieve(rq) rp = b.retrieve(rq)
print('Done.') print(' Done.')
return open(rp[0], 'rb') return open(rp[0], 'rb')
def _get_date() -> DTDate: def _get_date() -> DTDate:
@ -190,7 +195,10 @@ if args.date:
else: else:
retrieve_date = _get_date() retrieve_date = _get_date()
filepath = pathlib.Path(appdirs.user_cache_dir('FirstDataCrawler', 'iwonder')) if args.cache_dir is not None:
filepath = pathlib.Path(os.path.abspath(args.cache_dir))
else:
filepath = pathlib.Path(appdirs.user_cache_dir('FirstDataCrawler', 'iwonder'))
if not filepath.exists(): if not filepath.exists():
filepath.mkdir(parents=True) filepath.mkdir(parents=True)
filename = filepath / (retrieve_date.strftime('%Y%m%d') + '.pdf') filename = filepath / (retrieve_date.strftime('%Y%m%d') + '.pdf')
@ -209,7 +217,6 @@ else:
if args.interactive: if args.interactive:
try: try:
print('Ready.')
while True: while True:
_process_stdin(input('> '), results) _process_stdin(input('> '), results)
except (KeyboardInterrupt, EOFError): except (KeyboardInterrupt, EOFError):