iwonder/first-data-crawler-pdf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

i don't even know, i'm drunk

Browse Source
This commit is contained in:
Christopher Teutsch 2019-05-04 21:54:34 +02:00
parent faab574f82
commit 25c7767b0d
Signed by: iwonder
GPG Key ID: 0EE33D788D50130D
2 changed files with 11 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
crawl.py
View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3
# pylint: disable=missing-docstring,invalid-name
# pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see
# import logging
import argparse
import sys

25
utils.py
View File

@ -1,17 +1,17 @@
#!/usr/bin/env python3
# pylint: disable=missing-docstring,invalid-name
#import argparse
from collections import namedtuple
import datetime
import random
import re
from typing import List, BinaryIO
from collections import namedtuple
from datetime import date as DTDate
from datetime import timedelta as DTTimeDelta
from datetime import datetime as DTDateTime
from typing import BinaryIO, List
import PyPDF3
import mechanize as m
import random
import PyPDF3
from dateutil.relativedelta import FR, relativedelta
Rate = namedtuple('Rate', ['abbr', 'full_name', 'ask', 'bid'])
@ -25,11 +25,6 @@ class CurrencyResult():
self.card_type = str()
self.date = None
def _select_date():
date = datetime.date.today()-datetime.timedelta(1)
if datetime.date().isoweekday() in [6, 0]:
date = datetime.date.today()
def _parse_rate(text: str) -> float or None:
if re.match('Keine Kursdaten vorhanden', text):
_r = None
@ -46,9 +41,9 @@ def _parse_card_type(text: str) -> str:
text = text.strip('" ')
return text
def _parse_date(text: str) -> datetime.date:
def _parse_date(text: str) -> DTDate:
text = text.split(': ')[1].rstrip()
return datetime.datetime.strptime(text, '%d.%m.%Y').date()
return DTDateTime.strptime(text, '%d.%m.%Y').date()
def _array_remove_empty(obj: list) -> List[str]:
try:
@ -99,6 +94,7 @@ def get_results_from_text(text: str, currency: str = None) -> CurrencyResult:
return result
def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> CurrencyResult:
print('Parsing data... ', end='')
reader = PyPDF3.PdfFileReader(buf)
text = str()
pages = []
@ -106,8 +102,10 @@ def get_results_from_pdf(buf: BinaryIO or str, currency: str = None) -> Currency
pages.append(reader.getPage(num))
for page in pages:
text += page.extractText()
print('Done.')
return get_results_from_text(text, currency=currency)
def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # pylint: disable=dangerous-default-value
# pylint: disable=no-member # mechanize.Browser has some lazy-loading methods that pylint doesn't see
print('Downloading rates for ' + date.strftime('%Y-%m-%d') + '... ', end='')
b = m.Browser()
# Firefox 64 User-Agent
@ -139,4 +137,3 @@ def get_date() -> DTDate:
else:
date = DTDate.today() - DTTimeDelta(1)
return date