iwonder/first-data-crawler-pdf
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

New URL with robots.txt

Browse Source
This commit is contained in:
iwonder 2020-08-18 21:49:19 +00:00
parent 6df9e32d8f
commit 1d5283d382
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
utils.py
View File

@ -125,13 +125,13 @@ def get_fileio(date: DTDate, card_type: List[str] = CARD_VISA) -> BinaryIO: # py
# ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0' # ua = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 Firefox/64.0'
# b.set_header('User-Agent', ua) # b.set_header('User-Agent', ua)
# Ignore robots.txt # Ignore robots.txt
# b.set_handle_robots(False) b.set_handle_robots(False)
# Debugging flags # Debugging flags
# b.set_debug_http(True) # b.set_debug_http(True)
# b.set_debug_redirects(True) # b.set_debug_redirects(True)
# b.set_debug_responses(True) # b.set_debug_responses(True)
# PDF URL # PDF URL
b.open('https://misc.firstdata.eu/CurrencyCalculator/fremdwaehrungskurse/pdf') b.open('https://online.firstdata.com/CurrencyCalculator/fremdwaehrungskurse/pdf')
fm = b.forms()[0] fm = b.forms()[0]
# This must be done because I can't change the options otherwise # This must be done because I can't change the options otherwise
fm.set_all_readonly(False) fm.set_all_readonly(False)