I am using a python scraper code to grab publicly available data from http://103.48.16.132/echalan/ but it takes almost ~6gb of memory and more cpu. I need to run multiple instance of the this code which is not possible thusly. Can anyone suggest me a tweak or edit to lessen the memory and cpu footprint of this code. I already added gc and deleted unused variable but all in vain.
Workflow of the code roughly:
- Generate date from a give date range and iterate over each date for multiple challan no for each bank branch to collect data.
- Save data to a dataframe and finally to csv Save the collected data ID and no-data ID to a file with extension
.dat - If resumed then analyse the done and no-data IDs and proceed with the not-done IDs.
Code I use-
# -*- coding: utf-8 -*- import warnings import re warnings.filterwarnings("ignore", category=DeprecationWarning) warnings.simplefilter(action='ignore', category=FutureWarning) import requests,sys,os from datetime import datetime from itertools import * import time,gc import pandas as pd import lxml.html as LH from datetime import timedelta, date import pathlib from collections.abc import Iterable import requests,aiohttp,asyncio start_timeTot = time.time() bank_brnch = {'2573838': 'Agargaon (SB)', '42238412': 'AGLA, DHAKA', '41829141': 'Agrani Balika Bidyalaya (SB)', '42238417': 'AMIN BAZAR, DHAKA', '42238434': 'ARMANITOLA, DHAKA', '44205532': 'ASHULIA BAZAR', '42238436': 'ATI BAZAR, DHAKA', '42238443': 'AWLAD HOSSAIN MARKET, DHAKA', '1217867': 'B.B. Avenue Corp,Dhaka (SB)', '42238447': 'B.I.S.E., DHAKA', '42238448': 'B.M.E. BOARD, DHAKA', '42238451': 'B.U.E.T., DHAKA', '42238452': 'BABUBAZAR, DHAKA', '1218486': 'Badda, Dhaka (SB)', '1218487': 'Baitul Mokarrom,Dhaka (SB)', '42238453': 'BAJME KADERIA COMPLEX, DHAKA', '42238455': 'BANANI BAZAR, DHAKA', '1218488': 'Banani, Dhaka (SB)', '42238461': 'BANGA BANDHU JATIO STADIUM, DHAKA', '42238466': 'BANGA BHABAN, DHAKA', '2549487': 'Baridhara (SB)', '42238480': 'BASABO, DHAKA', '42238486': 'BAWANINAGAR, DHAKA', '1218708': 'Begum Rokeya Sarani,Dhaka (SB)', '1218720': 'Chawk Bazar,Dhaka', '42238489': 'CHURAIN, DHAKA', '41458064': 'COLLEGE GATE (SB)', '1218496': 'Custom House, Dhaka', '1218721': 'D.C.Hall, Dhaka', '1218715': 'D.E.P.Z,Dhaka', '1218489': 'Dhaka Cantt., Dhaka (SB)', '1218497': 'Dhaka Registration Com.,Dhaka', '41614746': 'DHAKA UNIVERSITY CAMPUS (SB)', '115': 'Dhamrai (SB)', '3649899': 'Dhanmondi Corp. (SB)', '1218502': 'Dilkusha Corp.Br., Dhaka (SB)', '42238494': 'DISTILARY ROAD, DHAKA', '1218500': 'Doyagonj, Dhaka', '1218503': 'Fakirapool,Dhaka (SB)', '1857462': 'Farash gonj, Dhaka (SB)', '1218490': 'Farmgate, Dhaka (SB)', '42238498': 'FOREIGN EXCHANGE CORPORATE, DHAKA', '42238501': 'GANA BHABAN, DHAKA', '42238505': 'GORAN, DHAKA', '42238507': 'GREEN ROAD, DHAKA', '42070627': 'GULSHAN (SB)', '1218491': 'Gulshan New North,Dhaka (SB)', '42238511': 'HAZARIBAG, DHAKA', '41293811': 'HAZRAT SHAHJALAL INTL AIRPORT', '42238512': 'Hotel Inter-Continental Br(SHERATAN),DHAKA', '42238517': 'IBRAHIMPUR, DHAKA', '42238524': 'ISHWARCHANDRA STREET, DHAKA', '36250033': 'JATIO SANGSAD BHABAN BR.', '1218651': 'Jatrabari, Dhaka (SB)', '417': 'Joypara (SB)', '1218696': 'Kakrail,Dhaka (SB)', '42238528': 'KALAKOPA, DHAKA', '42238533': 'KALAMPUR, DHAKA', '42238536': 'KALATIA, DHAKA', '41839603': 'KALYAN PUR (SB)', '5602261': 'Kamlapur Rly. St. ICD Br.', '42238538': 'KAWRAN BAZAR, DHAKA,SB', '418': 'Keraniganj (SB)', '1218654': 'Khilgaon, Dhaka (SB)', '42143382': 'KRISHI BAZAR MOHAMMADPUR', '41751373': 'KRISHI BHABAN (SB)', '42238541': 'KURMITOLA, DHAKA', '1218723': 'Lalbagh,Dhaka (SB)', '1218698': 'Lalmatia,Dhaka (SB)', '1857477': 'Laxmi Bazar, Dhaka (SB)', '1217860': 'Local Office,Dhaka', '42238544': 'MAKIM KATRA, DHAKA', '1218656': 'Malibagh,Dhaka (SB)', '42241715': 'MANIK MIAH AVENUE, DHAKA', '1218700': 'Md.Pur Bazar, Dhaka (SB)', '42238546': 'MIRPUR CANTT., DHAKA', '1218711': 'Mirpur I/A, Dhaka', '2717246': 'Mirpur Sec-1', '42238547': 'MITFORD ROAD, DHAKA', '1218493': 'Mogh Bazar, Dhaka (SB)', '1218494': 'Mohakhali, Dhaka (SB)', '1218498': 'N.C.T.B,Dhaka (SB)', '2549438': 'Nagar Bhabon (SB)', '42238548': 'NAJIRABAZAR, DHAKA', '41829146': 'Naval H/Q (SB)', '419': 'Nawabganj (Dhaka)', '1218724': 'Nawabpur Road,Dhaka', '42238563': 'NAYABAZAR, DHAKA', '42238570': 'NAYARHAT, DHAKA', '1218762': 'Nazimuddin Road, Dhaka (SB)', '1218665': 'New Market, Dhaka', '2452744': 'North South Road Br. Dhaka (SB)', '42238573': 'P.A.T.C. (SAVAR), DHAKA', '42238574': 'PALAMGANJ, DHAKA', '1218699': 'Pallabi Br. (Mirpur-12 ), Dhaka', '44332559': 'PANGAON ICT BR.', '1218725': 'Postagola,Dhaka (SB)', '41581585': "PRIME MINISTER'S OFFICE (SB)", '40338614': 'Public Service Commission Branch (Dhaka Airport Branch)', '42238578': 'RAJUK BHABAN, DHAKA', '4039439': 'Ramna Corporate Branch (SB)', '42238581': 'RAMPURA, DHAKA', '42238583': 'RASULPUR BAZAR, DHAKA', '42238588': 'RUHITPUR, DHAKA', '1218726': 'Sadarghat Corp. Br,Dhaka (SB)', '42238593': 'SAIDABAD BUS TERMINAL, DHAKA', '1218701': 'Sat Masjid, Dhaka (SB)', '325': 'Savar (SB)', '1218702': 'Savar Cantt.,Dhaka (SB)', '41423293': 'SEGUN BAGICHA (SB)', '41501647': 'Shahjanpur (SB)', '1218659': 'Shilpa Bhaban,Dhaka (SB)', '1218704': 'Sonargaon Road,Dhaka (SB)', '42139442': 'Sonargoan Hotel (SB)', '1218706': 'Supreme Court,Dhaka (SB)', '42238602': 'TEJGAON INDUSTIAL AREA, DHAKA', '42238606': 'URDU ROAD, DHAKA', '41583041': 'UTTAR KHAN', '41582663': 'UTTARA MODEL TOWN (SB)', '41426798': 'VIQUARUN NESA NOON SCHOOL (SB)', '41660316': 'Wage Earners Corporate (SB)', '2452798': 'WAPDA Building Br.', '1218750': 'Wari, Dhaka (SB)', '1218695': 'Zigatola,Dhaka (SB)'} ouputpath = os.path.join(os.path.dirname(sys.argv[0]),'C_{0}'.format(time.strftime("%Y%d%m")))# merge_folder_csv_after_run = False batch_size_for_async_request = 1000 time_out_for_request_wait = 300 process_missed_accounts_flag = True headerWriteFlag = True donelist = missedlist = noaclist= whataclist = dfschallan = [] start_sd_index = 0 #end_sd_index = len(accounts) accountErrorFlag = [] switch_code_if_not_found = 20000 total_no_acc_to_change = [] switch_code_if_not_found_in_total = 50000 filepath = "" def wait_for_internet_connection(): while True: try: response = requests.get('https://www.google.com/?hl=bn',timeout=5) if response.ok: return except Exception: time.sleep(5) print("Waited for internet to connect {}.".format(datetime.now())) pass wait_for_internet_connection() try: with open('missedCHALLAN.dat','r') as f: missedlist = f.readlines() missedlist = list(filter(None,list(set(list(map(str.strip,missedlist)))))) except: pass try: with open('noacCHALLAN.dat','r') as f: noaclist = f.readlines() noaclist = list(filter(None,list(set(list(map(str.strip,noaclist)))))) except: pass try: with open('whatacCHALLAN.dat','r') as f: whataclist = f.readlines() whataclist = list(filter(None,list(set(list(map(str.strip,whataclist)))))) except: pass try: with open('doneCHALLAN.dat','r') as f: donelist = f.readlines() donelist = list(filter(None,list(map(str.strip,donelist)))) #donelistSet = set(done_list)# for faster performance convert to set #TINSUnique = [tn__ for tn__ in uTINS if tn__ not in donelistSet] #TINS = TINSUnique except: pass #prevent duplicate header write if len(donelist)>0: headerWriteFlag = False def daterange(date1, date2): for n in range(int ((date2 - date1).days)+1): yield date1 + timedelta(n) def _strftime(date): return date.strftime(DATE_FORMAT) def flatten(xs): for x in xs: if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): yield from flatten(x) else: yield x def _date_range_parameters(start, end, span_days): start = _strptime(start) end = _strptime(end) span = timedelta(days=span_days) return start, end, span #File and folder name sanitizer def sanitize_file_folder_name(ffname): reserved_chars = [':','>','<','"','/','\\','*','?','|'] for rc in reserved_chars: ffname = ffname.replace(rc,'_').strip() return ffname def forward_date_range(start, end, span_days): """ Generate tuples with intervals from given range of dates (forward). forward_date_range('2012-01-01', '2012-01-5', 2) 1st yield = ('2012-01-01', '2012-01-03') 2nd yield = ('2012-01-04', '2012-01-05') """ start, end, span = _date_range_parameters(start, end, span_days) stop = end - span while start < stop: current = start + span yield _strftime(start), _strftime(current) start = current + DATE_STEP yield _strftime(start), _strftime(end) s def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): yield lst[i:i + n] def dedeuper(seq): seen = set() seen_add = seen.add return [x for x in seq if not (x in seen or seen_add(x))] async def fetch(acc_,url_,payld_,timout,hdr,check_test_lst,no_acc_check_lst): dt = [] try: #connector = aiohttp.TCPConnector(limit=10,force_close=True) #challanno = payld_['chalan_no'] async with aiohttp.ClientSession() as session: unqID = acc_+"_"+str(payld_['chalan_no']) async with session.post(url_,headers=hdr,data = payld_,timeout=timout) as response: resp = await response.read() #resp = await response.text() root = LH.fromstring(resp) txt = root.text_content() tds = root.xpath("//td") #print(root.text_content()) #AccountName = root.xpath("((//div[contains(@class,'col-lg-9 col-md-9')]//table)[3]//div)[1]")[0].text.strip() if all([i not in txt for i in check_test_lst]): #print(txt) dt_dict = [{'UniqueID':acc_, 'challan_no':tds[2].text_content().split(":")[-1].strip(), 'date' : tds[3].text_content().split(":")[-1].strip(), 'bank':tds[5].text_content().split(":")[-1].strip(), 'branch' : tds[6].text_content().split(":")[-1].strip(), 'code' : tds[7].text_content().split(":")[-1].strip(), 'name' : tds[17].text_content().split(":")[-1].strip(), 'Amount' : tds[19].text_content().split(":")[-1].strip(), 'Timestamp':str(datetime.now())}] dfName1 = pd.DataFrame(dt_dict) dt = [dfName1,acc_] print("Successfully got account {} with response of length {}.".format(unqID, len(resp))) elif all([i in txt for i in no_acc_check_lst]): dt = ['NoAC',acc_] print("No account found for {}.".format(unqID)) except Exception as e: #pass print("Unable to get account {} due to {}.".format(unqID, e.__class__)) dt = ['Error',acc_] print("Error while trying to collect for {}.".format(unqID)) return dt async def get(_brcd,_postQry,_dat): #print('Working on {}'.format(_dat)) LOGIN_URL = 'http://103.48.16.132/echalan/VerifyChalan_new.php' #'https://CHALLAN.org.bd/service/ebill' dt_get = [] dtframes = [] challan_miss_counter = [] time_out_for_request_wait = 3 headers = { 'Accept': 'text/javascript, text/html, application/xml, text/xml, */*', 'Accept-Language': 'en-US,en;q=0.9', 'Connection': 'keep-alive', 'Content-type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Origin': 'http://103.48.16.132', 'Referer': 'http://103.48.16.132/echalan/echalan_iframe.php', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.0.0 Safari/537.36', 'X-Prototype-Version': '1.6.1', 'X-Requested-With': 'XMLHttpRequest', } challan_nos = list(range(1,99999999)) breakwhile = False while True: #global dt_get for challan_no in challan_nos: payload = { 'bank_branch_id': _brcd, 'chalan_date': _dat, 'chalan_no': challan_no, 'trans_type' : _postQry, 'counter_no': '0', 'bank_id' : '2', '_' : '' } uniqIDforDATE = _brcd+"_"+_postQry+"_"+_dat #pass the weekends #if parser.parse(_dat).weekday() in [4,5]: #continue dt_get = await fetch(uniqIDforDATE,LOGIN_URL,payload,time_out_for_request_wait,headers,['Chalan not found'],['Chalan not found']) #time.sleep(random.uniform(0.2,0.8)) if 'NoAC' in dt_get[0]: challan_miss_counter.append('bypass') elif isinstance(dt_get[0],pd.DataFrame): dtframes.append(dt_get[0]) if len(challan_miss_counter)>maxCheckToPassSingleDate: breakwhile = True break elif isinstance(dt_get[0],pd.DataFrame): challan_miss_counter = [] #safety measure for not data if len(dt_get) < 2: dt_get = ['What',uniqIDforDATE] if breakwhile: break if len(dtframes): dfconcated = pd.concat(dtframes) dt_get = [dfconcated,uniqIDforDATE] return dt_get async def main(br_code,post_qry_type,date_batch): global accountErrorFlag global total_no_acc_to_change global headerWriteFlag #wait for internet connection wait_for_internet_connection() dones,pendings = await asyncio.wait([get(br_code,post_qry_type,dat) for dat in date_batch]) #print("Finalized all. ret is a list of len {} outputs.".format(len(dones))) data_results = [i.result() for i in dones] dfs = [i[0] for i in data_results if isinstance(i[0],pd.DataFrame)] if len(dfs)>0: dfNameConcated = pd.concat(dfs) dfschallan.append(dfNameConcated) name_suffix_done = "_"+sanitize_file_folder_name(bank_brnch[br_code]+"_"+dfNameConcated['UniqueID'].unique().tolist()[0]+"_") #Create a folder curdir = pathlib.Path().absolute() flderpath = curdir.joinpath(sanitize_file_folder_name(bank_brnch[br_code])) flderpath.mkdir(parents=True, exist_ok=True) nam = '_{0}_{1}'.format(post_qry_type,time.strftime("%d%m%Y"))+name_suffix_done+'.csv' filepath = curdir.joinpath(sanitize_file_folder_name(bank_brnch[br_code]))/nam dfNameConcated.to_csv(filepath,encoding='utf-8-sig',index=False,mode='a',header=headerWriteFlag) done_accounts = sorted(dfNameConcated['UniqueID'].unique()) #write done accounts df_done = pd.DataFrame(done_accounts, columns=["done_accounts"]) done_file_name = 'doneCHALLAN.dat' df_done.to_csv(done_file_name, mode = 'a', index=False,header=False) headerWriteFlag = False no_ac_data = [i[0] for i in [i for i in data_results if not isinstance(i[0],pd.DataFrame)] if i[0] == 'NoAC'] if len(no_ac_data) == batch_size_for_async_request: accountErrorFlag = accountErrorFlag+no_ac_data elif len(dfs)>0: accountErrorFlag=[] if len(no_ac_data)>0: total_no_acc_to_change = total_no_acc_to_change+no_ac_data missed_accounts = [i[1] for i in [i for i in data_results if not isinstance(i[0],pd.DataFrame)] if i[0] == 'Error'] #write missed accounts df_missed = pd.DataFrame(missed_accounts, columns=["missed_accounts"]) missed_csv_name = 'missedCHALLAN.dat' df_missed.to_csv(missed_csv_name,mode = 'a', index=False,header=False) #process no accounts noac_accounts = [i[1] for i in [i for i in data_results if not isinstance(i[0],pd.DataFrame)] if i[0] == 'NoAC'] #write missed accounts df_noac = pd.DataFrame(noac_accounts, columns=["no_accounts"]) noac_csv_name = 'noacCHALLAN.dat' df_noac.to_csv(noac_csv_name,mode = 'a', index=False,header=False) #process what accounts whatac_accounts = [i[1] for i in [i for i in data_results if not isinstance(i[0],pd.DataFrame)] if i[0] == 'What'] #write missed accounts df_whatac = pd.DataFrame(whatac_accounts, columns=["what_accounts"]) whatac_csv_name = 'whatacCHALLAN.dat' df_whatac.to_csv(whatac_csv_name,mode = 'a', index=False,header=False) print("Acounts collection perfomance=========================noac/missed/done/what = {0} / {1} / {2} / {3}===========================\ ".format(len(no_ac_data),len(missed_accounts), len(dfs),len(whatac_accounts))) total_accounts_scraped = 0 xSearchDateList = [] #accounts_ = accounts[start_sd_index:end_sd_index] accounts_offices = list(bank_brnch.items()) for accounts_office in accounts_offices: dtS_ = '2013-01-01' #(Y-M-D) start date dtE_ = '2022-06-30' #end date dateChunkSixe = 1 #fix 1 for better performance maxCheckToPassSingleDate = 500 batch_size_for_async_request = 1 postQueryTypes = ['C','L'] #argmnts = list(forward_date_range(dtS_, dtE_, dateChunkSixe)) #argmnts = list(set(argmnts)) searchable_dates =[] dtS = time.strptime(dtS_, '%Y-%m-%d') dtE = time.strptime(dtE_, '%Y-%m-%d') dt_ranges = [date(dtS.tm_year, dtS.tm_mon, dtS.tm_mday),date(dtE.tm_year, dtE.tm_mon, dtE.tm_mday)] if isinstance(xSearchDateList,list) and len(xSearchDateList)>0: searchable_dates = xSearchDateList elif len(dt_ranges)>0 and dt_ranges[0].year != 1900 : for dt in daterange(dt_ranges[0], dt_ranges[1]): searchable_dates.append(dt.strftime("%d-%m-%Y")) else: raise Exception searchable_dates_weekdays = searchable_dates #[onday for onday in searchable_dates if parser.parse(onday).weekday() not in [4,5]] total_accounts_scraped=0 for bank_br in bank_brnch.items(): for postQueryType in postQueryTypes: dfschallan = [] branchCode = bank_br[0] accountsOfficeName = bank_br[-1] dones = [donedate.split("_")[-1] for donedate in donelist if branchCode+"_"+postQueryType in donedate] noacs = [donedate.split("_")[-1] for donedate in noaclist if branchCode+"_"+postQueryType in donedate] D1 = set(dones) D2 = set(noacs) D = D1.union(D2) searchable_dates_not_dones = [sd for sd in searchable_dates if sd not in D]+missedlist date_batches = [chunk for chunk in chunks(searchable_dates_not_dones,batch_size_for_async_request)]#[:1] del D1,D2,noacs,dones,searchable_dates_not_dones for batch in date_batches: start_time = time.time() asyncio.get_event_loop().run_until_complete(main(branchCode,postQueryType,batch)) end_time = time.time() total_accounts_scraped += len(batch) print("Took {} seconds to pull {} accounts.....................................................\ ".format(end_time - start_time, total_accounts_scraped)) gc.collect() #Write for each branch code for each postQuery Type if merge_folder_csv_after_run: date_batches = searchable_dates_weekdays elif len(date_batches)<1: continue curdir = pathlib.Path().absolute() bank_folder = sanitize_file_folder_name(bank_brnch[branchCode]) csv_paths = curdir.joinpath(bank_folder) flat_list_dates_done = list(flatten(date_batches)) file_name = sanitize_file_folder_name(bank_folder+"_"+postQueryType+"_"+flat_list_dates_done[0]+"_to_"+flat_list_dates_done[-1])+"_"+'.csv' write_path = csv_paths.joinpath(file_name) column_headers = ['UniqueID', 'challan_no', 'date', 'bank', 'branch', 'code', 'name', 'Amount', 'Timestamp'] matched_csvs=[] pattern_sample = '{}_{}_{}'.format(bank_folder,branchCode,postQueryType) for item in csv_paths.glob(r'**/*'): if pattern_sample in str(item): # retrieve the groups of interest matched_csvs.append(str(item)) if len(matched_csvs)>0: dfcsv_br_code = pd.concat([pd.read_csv(f,header=None,names=column_headers) for f in list(set(matched_csvs))],axis=0,ignore_index=True) dfcsv_br_code_unique = dfcsv_br_code.drop_duplicates(subset=column_headers[1:-1], keep='first') #remove any row contains bank dfcsv_br_code_unique = dfcsv_br_code_unique[dfcsv_br_code_unique['bank'] != 'bank'] dfcsv_br_code_unique.to_csv(write_path,encoding='utf-8-sig',index=False,mode='w',header=True) ##Write save df while running try: flat_list_dates_done = list(flatten(date_batches)) file_name = sanitize_file_folder_name(bank_brnch[branchCode]+"_"+postQueryType+"_"+flat_list_dates_done[0]+"_to_"+flat_list_dates_done[-1])+"_"+'.csv' if len(dfschallan)>0: dfschallanConcated = pd.concat(dfschallan) dfschallanConcated.to_csv(file_name,encoding='utf-8-sig',index=False,mode='w',header=True) except: pass gc.collect() 
