Here is my data below which is in string format:
temp_variables.json
nls_mssql2_gbq_wti------'{"dataset": "nls","nls_tables_wti":[{ "source":"loanacct_trans_history" ,"patha":" SELECT a.* FROM loanacct_trans_history a WHERE CAST(a.transaction_date AS DATE) = " ,"pathb":" ) UNION SELECT a.* FROM loanacct_trans_history a INNER JOIN ( SELECT reversal_transrefno from loanacct_trans_history WHERE (transaction_code % 2) = 1 and CAST(transaction_date AS DATE) >= " ,"pathc":" and transrefno <> 0 ) b ON a.transrefno = b.reversal_transrefno --" ,"pathd":"--" ,"pathe":"--" ,"pathf":"--" ,"pathg":"--" ,"pathh":"--" ,"pathi":"--" ,"pathj":"--" ,"pathk":"--" ,"pathl":"t_loanacct_trans_history" ,"pathm":"WRITE_TRUNCATE" ,"pathn":"b" } ,{ "source":"loanacct_payment_history" ,"patha":" SELECT a.* FROM loanacct_payment_history a WHERE CAST(a.gl_date AS DATE) = " ,"pathb":" ) --" ,"pathc":"--" ,"pathd":"--" ,"pathe":"--" ,"pathf":"--" ,"pathg":"--" ,"pathh":"--" ,"pathi":"--" ,"pathj":"--" ,"pathk":"--" ,"pathl":"t_loanacct_payment_history" ,"pathm":"WRITE_TRUNCATE" ,"pathn":"c" } ,{ "source":"loanacct_statistics" ,"patha":" SELECT a.* FROM loanacct_statistics a WHERE a.master_record = 0 OR ( a.master_record = 1 and a.year_number = DATEPART(Year, " ,"pathb":" )) OR (a.master_record = 2 and a.month_number = CONVERT(INT,LEFT(REPLACE( " ,"pathc":" ,'"'"'-'"'"','"'"''"'"') ,6)))) --" ,"pathd":"--" ,"pathe":"--" ,"pathf":"--" ,"pathg":"--" ,"pathh":"--" ,"pathi":"--" ,"pathj":"--" ,"pathk":"--" ,"pathl":"t_loanacct_statistics" ,"pathm":"WRITE_TRUNCATE" ,"pathn":"d"}]}'
rep_mysql_loscore_delete_tables------'{"loscore_tables":["third_party_validation", "uw_auto_decision", "decline_reason", "uw_response", "uw_response_history", "folder_history", "decline_reason_history", "lead_source_tracking", "credit_inquiry", "parsed_credit_report_data_history", "contract_detail", "account", "funding_details", "idverify_details", "disbursement_history", "disbursement", "contract_token", "prescreen_netconnect_response", "prescreen_xpn_data", "contract_token_history", "partner_variable_configuration", "partner_configuration", "partner_rate", "rate_summary", "xpn_attributes_history", "parsed_credit_report_history", "sendgrid_templates", "partner_rate_history", "experian_auth", "prescreen_xpn_data_history", "experian_auth_history", "prescreen_netconnect_response_history", "sendgrid_templates_history"]}'
nls_mssql2_gbq_replication_mid_day------'{"dataset": "nls","nls_tables_mid_day":[{"source":"campaign_list","top":"" ,"path":";"} ,{"source":"campaign_setup","top":"" ,"path":";"}]}'
rep_sapi_pcl_data------'{"api_request_url": "https://websvc.globalclientsolutions.com/v3.0/webservices.asmx", "gbq_dataset": "pcl", "pcl_tables": [{"file_name": "pcl_payees", "file_type": "csv", "target_table": "pcl_payees", "tag_name": "PAYEES", "field_list" : ["PAYEE_ID","COMPANY_ID","DRC_PAYEE_ID","PAYEE_NAME","ACTIVE_FLAG","CREATION_DATE","MODIFIED_DATE","MODIFIED_BY","ORIGINAL_ID"], "api_request_body": "<?xml version='"'"'1.0'"'"' encoding='"'"'utf-8'"'"'?><soap:Envelope xmlns:xsi='"'"'http://www.w3.org/2001/XMLSchema-instance'"'"' xmlns:xsd='"'"'http://www.w3.org/2001/XMLSchema'"'"' xmlns:soap='"'"'http://schemas.xmlsoap.org/soap/envelope/'"'"'><soap:Body><PayeesGetXML xmlns='"'"'https://websvc.globalclientsolutions.com/v3.0/'"'"'><userName>websrvc22001001</userName><password>Fn6o7YXz</password><pageSize>5000000</pageSize></PayeesGetXML></soap:Body></soap:Envelope>"}, {"file_name": "pcl_transactions", "file_type": "csv", "target_table": "pcl_transactions", "tag_name": "TRANSACTIONS", "field_list" : ["TRANSACTION_ID","ACCOUNT_ID","TRANSACTION_DATE","DEBIT_DATE","POST_DATE","DAY_OF_MONTH","TRANSACTION_CLASS","TRANSACTION_CLASS_DESCRIPTION","TRANSACTION_TYPE","TRANSACTION_TYPE_DESCRIPTION","TRANSACTION_STATUS","TRANSACTION_STATUS_DESCRIPTION","TRANSACTION_AMOUNT","EVALUCHECK_ID","REFERENCE_ID","FROM_ACCOUNT","TO_ACCOUNT","MEMO","CREATION_DATE","MODIFIED_DATE","MODIFIED_BY"], "api_request_body": "<?xml version='"'"'1.0'"'"' encoding='"'"'utf-8'"'"'?><soap:Envelope xmlns:xsi='"'"'http://www.w3.org/2001/XMLSchema-instance'"'"' xmlns:xsd='"'"'http://www.w3.org/2001/XMLSchema'"'"' xmlns:soap='"'"'http://schemas.xmlsoap.org/soap/envelope/'"'"'><soap:Body><TransactionsGetXML xmlns='"'"'https://websvc.globalclientsolutions.com/v3.0/'"'"'><userName>websrvc22001001</userName><password>Fn6o7YXz</password><accountID/><transactionClass/><transactionType/><transactionStatus/><pageSize>5000000</pageSize></TransactionsGetXML></soap:Body></soap:Envelope>"}, {"file_name": "pcl_payments", "file_type": "csv", "target_table": "pcl_payments", "tag_name": "PAYMENTS", "field_list" : ["PAYMENT_ID","COMPANY_ID","ACCOUNT_ID","PAYMENT_CLASS","PAYMENT_TYPE","EFFECTIVE_DATE","PAYMENT_AMOUNT","FEE_AMOUNT","FAIRSHARE_AMOUNT","ACTIVE_FLAG","PAYMENT_STATUS","PAYMENT_AUTHORIZED","PAYMENT_AUTHORIZED_BY","PAYEE_ID","PAYEE_ADDRESS_ID","PAYEE_BANK_ID","PAYEE_CONTACT_ID","PAYEE_CLIENT_NUM","TRACKING_NUMBER","CHECK_NUMBER","MEMO_1","MEMO_2","MEMO_3","CLEARED_FLAG","CLEARED_DATE","DRC_TRANSACTION_ID","CREATION_DATE","MODIFIED_DATE","MODIFIED_BY","ORIGINAL_ID","AUTHORIZE_BY_DATE","PAYEE_NAME"], "api_request_body": "<?xml version='"'"'1.0'"'"' encoding='"'"'utf-8'"'"'?><soap:Envelope xmlns:xsi='"'"'http://www.w3.org/2001/XMLSchema-instance'"'"' xmlns:xsd='"'"'http://www.w3.org/2001/XMLSchema'"'"' xmlns:soap='"'"'http://schemas.xmlsoap.org/soap/envelope/'"'"'><soap:Body><PaymentsGetXML xmlns='"'"'https://websvc.globalclientsolutions.com/v3.0/'"'"'><userName>websrvc22001001</userName><password>Fn6o7YXz</password><accountId/><effectiveDateGTE/><effectiveDateLTE/><activeFlag/><paymentStatus/><payeeClientNum/><clearedFlag/><clearedDateGTE/><clearedDateLTE/><modifiedDateGTE/><modifiedDateLTE/><authorizeByDateGTE/><authorizeByDateLTE/><pageSize>5000000</pageSize></PaymentsGetXML></soap:Body></soap:Envelope>"}, {"file_name": "pcl_clients", "file_type": "csv", "target_table": "pcl_clients", "tag_name": "CLIENTS", "field_list" : ["ACCOUNT_ID","COMPANY_ID","CLIENT_ID","LAST_NAME","FIRST_NAME","MIDDLE_INITIAL","SOC_SEC_NUM","DATE_OF_BIRTH","CO_LAST_NAME","CO_FIRST_NAME","CO_MIDDLE_INITIAL","CO_SOC_SEC_NUM","CO_DATE_OF_BIRTH","ADDRESS1","ADDRESS2","CITY","STATE","COUNTRY","ZIPCODE","PHONE_NUM","EMAIL_ADDR","CHALLENGE_PHRASE","BANK_ROUTING_NUM","BANK_ACCOUNT_NUM","BANK_ACCOUNT_TYPE","ACTIVE_FLAG","CREATION_DATE","MODIFIED_DATE","MODIFIED_BY","ORIGINAL_ID","ACCOUNT_BALANCE","PENDING_BALANCE","POLICY_GROUP_ID","CLOSE_SCHED_DATE","CLOSE_DATE","DRAFTS_AUTHORIZED","DOCUMENT_STATUS","STATUS","CLIENT_STATUS_DESCRIPTION","DOCUMENT_STATUS_DESCRIPTION","TSR_EXEMPT"] , "api_request_body": "<?xml version='"'"'1.0'"'"' encoding='"'"'utf-8'"'"'?><soap:Envelope xmlns:xsi='"'"'http://www.w3.org/2001/XMLSchema-instance'"'"' xmlns:xsd='"'"'http://www.w3.org/2001/XMLSchema'"'"' xmlns:soap='"'"'http://schemas.xmlsoap.org/soap/envelope/'"'"'><soap:Body><ClientsGetXML xmlns='"'"'https://websvc.globalclientsolutions.com/v3.0/'"'"'><userName>websrvc1</userName><password>F789trfz</password><accountID/><lastName/><firstName/><clientID/><socSecNum/><dateOfBirthGTE/><dateOfBirthLTE/><state/><zipcode/><bankRoutingNum/><activeFlag/><documentStatus/><draftsAuthorized/><closeSchedDateGTE/><closeSchedDateLTE/><closeDateGTE/><closeDateLTE/><status/><lockType/><pageSize>5000000</pageSize></ClientsGetXML></soap:Body></soap:Envelope>"}]}'
bucket------dw_workers_dev
my code:
import json
import re
import time
import ast
file1 = open('temp_variables.json', 'r')
json_data = {}
for line in file1.readlines():
if len(line.split("------")) == 2:
line = line.strip()
line = line.split("------")
key = line[0]
value = line[1]
if value.startswith("'"):
data = json.loads(value)
print(data)
break
else:
mylist2 = ''.join(list(value))
#print(key, repr(mylist2))
json_data[key] = mylist2
error:
Traceback (most recent call last):
File "deploy_variables.py", line 48, in <module>
value = json.loads(value)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/__init__.py", line 357, in loads
return _default_decoder.decode(s)
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/Library/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
I am trying above data to proper json format but, somehow i am getting above error. Please have a look what i am missing.
when i am assigning json to a variables it is working but when m reading from filr and converting it doesn't work
