I have written a python code which gets a json file from a MongoDB database and performs somewhat of an ETL process. My problem is when I run my script on it's own it works perfectly fine, however, when I place the code inside the main init.py file so that it can run using a timer trigger it fails.
Am I doing something wrong in the main init.py file or is there something i'm missing out?
This is the error i'm receiving when I debug it locally:
Executed 'Functions.TestTimerTrigger' (Failed, Id=XXXX, Duration=30285ms)
[2021-01-18T10:41:11.323Z] System.Private.CoreLib: Exception while executing function: Functions.TestTimerTrigger. System.Private.CoreLib: Result: Failure
Exception: ServerSelectionTimeoutError: XXXX:27017: timed out, Timeout: 30s, Topology Description: <TopologyDescription id: 600565a938892b2d0c79ba96, topology_type: Single, servers: [<ServerDescription ('XXXX', 27017) server_type: Unknown, rtt: None, error=NetworkTimeout('XXXX:27017: timed out')>]>
Running my python script on it's own:
from pymongo import MongoClient
import pandas as pd
from azure.storage.filedatalake import DataLakeServiceClient
from azure.core._match_conditions import MatchConditions
from azure.storage.filedatalake._models import ContentSettings
from pandas import json_normalize
from datetime import datetime, timedelta
mongo_client = MongoClient("XXXX")
db = mongo_client.x_db #database name
table = db.levels #collection name
document = table.find()
mongo_docs = list(document)
mongo_docs = json_normalize(mongo_docs)
mongo_docs.to_csv("test.csv", sep = ",", index=False)
try:
global service_client
service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format(
"https", "XXXX"), credential='XXXX')
file_system_client = service_client.get_file_system_client(file_system="root") name
directory_client = file_system_client.get_directory_client("testfolder")
file_client = directory_client.create_file("test.csv")
local_file = open(r"XXX\test.csv",'rb')
file_contents = local_file.read()
file_client.upload_data(file_contents, overwrite=True)
except Exception as e:
print(e)
Placed my script into the main init.py file:
import datetime
import logging
import json
import requests
import azure.functions as func
from pymongo import MongoClient
import pandas as pd
from azure.storage.filedatalake import DataLakeServiceClient
from azure.core._match_conditions import MatchConditions
from azure.storage.filedatalake._models import ContentSettings
from pandas import json_normalize
from datetime import datetime, timedelta
def main(mytimer: func.TimerRequest) -> None:
mongo_client = MongoClient("XXXX") #MongoDB Connection String
db = mongo_client.x_db #database name
table = db.levels #collection name
document = table.find()
mongo_docs = list(document)
mongo_docs = json_normalize(mongo_docs)
mongo_docs.to_csv("test.csv", sep = ",", index=False)
try:
global service_client
service_client = DataLakeServiceClient(account_url="{}://{}.dfs.core.windows.net".format(
"https", "XXXX"), credential='XXXX')
file_system_client = service_client.get_file_system_client(file_system="root")
directory_client = file_system_client.get_directory_client("testfolder")
file_client = directory_client.create_file("test.csv") #name of file being created
local_file = open(r"XXXX\test.csv",'rb') #file which you want to open
file_contents = local_file.read()
file_client.upload_data(file_contents, overwrite=True)
except Exception as e:
print(e)
utc_timestamp = datetime.datetime.utcnow().replace(
tzinfo=datetime.timezone.utc).isoformat()
if mytimer.past_due:
logging.info('The timer is past due!')
logging.info('Python timer trigger function ran at %s', utc_timestamp)
Any help will be appreciated.