Commit 93f354a4 authored by Neil Spink's avatar Neil Spink

started creating module for data storage

parent 65fda6c9
import json
import boto3
import json
class FlatFileStorage(object):
@staticmethod
def log_last_execution(event, job_name):
@staticmethod
def _append_to_json(data, path):
with open(path, 'ab+') as f:
f.seek(0,2) #Go to the end of file
if f.tell() == 0 : #Check if file is empty
f.write(json.dumps([data]).encode()) #If empty, write an array
else :
f.seek(-1,2)
f.truncate() #Remove the last character, open the array
f.write(' , '.encode()) #Write the separator
f.write(json.dumps(data).encode()) #Dump the dictionary
f.write(']'.encode()) #Close the array
@staticmethod
def _get_file_stream(event):
if event is None:
raise Exception('event parameter should be a dictionary')
if event['source'] is None:
raise Exception('event dictionary should contain source s3 or n/a')
if event['bucket_name'] is None:
raise Exception('event dictionary should contain bucket_name for s3 or n/a')
if event['object_name'] is None:
raise Exception('event dictionary should contain object_name for s3 bucket or full path')
if event['source'].startswith('s3'):
s3 = boto3.resource('s3')
bucket = event['bucket_name']
key = event['object_name']
obj = s3.Object(bucket, key)
stream = obj.get()['Body'].read().decode('utf-8')
else:
stream = file(event['object_name'], 'r')
return stream
import boto3
import yaml
import json
import datetime
class Invoker(object):
......@@ -31,20 +32,17 @@ class Invoker(object):
for site in website_list['sites']: # passes Dictionary to iterator
job = {}
job['job'] = site['job_name']
job = {
'job': site['job_name']
}
# 'executed': datetime.datetime.utcnow()
payload = json.dumps(site['payload']) # containing url and html query
try:
response = self._invoke_lambda(payload)
job_result = self._convert_json_payload(response)
response = self._invoke_lambda(payload)
job_result = self._convert_json_payload(response)
job['result'] = job_result
except Exception, e:
print "Problem invoking aws-lambda-price-grabber"
raise e
job['result'] = job_result
result.append(job)
......
class Watcher(object):
@staticmethod
def place_holder():
print "..."
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment