Commit 77e0bbd4 authored by mail-related applications's avatar mail-related applications
Browse files

added error checking to the file upload in sinfter and encoding to the sifter

parent 873999cf
......@@ -196,7 +196,7 @@ class Sifter:
def write_logfiles(self):
if len(self.transfer_files) > 0:
target = '{}/transfer_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -206,7 +206,7 @@ class Sifter:
if len(self.private_files) > 0:
target = '{}/private_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -216,7 +216,7 @@ class Sifter:
if len(self.unknown_files) > 0:
target = '{}/unknown_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -226,7 +226,7 @@ class Sifter:
if len(self.shared_files) > 0:
target = '{}/shared_files.csv'.format(self.log_path)
with open(target, 'w+') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'created']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......
......@@ -13,7 +13,7 @@ from datetime import datetime
devnull = open(os.devnull, 'w+')
file_errors = [{"id": "eraeraeraer", "payload": "adsrasr"}]
file_errors = []
class Mover:
......@@ -48,6 +48,7 @@ class Mover:
reader = csv.DictReader(csvfile)
for item in reader:
transfer_error = False
error_text = ""
try:
cmd([
self.config['google']['gam_command'],
......@@ -94,11 +95,14 @@ class Mover:
extra={'entity': item['owner']}
)
transfer_error = True
error_text = e.output
pass
#continue on error
if transfer_error == True:
self.logger.info('{} can not be transferred moving on to next file'.format(item['id']), extra={'entity': item['owner']})
file_errors.append({'id': item['id'],'payload': item})
file_errors.append({'id': item['id'],'name': item['name'], 'link': item['link'], 'parent': item['parent'], 'error': error_text })
else:
self.logger.info(
'{} transferred successfully'.format(item['id']),
......@@ -133,32 +137,40 @@ class Mover:
if len(file_errors) > 0:
target = '{}/Drive Transfer Errors: {} {}.csv'.format(os.getcwd(),owner,timestamp)
with open(target, 'w+') as csvfile:
fieldnames = ['id', 'payload']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for item in file_errors:
writer.writerow(item)
csvfile.close()
try:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['id', 'name', 'link', 'parent', 'error']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for item in file_errors:
writer.writerow(item)
csvfile.close()
#with open('test.csv', 'w', encoding='utf8', newline='') as output_file:
a = cmd([
self.config['google']['gam_command'],
'user',
str(os.getlogin()),
'add',
'drivefile',
'localfile',
target,
'convert'
a = cmd([
self.config['google']['gam_command'],
'user',
str(os.getlogin()),
'add',
'drivefile',
'localfile',
target,
'convert'
])
a = str(a)
a = (a[a.find('(')+1:a.find(')')])
a = ("https://docs.google.com/spreadsheets/d/{}").format(a)
print ("\n Drive Transfer Errors: {}".format(a))
os.remove(target)
# this code will pull the Document ID from the () in the outpu nd build a URL.
a = str(a)
a = (a[a.find('(')+1:a.find(')')])
a = ("https://docs.google.com/spreadsheets/d/{}").format(a)
print ("\n Drive Transfer Errors: {}".format(a))
# remove tmp sheet file
os.remove(target)
except CalledProcessError as e:
self.logger.warning(e.output, extra={'entity': item['owner']})
def main():
helptext = '''examples:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment