Commit 33911cc0 authored by Richard Robert Sawoscinski's avatar Richard Robert Sawoscinski
Browse files

Merge branch 'newbranch' into 'main'

added error checking to the file upload in sinfter and encoding to the sifter

See merge request !13
parents cf4f61ab 77e0bbd4
......@@ -196,7 +196,7 @@ class Sifter:
def write_logfiles(self):
if len(self.transfer_files) > 0:
target = '{}/transfer_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -206,7 +206,7 @@ class Sifter:
if len(self.private_files) > 0:
target = '{}/private_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -216,7 +216,7 @@ class Sifter:
if len(self.unknown_files) > 0:
target = '{}/unknown_files.csv'.format(self.log_path)
with open(target, 'w+', newline='') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'parent', 'created'] # NOQA 501
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......@@ -226,7 +226,7 @@ class Sifter:
if len(self.shared_files) > 0:
target = '{}/shared_files.csv'.format(self.log_path)
with open(target, 'w+') as csvfile:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['owner', 'name', 'id', 'link', 'created']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
......
......@@ -13,7 +13,7 @@ from datetime import datetime
devnull = open(os.devnull, 'w+')
file_errors = [{"id": "eraeraeraer", "payload": "adsrasr"}]
file_errors = []
class Mover:
......@@ -48,6 +48,7 @@ class Mover:
reader = csv.DictReader(csvfile)
for item in reader:
transfer_error = False
error_text = ""
try:
cmd([
self.config['google']['gam_command'],
......@@ -94,11 +95,14 @@ class Mover:
extra={'entity': item['owner']}
)
transfer_error = True
error_text = e.output
pass
#continue on error
if transfer_error == True:
self.logger.info('{} can not be transferred moving on to next file'.format(item['id']), extra={'entity': item['owner']})
file_errors.append({'id': item['id'],'payload': item})
file_errors.append({'id': item['id'],'name': item['name'], 'link': item['link'], 'parent': item['parent'], 'error': error_text })
else:
self.logger.info(
'{} transferred successfully'.format(item['id']),
......@@ -133,32 +137,40 @@ class Mover:
if len(file_errors) > 0:
target = '{}/Drive Transfer Errors: {} {}.csv'.format(os.getcwd(),owner,timestamp)
with open(target, 'w+') as csvfile:
fieldnames = ['id', 'payload']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for item in file_errors:
writer.writerow(item)
csvfile.close()
try:
with open(target, 'w+', encoding='utf8', newline='') as csvfile:
fieldnames = ['id', 'name', 'link', 'parent', 'error']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for item in file_errors:
writer.writerow(item)
csvfile.close()
#with open('test.csv', 'w', encoding='utf8', newline='') as output_file:
a = cmd([
self.config['google']['gam_command'],
'user',
str(os.getlogin()),
'add',
'drivefile',
'localfile',
target,
'convert'
a = cmd([
self.config['google']['gam_command'],
'user',
str(os.getlogin()),
'add',
'drivefile',
'localfile',
target,
'convert'
])
a = str(a)
a = (a[a.find('(')+1:a.find(')')])
a = ("https://docs.google.com/spreadsheets/d/{}").format(a)
print ("\n Drive Transfer Errors: {}".format(a))
os.remove(target)
# this code will pull the Document ID from the () in the outpu nd build a URL.
a = str(a)
a = (a[a.find('(')+1:a.find(')')])
a = ("https://docs.google.com/spreadsheets/d/{}").format(a)
print ("\n Drive Transfer Errors: {}".format(a))
# remove tmp sheet file
os.remove(target)
except CalledProcessError as e:
self.logger.warning(e.output, extra={'entity': item['owner']})
def main():
helptext = '''examples:
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment