Commit ec4853f2 authored by Rob Carleski's avatar Rob Carleski 🇮🇸
Browse files

Cleanup for archive_data

parent fab9a35a
......@@ -14,8 +14,12 @@ from subprocess import check_output as cmd
from subprocess import CalledProcessError
from os.path import exists
from os import makedirs
from os import devnull
from os import stat
from shutil import rmtree
devnull = open(devnull, 'w')
class Archiver():
......@@ -49,9 +53,15 @@ class Archiver():
]).decode('utf-8')
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error('Persistent errors fetching resource data.')
self.logger.error(
'Persistent errors fetching resource data.',
extra={'entity': 'google-archive-data'}
)
exit(2)
# Write user data to logfile.
......@@ -82,9 +92,15 @@ class Archiver():
]).decode('utf-8')
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error('Persistent errors fetching group data.')
self.logger.error(
'Persistent errors fetching resource data.',
extra={'entity': 'google-archive-data'}
)
exit(2)
# Write group data to logfile.
......@@ -116,9 +132,15 @@ class Archiver():
]).decode('utf-8')
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error('Persistent errors fetching resource data.')
self.logger.error(
'Persistent errors fetching resource data.',
extra={'entity': 'google-archive-data'}
)
exit(2)
# Write resource data to logfile.
......@@ -158,9 +180,15 @@ class Archiver():
]).decode('utf-8')
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error('Persistent errors fetching oauth data.')
self.logger.error(
'Persistent errors fetching resource data.',
extra={'entity': 'google-archive-data'}
)
exit(2)
_ignore = []
......@@ -203,7 +231,10 @@ class Archiver():
self.upload_to_drive('oauth')
def log_logins(self):
_lookback = datetime.now() - timedelta(hours=(self.lookback + 5))
lookback_start = datetime.now() - timedelta(hours=(self.lookback + 5))
lookback_end = datetime.now() - timedelta(hours=(self.lookback + 4))
hour = datetime.now().hour - self.lookback
for i in range(5):
try:
loginData = cmd([
......@@ -211,21 +242,30 @@ class Archiver():
'report',
'login',
'start',
datetime.strftime(_lookback, '%Y-%m-%dT00:00:00Z')
datetime.strftime(lookback_start, '%Y-%m-%dT00:00:00Z'),
'end',
datetime.strftime(lookback_end, '%Y-%m-%dT00:00:00Z')
]).decode('utf-8')
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error('Persistent errors fetching login data.')
self.logger.error(
'Persistent errors fetching resource data.',
extra={'entity': 'google-archive-data'}
)
exit(2)
# Write resource data to logfile.
with open(self.data_dir + '/logins.csv', 'w') as stream:
logfile = '/logins-{}.csv'.format(hour)
with open(self.data_dir + logfile, 'w') as stream:
stream.write(loginData)
# Log data to syslog.
with open(self.data_dir + '/logins.csv', 'r') as csvfile:
with open(self.data_dir + logfile, 'r') as csvfile:
reader = csv.DictReader(csvfile)
for line in reader:
if line['actor.email']:
......@@ -252,11 +292,14 @@ class Archiver():
# Upload logfile to Google
log = '{}-{}'.format(
'logins',
datetime.now().hour - 1
hour
)
self.upload_to_drive(log)
def upload_to_drive(self, log_type):
logfile = '{}/{}.csv'.format(self.data_dir, log_type)
if stat(logfile).st_size == 0:
exit(2)
for i in range(5):
try:
cmd([
......@@ -266,19 +309,20 @@ class Archiver():
'add',
'drivefile',
'localfile',
'{}/{}.csv'.format(
self.data_dir,
log_type
),
logfile,
'parentid',
self.log_dir_id
])
break
except CalledProcessError as e:
self.logger.warning(e.output)
self.logger.warning(
e.output,
extra={'entity': 'google-archive-data'}
)
else:
self.logger.error(
'Persistent failures uploading {} log.'.format(log_type)
'Persistent failures uploading {} log.'.format(log_type),
extra={'entity': 'google-archive-data'}
)
def create_google_report_directory(self):
......@@ -292,8 +336,9 @@ class Archiver():
'filelist',
'query',
"title='{}'".format(self.drive_folder)
]).decode('UTF-8')
], stderr=devnull).decode('UTF-8')
print(top_level_check)
if len(top_level_check.split('\n')) <= 2:
results = cmd([
self.config['google']['gam_command'],
......@@ -305,7 +350,7 @@ class Archiver():
self.drive_folder,
'mimetype',
'gfolder'
]).decode('UTF-8')
], stderr=devnull).decode('UTF-8')
regex = r'\((.*)\)'
parent_id = re.findall(regex, results)[0]
else:
......@@ -320,7 +365,7 @@ class Archiver():
'show',
'filelist',
'query',
"title='{}' AND {} in parents".format(
"title='{}' and '{}' in parents".format(
self.today,
parent_id
)
......@@ -345,13 +390,19 @@ class Archiver():
regex = r'\/drive\/folders\/([a-zA-Z0-9\-\_]+)'
self.log_dir_id = re.findall(regex, log_dir_check)[0]
except CalledProcessError as e:
self.logger.error(e.output)
self.logger.error(
e.output,
extra={'entity': 'google_archive_data'}
)
def cleanup(self):
try:
rmtree(self.data_dir)
except IOError as e:
self.logger.error(e)
self.logger.error(
e,
extra={'entity': 'google_archive_data'}
)
def main():
......
from collab_admin_kit import archive_data
import datetime
import os
import yaml
......@@ -44,7 +45,8 @@ def test_log_oauth():
def test_log_logins():
archiver.log_logins()
logfile = archiver.data_dir + '/logins.csv'
hour = datetime.datetime.now().hour - archiver.lookback
logfile = archiver.data_dir + '/logins-{}.csv'.format(hour)
assert os.path.exists(logfile)
assert os.stat(logfile).st_size > 0
archiver.cleanup()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment