Fixes and updates for archive data

* Add new log formatting for dates
* Add more test data for logins
* Remove extra cleanup from test

collab_admin_kit/archive_data.py

@@ -274,13 +274,20 @@ class Archiver():
             reader = csv.DictReader(csvfile)
             for line in reader:
                 if line['actor.email']:
-                    self.logger.info(
-                        'type=login,'
-                        'type={},'
-                        'suspicious={}'.format(
-                            line['name'],
+                    body = ('type=login,'
+                            'type={},'
+                            'datetime={},'
+                            'ip_address={}'.format(
+                                line['name'],
+                                line['id.time'],
+                                line['ipAddress']
+                             ))
+                    if line['parameters.2.boolValue']:
+                        body += 'suspicious={}'.format(
                             line['parameters.2.boolValue']
-                        ),
+                        )
+                    self.logger.info(
+                        body,
                         extra={'entity': line['actor.email']}
                     )
                 elif line['actor.callerType']:
@@ -470,13 +477,19 @@ Used with login reporting to account for inherant delay.''',
         address='/dev/log',
         facility=config['general']['log_facility']
     )
-    formatter = logging.Formatter(config['general']['log_format'])
+    formatter = logging.Formatter(
+        config['general']['log_format'],
+        config['general']['log_date_format']
+    )
     sh.setFormatter(formatter)
     logger.addHandler(sh)
 
     # Create a console handler, set format, and associate.
     ch = logging.StreamHandler()
-    formatter = logging.Formatter(config['general']['console_format'])
+    formatter = logging.Formatter(
+        config['general']['console_format'],
+        config['general']['log_date_format']
+    )
     ch.setFormatter(formatter)
     logger.addHandler(ch)
 

tests/data/mock_gam/report_login.txt

 name,actor.callerType,actor.email,actor.key,actor.profileId,id.applicationName,id.customerId,id.time,id.uniqueQualifier,ipAddress,parameters.0.name,parameters.0.value,parameters.1.multiValue.0,parameters.1.multiValue.1,parameters.1.name,parameters.2.boolValue,parameters.2.name,parameters.2.value,type
 suspicious_login_less_secure_app,KEY,,Google,,login,A089913kg,2019-01-18T14:39:07.000Z,-1591778319305,192.168.1.1,affected_email_address,user1@domain.tld,,,,,,,account_warning
-login_failure,,user2@domain.tld,,115723526446717661397,login,A089913kg,2019-01-18T14:00:40.843Z,3257829353467,192.168.1.1,login_type,google_password,password,,login_challenge_method,,,,login
+login_failure,user2@domain.tld,,115723526446717661397,login,A089913kg,2019-01-18T14:00:40.843Z,3257829353467,192.168.1.1,login_type,google_password,password,,login_challenge_method,,,,login
+login_success,user3@umich.edu,107197167066128203396,login,A089913kg,2019-07-31T12:55:20.976Z,669186676533,141.211.64.223,login_type,saml,none,login_challenge_method,False,is_suspicious,login
+login_success,user4@umich.edu,107062963620782853109,login,A089913kg,2019-07-31T12:55:20.465Z,380534098336,108.184.173.76,login_type,saml,none,login_challenge_method,False,is_suspicious,login
+login_success,user5@umich.edu,106131086245465656739,login,A089913kg,2019-07-31T12:55:18.626Z,6810332896,2601:154:8300:32c8:b424:f867:9485:3c23,login_type,saml,none,login_challenge_method,False,is_suspicious,login
+login_success,user6@umich.edu,106613223900339463265,login,A089913kg,2019-07-31T12:55:18.502Z,235535171151,141.213.145.57,login_type,saml,none,login_challenge_method,False,is_suspicious,login

tests/test_archive_data.py

@@ -14,7 +14,6 @@ archiver = archive_data.Archiver(
     yaml.load(open(config), Loader=yaml.BaseLoader)
 )
 
-
 def test_log_users():
     archiver.log_users()
     logfile = archiver.data_dir + '/user.csv'
@@ -49,7 +48,6 @@ def test_log_logins():
     logfile = archiver.data_dir + '/logins-{}.csv'.format(hour)
     assert os.path.exists(logfile)
     assert os.stat(logfile).st_size > 0
-    archiver.cleanup()
 
 
 def test_log_logins_custom_lookback():