Merge pull request nguoianphu#1 from CharlesJUDITH/linter

Apply PEP8 recommandations

Author: fjudith

test/import_fakenames.py

@@ -4,22 +4,28 @@
 import pyodbc
 import csv
 
-driver='FreeTDS'
+driver = 'FreeTDS'
 
 parser = argparse.ArgumentParser()
 parser.parse_known_args()
 # Input file related arguments
-parser.add_argument('-f', '--filename', help='that contains the data to import', default=os.environ.get('FK_FILENAME', default=None ))
+parser.add_argument('-f', '--filename', help='that contains the data to import',
+                    default=os.environ.get('FK_FILENAME', default=None))
 parser.add_argument('--comma-csv', help="CSV formatted file format", action="store_true", default=True)
 parser.add_argument('--comma-delimited', help="comma ',' delimited file format", action="store_true")
 parser.add_argument('--pipe-delimited', help="Pipe '|' delimited file format", action="store_true")
 # SQL related arguments
-parser.add_argument('-S', '--server', help='target SQL server', default=os.environ.get('FK_SERVER', default="127.0.0.1"))
-parser.add_argument('-P', '--port', help='listen port port of the SQL service', default=os.environ.get('FK_PORT', default="5000"))
+parser.add_argument('-S', '--server', help='target SQL server',
+                    default=os.environ.get('FK_SERVER', default="127.0.0.1"))
+parser.add_argument('-P', '--port', help='listen port port of the SQL service',
+                    default=os.environ.get('FK_PORT', default="5000"))
 parser.add_argument('-D', '--database', help='taget SQL database', default=os.environ.get('FK_DATBASE', default="cidb"))
-parser.add_argument('-T', '--table', help='listen port port of the SQL service', default=os.environ.get('FK_TABLE', default="fakenames"))
-parser.add_argument('-u', '--username', help='to authenticate to the SQL database', default=os.environ.get('FK_USERNAME', default="sa"))
-parser.add_argument('-p', '--password', help='to access to the SQL database', default=os.environ.get('FK_PASSWORD', default="myPassword"))
+parser.add_argument('-T', '--table', help='listen port port of the SQL service',
+                    default=os.environ.get('FK_TABLE', default="fakenames"))
+parser.add_argument('-u', '--username', help='to authenticate to the SQL database',
+                    default=os.environ.get('FK_USERNAME', default="sa"))
+parser.add_argument('-p', '--password', help='to access to the SQL database',
+                    default=os.environ.get('FK_PASSWORD', default="myPassword"))
 # Logger arguments
 parser.add_argument('-d', '--debug', help="Enable debug logging", action="store_true")
 args, unknown = parser.parse_known_args()
@@ -36,26 +42,28 @@
 ch.setFormatter(formatter)
 logger.addHandler(ch)
 
+
 def connect():
     # Connect to database
     logger.info('Connecting to server "{0}:{1}" database "{2}"'.format(args.server, args.port, args.database))
-    conn = pyodbc.connect(driver=driver, server=args.server, database=args.database ,port=args.port, uid=args.username, pwd=args.password)
+    conn = pyodbc.connect(driver=driver, server=args.server, database=args.database, port=args.port, uid=args.username,
+                          pwd=args.password)
     return conn
 
 
 def create_table(dbconn, tablename):
     logger.info('Check if table "{0}" exists'.format(tablename))
-    
+
     cursor = dbconn.cursor()
-    
+
     cursor.execute("""
         select name
         from sysobjects
         where type = "U" and name = '{0}'
         """.format(tablename)
-    )
-    
-    try: 
+                   )
+
+    try:
         cursor.fetchone()[0]
 
         logger.info('Table "{0}" already exists.'.format(tablename))
@@ -113,7 +121,7 @@ def create_table(dbconn, tablename):
         """.format(tablename))
 
         dbconn.commit()
-        
+
 
 def import_pipe_delimited_data(dbconn, tablename, filename):
     logger.info('Load data from file "{0}" to table "{1}"'.format(filename, tablename))
@@ -124,37 +132,36 @@ def import_pipe_delimited_data(dbconn, tablename, filename):
     cursor.commit()
 
     # Count total number of lines in the input file
-    with open (file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
+    with open(file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
         reader = csv.reader(f, delimiter='|', quotechar='"')
         next(reader)
         lines = len(list(reader))
-        
 
-    with open (file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
+    with open(file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
         reader = csv.reader(f, delimiter='|', quotechar='"')
         columns = next(reader)
         query = 'insert into ' + tablename + '({0}) values ({1})'
         query = query.format(','.join(columns), ','.join('?' * len(columns))).lower()
-        query = query.replace('mothersmaiden','maidenname')
-        query = query.replace('ups','upstracking')
-        
+        query = query.replace('mothersmaiden', 'maidenname')
+        query = query.replace('ups', 'upstracking')
+
         logger.info('Processing "{0}" lines'.format(lines))
         record = 0
 
         for data in reader:
             cursor = dbconn.cursor()
-        
-            data[0] = int(data[0])                # number
-            data[8] = data[8].replace('Ÿ','Y')    # city
-            data[15] = data[15].replace('œ','oe') # password
-            data[19] = int(data[19])              # telephonecountrycode
-            data[22] = int(data[22])              # age
-            data[38] = float(data[38])            # pounds
-            data[39] = float(data[39])            # kilograms
-            data[41] = int(data[41])              # centimeters
-            data[43] = float(data[43])            # latitude
-            data[44] = float(data[44])            # longitude
-            
+
+            data[0] = int(data[0])  # number
+            data[8] = data[8].replace('Ÿ', 'Y')  # city
+            data[15] = data[15].replace('œ', 'oe')  # password
+            data[19] = int(data[19])  # telephonecountrycode
+            data[22] = int(data[22])  # age
+            data[38] = float(data[38])  # pounds
+            data[39] = float(data[39])  # kilograms
+            data[41] = int(data[41])  # centimeters
+            data[43] = float(data[43])  # latitude
+            data[44] = float(data[44])  # longitude
+
             cursor.execute(query, data)
             cursor.commit()
             record += 1
@@ -164,41 +171,41 @@ def import_pipe_delimited_data(dbconn, tablename, filename):
 
 def import_csv_delimited_data(dbconn, tablename, filename):
     logger.info('Load CSV data from file "{0}" to table "{1}"'.format(filename, tablename))
-    
+
     # Table configuration
     cursor = dbconn.cursor()
     cursor.execute('SET IDENTITY_INSERT {0} ON'.format(tablename))
     cursor.commit()
 
     # Count total number of lines in the input file
-    with open (file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
+    with open(file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
         reader = csv.DictReader(f)
         lines = len(list(reader))
 
-    with open (file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
+    with open(file=filename, mode='r', newline=None, encoding='utf-8-sig') as f:
         reader = csv.DictReader(f)
         query = 'insert into ' + tablename + '({0}) values ({1})'
         query = query.format(','.join(reader.fieldnames), ','.join('?' * len(reader.fieldnames))).lower()
-        query = query.replace('mothersmaiden','maidenname')
-        query = query.replace('ups','upstracking')
+        query = query.replace('mothersmaiden', 'maidenname')
+        query = query.replace('ups', 'upstracking')
 
         logger.info('Processing "{0}" lines'.format(lines))
         record = 0
-        
+
         for data in reader:
             cursor = dbconn.cursor()
-        
+
             data['Number'] = int(data['Number'])
-            data['City'] = data['City'].replace('Ÿ','Y')
-            data['Password'] = data['Password'].replace('œ','oe')
+            data['City'] = data['City'].replace('Ÿ', 'Y')
+            data['Password'] = data['Password'].replace('œ', 'oe')
             data['TelephoneCountryCode'] = int(data['TelephoneCountryCode'])
             data['Age'] = int(data['Age'])
             data['Pounds'] = float(data['Pounds'])
             data['Kilograms'] = float(data['Kilograms'])
             data['Centimeters'] = int(data['Centimeters'])
             data['Latitude'] = float(data['Latitude'])
             data['Longitude'] = float(data['Longitude'])
-            
+
             cursor.execute(query, list(data.values()))
             cursor.commit()
             record += 1
@@ -212,18 +219,18 @@ def handle(event, context):
     if args.pipe_delimited:
         import_pipe_delimited_data(
             dbconn=dbconn,
-            tablename=args.table, 
+            tablename=args.table,
             filename=args.filename
-            )
+        )
     else:
         import_csv_delimited_data(
             dbconn=dbconn,
             tablename=args.table,
             filename=args.filename
-            )
+        )
 
 
 if __name__ == '__main__':
     event = {}
     context = {}
-    handle(event, context)
+    handle(event, context)

test/import_fakenames_panda.py

@@ -5,22 +5,28 @@
 import pandas as pd
 from collections import OrderedDict
 
-driver='FreeTDS'
+driver = 'FreeTDS'
 
 parser = argparse.ArgumentParser()
 parser.parse_known_args()
 # Input file related arguments
-parser.add_argument('-f', '--filename', help='that contains the data to import', default=os.environ.get('FK_FILENAME', default=None ))
+parser.add_argument('-f', '--filename', help='that contains the data to import',
+                    default=os.environ.get('FK_FILENAME', default=None))
 parser.add_argument('--comma-csv', help="CSV formatted file format", action="store_true", default=True)
 parser.add_argument('--comma-delimited', help="comma ',' delimited file format", action="store_true")
 parser.add_argument('--pipe-delimited', help="Pipe '|' delimited file format", action="store_true")
 # SQL related arguments
-parser.add_argument('-S', '--server', help='target SQL server', default=os.environ.get('FK_SERVER', default="127.0.0.1"))
-parser.add_argument('-P', '--port', help='listen port port of the SQL service', default=os.environ.get('FK_PORT', default="5000"))
+parser.add_argument('-S', '--server', help='target SQL server',
+                    default=os.environ.get('FK_SERVER', default="127.0.0.1"))
+parser.add_argument('-P', '--port', help='listen port port of the SQL service',
+                    default=os.environ.get('FK_PORT', default="5000"))
 parser.add_argument('-D', '--database', help='taget SQL database', default=os.environ.get('FK_DATBASE', default="cidb"))
-parser.add_argument('-T', '--table', help='listen port port of the SQL service', default=os.environ.get('FK_TABLE', default="fakenames"))
-parser.add_argument('-u', '--username', help='to authenticate to the SQL database', default=os.environ.get('FK_USERNAME', default="sa"))
-parser.add_argument('-p', '--password', help='to access to the SQL database', default=os.environ.get('FK_PASSWORD', default="myPassword"))
+parser.add_argument('-T', '--table', help='listen port port of the SQL service',
+                    default=os.environ.get('FK_TABLE', default="fakenames"))
+parser.add_argument('-u', '--username', help='to authenticate to the SQL database',
+                    default=os.environ.get('FK_USERNAME', default="sa"))
+parser.add_argument('-p', '--password', help='to access to the SQL database',
+                    default=os.environ.get('FK_PASSWORD', default="myPassword"))
 # Logger arguments
 parser.add_argument('-d', '--debug', help="Enable debug logging", action="store_true")
 args, unknown = parser.parse_known_args()
@@ -41,23 +47,24 @@
 def connect():
     # Connect to database
     logger.info('Connecting to server "{0}:{1}" database "{2}"'.format(args.server, args.port, args.database))
-    conn = pyodbc.connect(driver=driver, server=args.server, database=args.database ,port=args.port, uid=args.username, pwd=args.password)
+    conn = pyodbc.connect(driver=driver, server=args.server, database=args.database, port=args.port, uid=args.username,
+                          pwd=args.password)
     return conn
 
 
 def create_table(dbconn, tablename):
     logger.info('Check if table "{0}" exists'.format(tablename))
-    
+
     cursor = dbconn.cursor()
-    
+
     cursor.execute("""
         select name
         from sysobjects
         where type = "U" and name = '{0}'
         """.format(tablename)
-    )
-    
-    try: 
+                   )
+
+    try:
         cursor.fetchone()[0]
 
         logger.info('Table "{0}" already exists.'.format(tablename))
@@ -115,11 +122,11 @@ def create_table(dbconn, tablename):
         """.format(tablename))
 
         dbconn.commit()
-        
+
 
 def import_csv_delimited_data(dbconn, tablename, filename, delimiter):
     logger.info('Load CSV data from file "{0}" to table "{1}"'.format(filename, tablename))
-    
+
     # Table configuration
     cursor = dbconn.cursor()
     cursor.execute('SET IDENTITY_INSERT {0} ON'.format(tablename))
@@ -128,48 +135,54 @@ def import_csv_delimited_data(dbconn, tablename, filename, delimiter):
     # Import data
     reader = pd.read_csv(filepath_or_buffer=filename, encoding='utf-8-sig', sep=delimiter)
     df = pd.DataFrame(data=reader)
-    
+
     # Fix convertion
     df['CCNumber'] = df['CCNumber'].astype(str)
     df['WesternUnionMTCN'] = df['WesternUnionMTCN'].astype(str)
     df['CVV2'] = df['CVV2'].astype(str)
     df['MoneyGramMTCN'] = df['MoneyGramMTCN'].astype(str)
     df['ZipCode'] = df['ZipCode'].astype(str)
-    df['City'] = df['City'].str.replace('Ÿ','Y')
-    df['Password'] = df['Password'].str.replace('œ','oe')
+    df['City'] = df['City'].str.replace('Ÿ', 'Y')
+    df['Password'] = df['Password'].str.replace('œ', 'oe')
     df['Company'] = df['Company'].fillna('n/a')
 
     # Prepare query
     query = 'insert into ' + tablename + '({0}) values ({1})'
     query = query.format(','.join(df.columns.values), ','.join('?' * df.columns.size)).lower()
-    
+
     # Fit to database fields
     field_transform = {
         'mothersmaiden': 'maidenname',
         'ups': 'upstracking'
     }
-    for key,value in field_transform.items():
+    for key, value in field_transform.items():
         query = query.replace(key, value)
 
     amount_of_lines = df.values.shape[0]
     logger.info('Processing "{0}" lines'.format(amount_of_lines))
     amount_of_records = 0
-    
+
     for data in df.itertuples():
         cursor = dbconn.cursor()
 
         cursor.execute(query, [
-            data.Number, data.Gender, data.NameSet, data.Title, data.GivenName, data.MiddleInitial, data.Surname, data.StreetAddress, data.City, 
-            data.State, data.StateFull, data.ZipCode, data.Country, data.CountryFull, data.EmailAddress, data.Username, data.Password, data.BrowserUserAgent, data.TelephoneNumber, 
-            data.TelephoneCountryCode, data.MothersMaiden, data.Birthday, data.Age, data.TropicalZodiac, data.CCType, data.CCNumber, data.CVV2, data.CCExpires, data.NationalID, 
-            data.UPS, data.WesternUnionMTCN, data.MoneyGramMTCN, data.Color, data.Occupation, data.Company, data.Vehicle, data.Domain, data.BloodType, data.Pounds, 
+            data.Number, data.Gender, data.NameSet, data.Title, data.GivenName, data.MiddleInitial, data.Surname,
+            data.StreetAddress, data.City,
+            data.State, data.StateFull, data.ZipCode, data.Country, data.CountryFull, data.EmailAddress, data.Username,
+            data.Password, data.BrowserUserAgent, data.TelephoneNumber,
+            data.TelephoneCountryCode, data.MothersMaiden, data.Birthday, data.Age, data.TropicalZodiac, data.CCType,
+            data.CCNumber, data.CVV2, data.CCExpires, data.NationalID,
+            data.UPS, data.WesternUnionMTCN, data.MoneyGramMTCN, data.Color, data.Occupation, data.Company,
+            data.Vehicle, data.Domain, data.BloodType, data.Pounds,
             data.Kilograms, data.FeetInches, data.Centimeters, data.GUID, data.Latitude, data.Longitude
-            ]
-        )
+        ]
+                       )
         cursor.commit()
         amount_of_records += 1
 
-    logger.info('Imported {0}/{1} line from file "{2}" to table "{3}"'.format(amount_of_records, amount_of_lines, filename, tablename))
+    logger.info(
+        'Imported {0}/{1} line from file "{2}" to table "{3}"'.format(amount_of_records, amount_of_lines, filename,
+                                                                      tablename))
 
 
 def handle(event, context):
@@ -178,20 +191,20 @@ def handle(event, context):
     if args.pipe_delimited:
         import_csv_delimited_data(
             dbconn=dbconn,
-            tablename=args.table, 
+            tablename=args.table,
             filename=args.filename,
             delimiter='|'
-            )
+        )
     else:
         import_csv_delimited_data(
             dbconn=dbconn,
             tablename=args.table,
             filename=args.filename,
             delimiter=','
-            )
+        )
 
 
 if __name__ == '__main__':
     event = {}
     context = {}
-    handle(event, context)
+    handle(event, context)