completeted database script

format.py

@@ -2,6 +2,10 @@
 import sqlite3
 import pandas
 import numpy
+from datetime import datetime
+from dateutil.relativedelta import *
+
+
 conn = sqlite3.connect("data.sqlite3")
 cur = conn.cursor()
 df = pandas.read_csv("offencebymonth.csv")
@@ -10,34 +14,79 @@ new_df = df.dropna()
 LGA = df['LGA'].unique()
 subdiv = df['Statistical Division or Subdivision'].unique()
 cat = df["Offence category"].unique()
+cat = cat.tolist()
 subcat_v = df[['Offence category', 'Subcategory']].drop_duplicates()
 subcat_v = subcat_v.values.tolist()
+
+loc = df[['Statistical Division or Subdivision', 'LGA']].drop_duplicates()
+loc_arr = loc.values.tolist()
+subdiv = df['Statistical Division or Subdivision'].unique()
 #database split to 3 tables, main statistics, subdiv index and statistics4
 #Enale foreign key support
+#useful from 4 on
+biglst = df.values.tolist()
 
 cur.execute("PRAGMA foreign_keys = ON;")
 
 categories = """CREATE TABLE IF NOT EXISTS categories(id INTEGER PRIMARY KEY AUTOINCREMENT , name TEXT)""" 
 subcat = """CREATE TABLE IF NOT EXISTS subcat(id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, cat_id INTEGER, FOREIGN KEY(cat_id) REFERENCES categories(id))"""
 location = """CREATE TABLE IF NOT EXISTS location(id INTEGER PRIMARY KEY AUTOINCREMENT , subdiv TEXT, LGA TEXT)"""
-stat = """CREATE TABLE IF NOT EXISTS statistics(id INTEGER PRIMARY KEY AUTOINCREMENT , category_id INTEGER, subcat_id INTEGER, location_id INTEGER, month TEXT NOT NULL, count INTEGER NOT NULL,FOREIGN KEY(category_id) REFERENCES categories(id), FOREIGN KEY(subcat_id) REFERENCES subcat(id), FOREIGN KEY(location_id) REFERENCES location(id))"""
+stat = """CREATE TABLE IF NOT EXISTS statistics(id INTEGER PRIMARY KEY AUTOINCREMENT , category_id INTEGER, subcat_id INTEGER DEFAULT 0, location_id INTEGER, month TEXT NOT NULL, count INTEGER NOT NULL,FOREIGN KEY(category_id) REFERENCES categories(id), FOREIGN KEY(subcat_id) REFERENCES subcat(id), FOREIGN KEY(location_id) REFERENCES location(id))"""
 cur.execute(categories)
+conn.commit()
 cur.execute(subcat)
+conn.commit()
 cur.execute(location)
+conn.commit()
 cur.execute(stat)
+conn.commit()
 #insert statements
 categories_ins = """INSERT INTO categories VALUES(NULL, :category)"""
 subcat_ins = """INSERT INTO subcat VALUES(NULL, :name, :catid)"""
-location_ins = """INSERT INTO location(NULL, ?, ?)"""
+location_ins = """INSERT INTO location VALUES(NULL, :subdiv, :lga)"""
-stat_ins = """INSERT INTO statistics(NULL, ?,?,?,?,?)"""
+stat_ins = """INSERT INTO statistics VALUES(NULL,:cat_id,:subcat_id,:loc_id,:mnth,:cnt)"""
-cat = cat.tolist()
+
 for i in range(len(cat)):
     cur.execute(categories_ins, {"category": cat[i]})
 
 for i in range(len(subcat_v)):
     cur.execute("SELECT id FROM categories WHERE name='"+subcat_v[i][0]+"';")
-    cur.execute(subcat_ins, {"name": subcat_v[i][1], "catid" : cur.fetchall()[0][0]})
+    if type(subcat_v[i][1]) != float:
+        cur.execute(subcat_ins, {"name": subcat_v[i][1], "catid" : cur.fetchall()[0][0]})
+        conn.commit()
 
-cur.execute("SELECT * FROM subcat")
+for i in range(len(loc_arr)):
+    cur.execute(location_ins, {"subdiv": loc_arr[i][0], "lga": loc_arr[i][1]})
+    conn.commit()
+#215 columns darn it
+for i in range(len(biglst)):
+    cur.execute("SELECT id FROM location WHERE LGA='"+biglst[i][1]+"' AND subdiv='"+biglst[i][0]+"';")
+    locid = cur.fetchall()[0][0]
+    cur.execute("SELECT id FROM categories WHERE name='"+biglst[i][2]+"'")
+    catid = cur.fetchall()[0][0]
+    cur.execute("SELECT id FROM subcat WHERE cat_id='"+str(catid)+"'")
+    subcatid = 0
+    try:
+        subcatid = cur.fetchall()[0][0]
+    except:
+        subcatid = None
+    init_date = numpy.datetime64('1995-01')
+    init_lst = []
+
+    for j in range(4,220):
+
+        #witchcraft datetime I hate it
+        init_lst.append({"cat_id": catid, "subcat_id": subcatid, "loc_id": locid, "mnth": pandas.to_datetime(str(init_date+numpy.timedelta64(j-4, 'M'))).strftime('%Y.%m'),"cnt":biglst[i][j]})
+    print(str(i))
+    cur.executemany(stat_ins, init_lst)
+    conn.commit()
+
+#this program makes sure the data is clean and formats it to the database schema
 print(cur.fetchall())
-
+conn.commit()
+cur.close()
+conn.close()
+# cur.execute("SELECT * FROM categories")
+# # print(cur.fetchall())
+# cur.execute("SELECT * FROM location")
+# print(cur.fetchall())