Changeset 34

Timestamp:

Oct 13, 2006, 11:21:29 AM (14 years ago)

Author:

Nick Burch

Message:

More of the import script

Location:

trunk/npemap.org.uk/scripts/freethepostcode.org-importer

Files:

• . (modified) (1 prop)
• importer.py (modified) (3 diffs)

trunk/npemap.org.uk/scripts/freethepostcode.org-importer/importer.py

@@ -10 +10 @@
 
 from pyPgSQL import PgSQL
+from geo_helper import turn_wgs84_into_osgb36, turn_osgb36_into_eastingnorthing
+import os
 
 # Database settings
+dbtype = "postgres"
 dbname = "npe_postcodes"
 dbhost = "localhost"
@@ -20 +23 @@
 # Connect to the database
 dbh = None
-if len(dbhost):
-        dbh = PgSQL.connect(database=dbname, host=dbhost, user=dbuser, password=dbpass)
+if dbtype == "pgsql" or dbtype == "postgres" or dbtype == "postgresql":
+        if len(dbhost):
+                dbh = PgSQL.connect(database=dbname, host=dbhost, user=dbuser, password=dbpass)
+        else:
+                dbh = PgSQL.connect(database=dbname, user=dbuser, password=dbpass)
 else:
-        dbh = PgSQL.connect(database=dbname, user=dbuser, password=dbpass)
+        raise Exception("Unknown dbtype %s" % dbtype)
 
 
@@ -41 +47 @@
 
 
-# Download the latest list of postcodes
+# Download the latest list of postcodes if needed
+download = False
+current_file = None
+if os.path.isfile("currentlist"):
+        current_file = "currentlist"
+if os.path.isfile("/tmp/currentlist"):
+        current_file = "/tmp/currentlist"
+
+if not current_file == None:
+        print "Data found, do you wish to re-download?"
+        redownload = raw_input("")
+        if redownload == "y" or redownload == "yes":
+                download = True
+else:
+        download = True
+
+if download:
+        url = "http://www.freethepostcode.org/currentlist"
+        print "Downloading from %s" % url
+        os.system("wget -O currentlist '%s'" % url)
+        current_file = "currentlist"
+print ""
+
+
+# Read in the new list
+postcodes = []
+ftpc = open(current_file, 'r')
+for line in ftpc:
+        line = line[0:-1]
+        if line[0:1] == "#":
+                continue
+
+        parts = line.split(" ")
+        if not len(parts) == 4:
+                print "Invalid line '%s'" % line
+
+        # Turn lat+long into easting+northing
+        osll = turn_wgs84_into_osgb36(parts[0], parts[1], 0)
+        en = turn_osgb36_into_eastingnorthing(osll[0], osll[1])
+
+        pc = {}
+        pc["easting"] = en[0]
+        pc["northing"] = en[1]
+        pc["outer"] = parts[2]
+        pc["inner"] = parts[3]
+        pc["raw"] = "%s %s" % (parts[2],parts[3])
+
+        postcodes.append(pc)
+
 
 # Prompt before removing all the current ones
+sql = "SELECT COUNT(*) AS count FROM postcodes WHERE source = %s"
+sth = dbh.cursor()
+sth.execute(sql, ftp_source)
+counts = sth.fetchall()
+count = counts[0][0]
+
+print "There are currently %d entries in the database from freethepostcode.org" % count
+print "The new import contains %d entries" % len(postcodes)
+print "Are you sure you wish to remove the old entries, to add new ones?"
+confirm = raw_input("")
+
+if confirm == "y" or confirm == "yes":
+        # Good, go ahead
+        pass
+else:
+        print ""
+        raise Exception("Aborting import")
+
+
+# Delete the old entries
+sql = "DELETE FROM postcodes WHERE source = %s"
+sth = dbh.cursor()
+sth.execute(sql, ftp_source)
+
 
 # Add the latest list to the database
+
+