Changeset 671

Timestamp:

Sep 26, 2008, 11:29:27 AM (11 years ago)

Author:

Nick Burch

Message:

Ensure imported postcodes are always unique

File:

• trunk/npemap.org.uk/scripts/generic-python-import/generic_importer.py (modified) (2 diffs)

trunk/npemap.org.uk/scripts/generic-python-import/generic_importer.py

@@ -159 +159 @@
 
                 # Read in the new list
-                postcodes = []
+                raw_postcodes = {}
                 ftpc = open(current_file, 'r')
                 for line in ftpc:
                         pc = self.process_line(line)
                         if pc:
-                                postcodes.append(pc)
+                                raw_pc = pc["raw"]
+                                if not raw_postcodes.has_key(raw_pc):
+                                        raw_postcodes[raw_pc] = []
+                                raw_postcodes[raw_pc].append(pc)
+
+                # Ensure that the postcodes entries are unique, and
+                #  average them if not
+                postcodes = []
+                for raw_pc, arr_data in raw_postcodes.items():
+                        if len(arr_data) == 1:
+                                # Only the one, easy
+                                pc = arr_data[0]
+                        else:
+                                # Several, average
+                                lat = 0
+                                lng = 0
+                                for pc in arr_data:
+                                        lat += float(pc["latitude"])
+                                        lng += float(pc["longitude"])
+                                # Create the average
+                                pc = arr_data[0]
+                                pc["latitude"]  = lat / len(arr_data)
+                                pc["longitude"] = lng / len(arr_data)
+                                self.generate_easting_northing(pc)
+                        # Save the now always-unique postcode
+                        postcodes.append( pc )
 
 
@@ -183 +208 @@
                 count = len(spcs.keys())
 
-                print "There are currently %d entries in the database from %s" % (count, self.source_name)
+                # And for interest, the count of deleted ones
+                sql = "SELECT COUNT(id) FROM postcodes WHERE source = %s AND deleted"
+                sth = dbh.cursor()
+                sth.execute(sql, source_id)
+                deleted_count = sth.fetchone()[0]
+                sth.close()
+
+                print "There are currently %d entries in the database from the %s. (Additionally, there are %d deleted ones)" % (count, self.source_name, deleted_count)
                 print "The new import contains %d entries" % len(postcodes)