From c01ac30575a80fbd5671be10bca09f7c93a3a4b7 Mon Sep 17 00:00:00 2001 From: Johann Schwabe Date: Sun, 22 Sep 2024 20:13:35 +0200 Subject: [PATCH 1/2] first version --- .../path_transformers/naming_transformer.py | 4 +- swiss_TLM_api/app.py | 31 +++++---------- swiss_TLM_api/index_cache/.gitignore | 3 -- swiss_TLM_api/point_valuation.csv | 9 +++++ swiss_TLM_api/requirements.txt | 4 +- .../name_finding/helper_index/street_index.py | 15 +++++-- .../index_builder/forest_borders.py | 4 +- .../index_builder/index_builder.py | 7 +++- .../name_finding/index_builder/tlm_streets.py | 2 +- .../swiss_TML_api/name_finding/interest.py | 39 +++++++++++++++++++ .../swiss_TML_api/name_finding/name_finder.py | 10 +++-- .../swiss_TML_api/name_finding/name_index.py | 8 ++-- .../swiss_TML_api/name_finding/swiss_name.py | 3 +- .../test_helper/print_map_snipped.py | 2 +- swiss_TLM_api/test_name_finding.py | 20 ++++++---- 15 files changed, 106 insertions(+), 55 deletions(-) delete mode 100644 swiss_TLM_api/index_cache/.gitignore create mode 100644 swiss_TLM_api/point_valuation.csv create mode 100644 swiss_TLM_api/swiss_TML_api/name_finding/interest.py diff --git a/backend/automatic_walk_time_tables/path_transformers/naming_transformer.py b/backend/automatic_walk_time_tables/path_transformers/naming_transformer.py index d4a980a8..1ecbd109 100644 --- a/backend/automatic_walk_time_tables/path_transformers/naming_transformer.py +++ b/backend/automatic_walk_time_tables/path_transformers/naming_transformer.py @@ -32,9 +32,7 @@ def transform(self, path_: Path) -> Path: req = requests.request("GET", url, headers=headers, data=payload) resp = req.json() - # Use coordinate if next name is more than 100 meters away - if resp[0]["offset"] <= 100: - pt.name = resp[0]["swiss_name"] + pt.name = resp[0]["swiss_name"] except requests.exceptions.ConnectionError: logger.error( diff --git a/swiss_TLM_api/app.py b/swiss_TLM_api/app.py index 2db471ef..9a66b3d6 100644 --- a/swiss_TLM_api/app.py +++ b/swiss_TLM_api/app.py @@ -9,6 +9,7 @@ from shapely.geometry import Point from swiss_TML_api.logging.log_helper import setup_recursive_logger +from swiss_TML_api.name_finding.interest import Interest setup_recursive_logger(logging.INFO) logger = logging.getLogger(__name__) @@ -28,20 +29,21 @@ name_index: NameFinder | None = None map_number_index: MapNumberIndex | None = None +interest: Interest | None = None # The NameFinder is a shared object, thus the index get only loaded once def _load_indexes(): logger.info("Loading indexes...") - global name_index, map_number_index + global name_index, map_number_index, interest try: name_index = NameFinder(force_rebuild=False, reduced=False) + interest = Interest(name_index) except Exception as e: logger.error("Error while loading name index. Forcing rebuild") logger.error(e) name_index = None name_index = NameFinder(force_rebuild=True, reduced=False) - try: map_number_index = MapNumberIndex(force_rebuild=False) except Exception as e: @@ -60,37 +62,24 @@ def ready(): @app.route("/swiss_name", methods=["GET"]) def get_name(): - global name_index + global interest global map_number_index try: lv95_coords = request.json response = [] - for lat, lon in lv95_coords: req_pkt = Point((lat, lon)) - swiss_names = name_index.get_names(lat, lon, 3) - swiss_name: SwissName = swiss_names[0] - - # If object_type is of type 'Weggabelung' and there exists a Hauptgipfel nearby, we take the Hauptgipfel - if swiss_name.object_type == "Weggabelung" and len(swiss_names) > 1: - for n in swiss_names[1:]: - tlm_pkt = Point((n.x, n.y)) - if ( - n.object_type == "Hauptgipfel" - and round(req_pkt.distance(tlm_pkt)) <= 250 - ): - swiss_name.name = "Weggabelung bei " + n.name - break + selected_name = interest.select_name((lat, lon)) response.append( { - "lv95_coord": (swiss_name.x, swiss_name.y), + "lv95_coord": (selected_name.x, selected_name.y), "offset": round( - req_pkt.distance(Point((swiss_name.x, swiss_name.y))) + req_pkt.distance(Point((selected_name.x, selected_name.y))) ), - "swiss_name": swiss_name.name, - "object_type": swiss_name.object_type, + "swiss_name": selected_name.name, + "object_type": selected_name.object_type, } ) diff --git a/swiss_TLM_api/index_cache/.gitignore b/swiss_TLM_api/index_cache/.gitignore deleted file mode 100644 index b19e80bf..00000000 --- a/swiss_TLM_api/index_cache/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.dat -*.idx -*.tar* \ No newline at end of file diff --git a/swiss_TLM_api/point_valuation.csv b/swiss_TLM_api/point_valuation.csv new file mode 100644 index 00000000..ed595b99 --- /dev/null +++ b/swiss_TLM_api/point_valuation.csv @@ -0,0 +1,9 @@ +object_type,interest +Weggabelung,14 +Bruecke,30 +Flurname swisstopo,20 +Lokalname swisstopo,20 +Wegende,20 +Antenne klein,30 +Waldrand,12 +Galerie,15 \ No newline at end of file diff --git a/swiss_TLM_api/requirements.txt b/swiss_TLM_api/requirements.txt index 0fb3d91d..7d1abb34 100644 --- a/swiss_TLM_api/requirements.txt +++ b/swiss_TLM_api/requirements.txt @@ -1,4 +1,3 @@ -pillow==10.3.0 rtree==1.0.1 fiona==1.8.22 grequests==0.6.0 @@ -6,4 +5,5 @@ shapely==1.8.5 Flask==2.3.2 gunicorn==22.0.0 flask-cors==4.0.1 -gdown==4.5.4 \ No newline at end of file +gdown==4.5.4 +pillow==10.4.0 \ No newline at end of file diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/helper_index/street_index.py b/swiss_TLM_api/swiss_TML_api/name_finding/helper_index/street_index.py index b60e85a3..1aafae61 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/helper_index/street_index.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/helper_index/street_index.py @@ -17,7 +17,7 @@ def __init__(self, shp_file): # If it does not exist, create the directory os.makedirs("./index_cache") - def get_street_index(self, street_types=("Weg", "Strasse", "Wegfragment", "Spur")): + def get_street_index(self, street_types=("Weg", "Strasse", "Wegfragment", "Spur"), bounds=None): path_index = RTreeIndex(self.index_file_path) if path_index.get_size() > 0: @@ -30,7 +30,7 @@ def get_street_index(self, street_types=("Weg", "Strasse", "Wegfragment", "Spur" with fiona.open(self.shp_file) as src: skipped_objects = set() - + skipped_by_bounds = 0 for counter, obj in enumerate(src): if not counter % 100_000: logger.debug( @@ -38,13 +38,22 @@ def get_street_index(self, street_types=("Weg", "Strasse", "Wegfragment", "Spur" ) geo = obj["geometry"]["coordinates"] + obj_type = obj["properties"]["OBJEKTART"] if ( any(s in obj_type for s in street_types) and obj["geometry"]["type"] == "LineString" ): + if bounds: + start_in_bounds = bounds[0][0] < geo[0][0] < bounds[1][0] and bounds[0][1] < geo[0][1] < bounds[1][1] + end_in_bounds = bounds[0][0] < geo[-1][0] < bounds[1][0] and bounds[0][1] < geo[-1][1] < bounds[1][1] + if not start_in_bounds and not end_in_bounds: + skipped_objects.add(obj_type) + skipped_by_bounds += 1 + continue path = [pkt[:-1] for pkt in geo] # remove elevation information + path = LineString(path) path_index.insert( id=0, @@ -54,7 +63,7 @@ def get_street_index(self, street_types=("Weg", "Strasse", "Wegfragment", "Spur" else: skipped_objects.add(obj_type) - + logger.info("\tSkipped {}/{} objects due to bounds.".format(skipped_by_bounds,path_index.get_size())) if len(skipped_objects): logger.info("\tSkipped Objects: {}".format(skipped_objects)) diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/forest_borders.py b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/forest_borders.py index c32094fa..9143dfd8 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/forest_borders.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/forest_borders.py @@ -66,7 +66,7 @@ def insert_forest_intersections(self, shp_files): street_index_builder = StreetIndex( self.base_path + "swissTLM3D_TLM_STRASSE.shp" ) - path_index = street_index_builder.get_street_index() + path_index = street_index_builder.get_street_index(bounds=self.bounds) path_index.close() # as we use multiprocessing we have to reload the index for each thread flurname_index_builder = FlurnamenIndex( @@ -108,7 +108,7 @@ def worker(res_set, set_lock): street_index_builder = StreetIndex( self.base_path + "swissTLM3D_TLM_STRASSE.shp" ) - path_index = street_index_builder.get_street_index() + path_index = street_index_builder.get_street_index(bounds=self.bounds) while True: batch = q.get() diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/index_builder.py b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/index_builder.py index d979f2a0..d19f8aa0 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/index_builder.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/index_builder.py @@ -4,21 +4,26 @@ class IndexBuilder: - def __init__(self, index, reduced=False): + def __init__(self, index, reduced=False, bounds=None): self.index = index # The reduced dataset only contains the city of Bern base_path = "./resources/swissTLM3D_LV95_data" self.base_path = base_path + ("_full/" if not reduced else "/") self.reduced = reduced + self.bounds = bounds def load(self): pass + def in_bounds(self, x, y): + return self.bounds[0][0] < x < self.bounds[1][0] and self.bounds[0][1] < y < self.bounds[1][1] def add_pkt_file(self, shp_file, has_name=True): with fiona.open(shp_file) as src: for obj in src: geo = obj["geometry"]["coordinates"] + if not self.in_bounds(geo[0], geo[1]): + continue obj_type = obj["properties"]["OBJEKTART"] name = obj["properties"]["NAME"] if has_name else obj_type diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/tlm_streets.py b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/tlm_streets.py index 8352af06..e22b9cee 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/tlm_streets.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/index_builder/tlm_streets.py @@ -15,7 +15,7 @@ def load(self): street_index_builder = StreetIndex( self.base_path + "swissTLM3D_TLM_STRASSE.shp" ) - path_index = street_index_builder.get_street_index() + path_index = street_index_builder.get_street_index(bounds=self.bounds) shp_file = self.base_path + "swissTLM3D_TLM_STRASSENINFO.shp" diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/interest.py b/swiss_TLM_api/swiss_TML_api/name_finding/interest.py new file mode 100644 index 00000000..e69cf4bc --- /dev/null +++ b/swiss_TLM_api/swiss_TML_api/name_finding/interest.py @@ -0,0 +1,39 @@ +import csv +from typing import List + +from swiss_TML_api.name_finding.name_finder import NameFinder +from swiss_TML_api.name_finding.swiss_name import SwissName + + +def distance_falloff(distance): + return (distance /1000 - 1) ** 4 + + +class Interest: + def __init__(self, name_finder: NameFinder): + self.interest = {} + self.name_finder = name_finder + with open("point_valuation.csv", mode='r') as file: + reader = csv.DictReader(file) + for row in reader: + self.interest[row["object_type"]] = int(row["interest"]) + print(reader.fieldnames) + + def value_point(self, position, point: SwissName): + distance = ((position[0] - point.x) ** 2 + (position[1] - point.y) ** 2) ** 0.5 + if distance > 150: + return 0 + object_type_interest = self.interest.get(point.object_type) + if object_type_interest is None: + print(f"Object type {point.object_type} not found in point_valuation.csv") + return 1 + interest = object_type_interest * distance_falloff(distance) + print(f"Point {point.name} has distance {distance} and interest {interest}") + return interest + def select_name(self, position): + points = self.name_finder.get_names(position[0], position[1], 15) + points.sort(key=lambda x: self.value_point(position, x), reverse=True) + if self.value_point(position, points[0]) < 1: + return SwissName("", "Coordinates", position[0], position[1], 0, 0) + return points[0] + diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/name_finder.py b/swiss_TLM_api/swiss_TML_api/name_finding/name_finder.py index 95943b8b..51231421 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/name_finder.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/name_finder.py @@ -1,3 +1,4 @@ +import csv import logging import time @@ -7,7 +8,7 @@ class NameFinder(NameIndex): - def __init__(self, force_rebuild=False, reduced=False): + def __init__(self, force_rebuild=False, reduced=True, bounds=None ): """ If force_rebuild is True, the index will be rebuilt (all old files will be removed). If force_rebuild is False, the index will be loaded from the filesystem (if it exists) @@ -20,12 +21,13 @@ def __init__(self, force_rebuild=False, reduced=False): """ start = time.time() - super().__init__(force_rebuild, reduced) + super().__init__(force_rebuild, reduced, bounds) end = time.time() + logger.info("Name Index loaded (after {}s)".format(str(end - start))) - def get_names(self, lat: float, lon: float, n=1): + def get_names(self, lat: float, lon: float, num_results=1): """ See also https://api3.geo.admin.ch/api/faq/index.html#which-layers-have-a-tooltip fair use limit 20 Request per minute @@ -39,7 +41,7 @@ def get_names(self, lat: float, lon: float, n=1): start = time.time() list_of_points = list( - self.index.nearest((lat, lon, lat, lon), num_results=n, objects="raw") + self.index.nearest((lat, lon, lat, lon), num_results=num_results, objects="raw") ) end = time.time() diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/name_index.py b/swiss_TLM_api/swiss_TML_api/name_finding/name_index.py index b2ba2a1c..cd5c22fa 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/name_index.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/name_index.py @@ -32,7 +32,7 @@ def delete_file(pattern): class NameIndex: - def __init__(self, force_rebuild, reduced): + def __init__(self, force_rebuild, reduced, bounds=None): # if ./index_cache does not exist, create it if not os.path.exists("./index_cache"): # If it does not exist, create the directory @@ -91,7 +91,7 @@ def __init__(self, force_rebuild, reduced): folder = "./resources/swissTLM3D_LV95_data_full/" self.__download_resources(url, folder) - self.generate_index(reduced) + self.generate_index(reduced, bounds) def __del__(self): if hasattr(self, "index"): @@ -121,7 +121,7 @@ def __download_resources(self, url: str, destination: str): os.rmdir(os.path.join(destination, directory)) os.remove(output) - def generate_index(self, reduced): + def generate_index(self, reduced, bounds=None): logger.info( "Start Creating Index: Save index in {}. This might take a few minutes.".format( self.index_file_path @@ -155,7 +155,7 @@ def generate_index(self, reduced): start = time.time() logger.info("\tInsertion of {} started...".format(index_part.__name__)) - loader = index_part(self.index, reduced=reduced) + loader = index_part(self.index, reduced=reduced, bounds=bounds) loader.load() end = time.time() diff --git a/swiss_TLM_api/swiss_TML_api/name_finding/swiss_name.py b/swiss_TLM_api/swiss_TML_api/name_finding/swiss_name.py index 43ac1218..87c651e3 100644 --- a/swiss_TLM_api/swiss_TML_api/name_finding/swiss_name.py +++ b/swiss_TLM_api/swiss_TML_api/name_finding/swiss_name.py @@ -1,11 +1,10 @@ class SwissName: - def __init__(self, name, object_type, x, y, h): + def __init__(self, name, object_type, x, y, h, interest=0): self.name = name self.object_type = object_type self.x = int(x) self.y = int(y) self.h = int(h) - def __str__(self): return "{} ({} | {}, {}, {})".format( self.name, self.object_type, self.x, self.y, self.h diff --git a/swiss_TLM_api/test_helper/print_map_snipped.py b/swiss_TLM_api/test_helper/print_map_snipped.py index e3b56189..a3f6f4e3 100644 --- a/swiss_TLM_api/test_helper/print_map_snipped.py +++ b/swiss_TLM_api/test_helper/print_map_snipped.py @@ -79,7 +79,7 @@ def __init__(self, lv03_pkt, zoom_level): response = results[index] index = index + 1 img = Image.open(BytesIO(response.content)) - img.thumbnail((self.TILE_WIDTH, self.TILE_WIDTH), Image.ANTIALIAS) + img.thumbnail((self.TILE_WIDTH, self.TILE_WIDTH), Image.Resampling.LANCZOS) w, h = img.size self.img.paste( img, diff --git a/swiss_TLM_api/test_name_finding.py b/swiss_TLM_api/test_name_finding.py index 19a816ad..a377d334 100644 --- a/swiss_TLM_api/test_name_finding.py +++ b/swiss_TLM_api/test_name_finding.py @@ -4,8 +4,10 @@ from PIL import ImageDraw from PIL import ImageFont +from shapely.geometry import LineString from swiss_TML_api.logging.log_helper import setup_recursive_logger +from swiss_TML_api.name_finding.interest import Interest from test_helper.print_map_snipped import MapImage setup_recursive_logger(logging.INFO) @@ -23,21 +25,23 @@ def get_random_coordinates_along_path(): pkt = lv03_path[random.randint(0, length - 1)] return pkt - if __name__ == "__main__": - name_index = NameFinder(force_rebuild=True, reduced=True) - lv03_coord = [2601835.8, 1196902.6] # get_random_coordinates_along_path() + lv03_coord = [2651537.71, 1114910.83] # get_random_coordinates_along_path() + bounds = [[lv03_coord[0] - 1000, lv03_coord[1] - 1000], [lv03_coord[0] + 1000, lv03_coord[1] + 1000]] + name_index = NameFinder(force_rebuild=False, reduced=False) + image = MapImage(lv03_coord, zoom_level=9) draw = ImageDraw.Draw(image.img) - fnt = ImageFont.truetype("Pillow/Tests/fonts/FreeMono.ttf", 18) - + fnt = ImageFont.load_default() # Set up the root handler - names = name_index.get_names(lv03_coord[0], lv03_coord[1], 75) - + names = name_index.get_names(lv03_coord[0], lv03_coord[1], 75) # for marking only + interest = Interest(name_index) + res = interest.select_name(lv03_coord) + print("bestRes", res) for name in names: - print(name) + #print(name) image.mark_point(draw, (name.x, name.y), 4) draw.rectangle((0, 0, image.img.size[0], 24), fill=(255, 255, 255)) From eb00e7733ef08852ddef58beadd5718a3af23473 Mon Sep 17 00:00:00 2001 From: Johann Schwabe Date: Mon, 7 Oct 2024 21:16:05 +0200 Subject: [PATCH 2/2] state --- swiss_TLM_api/test_helper/print_map_snipped.py | 1 - swiss_TLM_api/test_name_finding.py | 1 - 2 files changed, 2 deletions(-) diff --git a/swiss_TLM_api/test_helper/print_map_snipped.py b/swiss_TLM_api/test_helper/print_map_snipped.py index a3f6f4e3..8d11b2ec 100644 --- a/swiss_TLM_api/test_helper/print_map_snipped.py +++ b/swiss_TLM_api/test_helper/print_map_snipped.py @@ -72,7 +72,6 @@ def __init__(self, lv03_pkt, zoom_level): # Request images in parallel rs = (grequests.get(u) for u in urls) results = grequests.map(rs) - index = 0 for i in range(0, self.x_count): for j in range(1, self.y_count + 1): diff --git a/swiss_TLM_api/test_name_finding.py b/swiss_TLM_api/test_name_finding.py index a377d334..62e7dc2f 100644 --- a/swiss_TLM_api/test_name_finding.py +++ b/swiss_TLM_api/test_name_finding.py @@ -28,7 +28,6 @@ def get_random_coordinates_along_path(): if __name__ == "__main__": lv03_coord = [2651537.71, 1114910.83] # get_random_coordinates_along_path() - bounds = [[lv03_coord[0] - 1000, lv03_coord[1] - 1000], [lv03_coord[0] + 1000, lv03_coord[1] + 1000]] name_index = NameFinder(force_rebuild=False, reduced=False) image = MapImage(lv03_coord, zoom_level=9)