climbingTiles: exploration #2 – proper pg client + MVT z10 encoding

.gitignore

@@ -16,3 +16,4 @@ yarn-error.log*
 .env.local
 .env.sentry-build-plugin
 /public/icons-indoor
+/.xata/migrations/*.json

.xata/migrations/.ledger

@@ -0,0 +1,6 @@
+mig_csqvnk4icce8859die00
+mig_csqvvssicce8859die5g
+mig_ct2scmdc21vap51ejf8g
+sql_28451505e1dd5d
+sql_c642d1b1707c7a
+sql_8f7a225a10aa7d

.xata/version/compatibility.json

@@ -0,0 +1,10 @@
+{
+  "@xata.io/cli": {
+    "latest": "0.16.12",
+    "compatibility": [{ "range": ">=0.0.0" }]
+  },
+  "@xata.io/client": {
+    "latest": "0.30.1",
+    "compatibility": [{ "range": ">=0.0.0" }]
+  }
+}

.xatarc

@@ -0,0 +1,6 @@
+{
+  "databaseURL": "https://osmapp-tvgiad.us-east-1.xata.sh/db/db_with_direct_access",
+  "codegen": {
+    "output": "src/server/db/xata-generated.ts"
+  }
+}

package.json

@@ -26,20 +26,23 @@
     "@emotion/react": "^11.13.3",
     "@emotion/server": "^11.11.0",
     "@emotion/styled": "^11.13.0",
+    "@mapbox/tilebelt": "^2.0.2",
     "@mui/icons-material": "^6.1.3",
     "@mui/lab": "^6.0.0-beta.11",
     "@mui/material": "^6.1.3",
     "@mui/material-nextjs": "^6.1.3",
     "@openstreetmap/id-tagging-schema": "^6.8.1",
     "@sentry/nextjs": "^8.34.0",
     "@teritorio/openmaptiles-gl-language": "^1.5.4",
+    "@xata.io/client": "^0.0.0-next.va121e4207b94bfe0a3c025fc00b247b923880930",
     "@xmldom/xmldom": "^0.9.3",
     "accept-language-parser": "^1.5.0",
     "autosuggest-highlight": "^3.3.4",
     "canvas": "^2.11.2",
     "canvg": "^4.0.2",
     "date-fns": "^4.1.0",
     "dice-coefficient": "^2.1.1",
+    "geohashing": "^2.0.1",
     "image-size": "^1.1.1",
     "isomorphic-unfetch": "^4.0.2",
     "isomorphic-xml2js": "^0.1.3",
@@ -55,6 +58,8 @@
     "open-location-code": "^1.0.3",
     "opening_hours": "^3.8.0",
     "osm-auth": "^2.5.0",
+    "pg": "^8.13.1",
+    "pg-format": "^1.0.4",
     "react": "^18.3.1",
     "react-custom-scrollbars": "^4.2.1",
     "react-dom": "^18.3.1",

pages/api/climbing-tiles/refresh.ts

@@ -0,0 +1,24 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import { refresh } from '../../../src/server/climbing-tiles/refresh.js';
+
+export default async (req: NextApiRequest, res: NextApiResponse) => {
+  try {
+    res.writeHead(200, {
+      Connection: 'keep-alive',
+      'Content-Type': 'text/plain; charset=utf-8',
+      'Transfer-Encoding': 'chunked',
+    });
+
+    const writeCallback = (line: string) => {
+      console.log(line); // eslint-disable-line no-console
+      res.write(line + '\n');
+    };
+
+    await refresh(writeCallback);
+
+    res.end();
+  } catch (err) {
+    console.error(err); // eslint-disable-line no-console
+    res.status(err.code ?? 400).send(String(err));
+  }
+};

pages/api/climbing-tiles/tile.ts

@@ -0,0 +1,22 @@
+import type { NextApiRequest, NextApiResponse } from 'next';
+import {
+  climbingTile,
+  TileNumber,
+} from '../../../src/server/climbing-tiles/climbing-tile';
+
+export default async (req: NextApiRequest, res: NextApiResponse) => {
+  try {
+    const tileNumber = [req.query.z, req.query.x, req.query.y].map(
+      Number,
+    ) as TileNumber;
+
+    const buffer = await climbingTile(tileNumber, req.query.type);
+    res
+      .setHeader('Content-Type', 'application/x-protobuf')
+      .status(200)
+      .end(buffer);
+  } catch (err) {
+    console.error(err); // eslint-disable-line no-console
+    res.status(err.code ?? 400).send(String(err));
+  }
+};

src/components/Map/behaviour/useUpdateStyle.tsx

@@ -64,19 +64,25 @@ const addRasterOverlay = (
 };
 
 const addClimbingOverlay = (style: StyleSpecification, map: Map) => {
-  style.sources.climbing = EMPTY_GEOJSON_SOURCE;
-  style.layers.push(...climbingLayers); // must be also in `layersWithOsmId` because of hover effect
-  style.sprite = [...OSMAPP_SPRITE, CLIMBING_SPRITE];
-
-  fetchCrags().then(
-    (geojson) => {
-      const geojsonSource = map.getSource('climbing') as GeoJSONSource;
-      geojsonSource?.setData(geojson); // TODO can be undefined at first map render
-    },
-    (error) => {
-      console.warn('Climbing Layer failed to fetch.', error); // eslint-disable-line no-console
-    },
-  );
+  // style.sources.climbing = EMPTY_GEOJSON_SOURCE;
+  style.layers.push(
+    ...climbingLayers.map((x) => ({
+      ...x,
+      source: 'climbingTiles',
+      'source-layer': 'groups',
+    })),
+  ); // must be also in `layersWithOsmId` because of hover effect
+  // style.sprite = [...OSMAPP_SPRITE, CLIMBING_SPRITE];
+
+  // fetchCrags().then(
+  //   (geojson) => {
+  //     const geojsonSource = map.getSource('climbing') as GeoJSONSource;
+  //     geojsonSource?.setData(geojson); // TODO can be undefined at first map render
+  //   },
+  //   (error) => {
+  //     console.warn('Climbing Layer failed to fetch.', error); // eslint-disable-line no-console
+  //   },
+  // );
 };
 
 const addOverlaysToStyle = (

src/components/Map/consts.ts

@@ -50,6 +50,11 @@ export const OSMAPP_SOURCES: Record<string, SourceSpecification> = {
     type: 'vector' as const,
   },
   overpass: EMPTY_GEOJSON_SOURCE,
+  climbingTiles: {
+    type: 'vector' as const,
+    tiles: ['http://localhost:3000/api/climbing-tiles/tile?z={z}&x={x}&y={y}'],
+    maxzoom: 10,
+  },
 };
 
 export const BACKGROUND = [

src/components/Map/styles/layers/climbingLayers.ts

@@ -37,7 +37,7 @@ const linearByRouteCount = (
 ): ExpressionSpecification => [
   'interpolate',
   ['linear'],
-  ['get', 'osmappRouteCount'],
+  ['coalesce', ['get', 'osmappRouteCount'], 0],
   from,
   a,
   to,
@@ -75,8 +75,9 @@ const sortKey = [
   -1,
   [
     '+',
-    ['get', 'osmappRouteCount'],
-    ['case', ['get', 'osmappHasImages'], 99999, 0], // preference for items with images
+    ['to-number', ['get', 'osmappRouteCount']],
+    ['case', ['get', 'osmappHasImages'], 10000, 0], // preference for items with images
+    ['case', ['to-boolean', ['get', 'name']], 2, 0], // prefer items with name
   ],
 ] as DataDrivenPropertyValueSpecification<number>;
 
@@ -99,16 +100,19 @@ const step = (
 ];
 
 export const routes: LayerSpecification[] = [
+  {
+    id: 'climbing-3-routes-line',
+    type: 'line',
+    source: 'climbing',
+    minzoom: 16,
+    filter: ['all', ['==', 'type', 'route']],
+  },
   {
     id: 'climbing-3-routes-circle',
     type: 'circle',
     source: 'climbing',
     minzoom: 16,
-    filter: [
-      'all',
-      ['==', 'osmappType', 'node'],
-      ['==', 'climbing', 'route_bottom'],
-    ],
+    filter: ['all', ['==', 'type', 'route']],
     paint: {
       'circle-color': [
         'case',
@@ -125,11 +129,7 @@ export const routes: LayerSpecification[] = [
     type: 'symbol',
     source: 'climbing',
     minzoom: 19,
-    filter: [
-      'all',
-      ['==', 'osmappType', 'node'],
-      ['==', 'climbing', 'route_bottom'],
-    ],
+    filter: ['all', ['==', 'type', 'route']],
     layout: {
       'text-padding': 2,
       'text-font': ['Noto Sans Medium'],
@@ -179,11 +179,7 @@ const mixed: LayerSpecification = {
   type: 'symbol',
   source: 'climbing',
   maxzoom: 20,
-  filter: [
-    'all',
-    ['==', 'osmappType', 'relationPoint'],
-    ['in', 'climbing', 'area', 'crag'],
-  ],
+  filter: ['all', ['==', 'type', 'group']],
   layout: {
     'icon-image': ifCrag(
       byHasImages(CRAG, 'IMAGE'),

src/server/climbing-tiles/climbing-tile.ts

@@ -0,0 +1,60 @@
+import vtpbf from 'vt-pbf';
+import geojsonVt from 'geojson-vt';
+import { BBox } from 'geojson';
+import { getClient } from './db';
+
+const r2d = 180 / Math.PI;
+
+function tile2lon(x: number, z: number): number {
+  return (x / Math.pow(2, z)) * 360 - 180;
+}
+
+function tile2lat(y: number, z: number): number {
+  const n = Math.PI - (2 * Math.PI * y) / Math.pow(2, z);
+  return r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)));
+}
+export function tileToBBOX([z, x, y]: TileNumber): BBox {
+  const e = tile2lon(x + 1, z);
+  const w = tile2lon(x, z);
+  const s = tile2lat(y + 1, z);
+  const n = tile2lat(y, z);
+  return [w, s, e, n];
+}
+
+export type TileNumber = [number, number, number]; // z,x,y
+
+const fetchFromDb = async ([z, x, y]: TileNumber) => {
+  const start = performance.now();
+
+  const client = await getClient();
+
+  const bbox = tileToBBOX([z, x, y]);
+
+  const query =
+    z < 10
+      ? `SELECT geojson FROM climbing_tiles WHERE type='group' AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}`
+      : `SELECT geojson FROM climbing_tiles WHERE type IN ('group', 'route') AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}`;
+  const result = await client.query(query);
+  const geojson = {
+    type: 'FeatureCollection',
+    features: result.rows.map((record) => record.geojson),
+  } as GeoJSON.FeatureCollection;
+  console.log('climbingTilePg', performance.now() - start, result.rows.length);
+
+  return geojson;
+};
+
+export const climbingTile = async ([z, x, y]: TileNumber, type: string) => {
+  if (type === 'json') {
+    const orig = await fetchFromDb([z, x, y]);
+    return JSON.stringify(orig);
+  }
+
+  const orig = await fetchFromDb([z, x, y]);
+
+  const tileindex = geojsonVt(orig, { tolerance: 0 });
+  const tile = tileindex.getTile(z, x, y);
+  console.log(tile);
+
+  return tile ? vtpbf.fromGeojsonVt({ groups: tile }) : null;
+};

src/server/climbing-tiles/db.ts

@@ -0,0 +1,27 @@
+import { Client } from 'pg';
+
+if (!global.db) {
+  global.db = { pool: false };
+}
+
+export async function getClient() {
+  if (!global.db.pool) {
+    console.log('No pool available, creating new pool.');
+
+    const client = new Client({
+      user: 'tvgiad',
+      password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM',
+      host: 'us-east-1.sql.xata.sh',
+      port: 5432,
+      database: 'db_with_direct_access:main',
+      ssl: {
+        rejectUnauthorized: false,
+      },
+    });
+
+    await client.connect();
+
+    global.db.pool = client;
+  }
+  return global.db.pool;
+}