From 0885ec7a4ef562c5b6b53b4109830e43676e68b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Sun, 10 Nov 2024 11:23:26 +0100 Subject: [PATCH 01/22] xata init --- .xata/migrations/.ledger | 1 + .xata/version/compatibility.json | 10 +++++++++ .xatarc | 6 ++++++ package.json | 1 + src/server/db/db.ts | 3 +++ src/server/db/xata-generated.ts | 35 ++++++++++++++++++++++++++++++++ yarn.lock | 5 +++++ 7 files changed, 61 insertions(+) create mode 100644 .xata/migrations/.ledger create mode 100644 .xata/version/compatibility.json create mode 100644 .xatarc create mode 100644 src/server/db/db.ts create mode 100644 src/server/db/xata-generated.ts diff --git a/.xata/migrations/.ledger b/.xata/migrations/.ledger new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/.xata/migrations/.ledger @@ -0,0 +1 @@ + diff --git a/.xata/version/compatibility.json b/.xata/version/compatibility.json new file mode 100644 index 000000000..f4a7c9253 --- /dev/null +++ b/.xata/version/compatibility.json @@ -0,0 +1,10 @@ +{ + "@xata.io/cli": { + "latest": "0.16.12", + "compatibility": [{ "range": ">=0.0.0" }] + }, + "@xata.io/client": { + "latest": "0.30.1", + "compatibility": [{ "range": ">=0.0.0" }] + } +} diff --git a/.xatarc b/.xatarc new file mode 100644 index 000000000..48f846e4e --- /dev/null +++ b/.xatarc @@ -0,0 +1,6 @@ +{ + "databaseURL": "https://osmapp-tvgiad.us-east-1.xata.sh/db/osmapp", + "codegen": { + "output": "src/server/db/xata-generated.ts" + } +} \ No newline at end of file diff --git a/package.json b/package.json index 07ff9f8a5..14293ed3a 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "@openstreetmap/id-tagging-schema": "^6.8.1", "@sentry/nextjs": "^8.34.0", "@teritorio/openmaptiles-gl-language": "^1.5.4", + "@xata.io/client": "^0.30.1", "@xmldom/xmldom": "^0.9.3", "accept-language-parser": "^1.5.0", "autosuggest-highlight": "^3.3.4", diff --git a/src/server/db/db.ts b/src/server/db/db.ts new file mode 100644 index 000000000..a5cc4a3b5 --- /dev/null +++ b/src/server/db/db.ts @@ -0,0 +1,3 @@ +import { getXataClient } from './xata-generated'; + +export const xata = getXataClient(); diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts new file mode 100644 index 000000000..742811b0b --- /dev/null +++ b/src/server/db/xata-generated.ts @@ -0,0 +1,35 @@ +// Generated by Xata Codegen 0.30.1. Please do not edit. +import { buildClient } from '@xata.io/client'; +import type { + BaseClientOptions, + SchemaInference, + XataRecord, +} from '@xata.io/client'; + +const tables = [] as const; + +export type SchemaTables = typeof tables; +export type InferredTypes = SchemaInference; + +export type DatabaseSchema = {}; + +const DatabaseClient = buildClient(); + +const defaultOptions = { + databaseURL: 'https://osmapp-tvgiad.us-east-1.xata.sh/db/osmapp', +}; + +export class XataClient extends DatabaseClient { + constructor(options?: BaseClientOptions) { + super({ ...defaultOptions, ...options }, tables); + } +} + +let instance: XataClient | undefined = undefined; + +export const getXataClient = () => { + if (instance) return instance; + + instance = new XataClient(); + return instance; +}; diff --git a/yarn.lock b/yarn.lock index dedb7e949..8f6ddf620 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2276,6 +2276,11 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== +"@xata.io/client@^0.30.1": + version "0.30.1" + resolved "https://registry.yarnpkg.com/@xata.io/client/-/client-0.30.1.tgz#a7fe9729d7241f903d94d3b16c293816bea7d04b" + integrity sha512-dAzDPHmIfenVIpF39m1elmW5ngjWu2mO8ZqJBN7dmYdXr98uhPANfLdVZnc3mUNG+NH37LqY1dSO862hIo2oRw== + "@xmldom/xmldom@^0.9.3": version "0.9.3" resolved "https://registry.yarnpkg.com/@xmldom/xmldom/-/xmldom-0.9.3.tgz#a5d5362050767d8823b2b74e36cb2f059f58e797" From a39d0a409d0bcc75e11d97ed8a1264395320c2e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Sun, 10 Nov 2024 12:12:32 +0100 Subject: [PATCH 02/22] xata schema upload + xata pull main --- .xata/migrations/.ledger | 2 +- .../mig_cso99kp32vjpfm25jl70_e4b0e013.json | 92 +++++++++++++++++++ src/server/db/xata-generated.ts | 24 ++++- 3 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 .xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json diff --git a/.xata/migrations/.ledger b/.xata/migrations/.ledger index 8b1378917..6d0fcbf79 100644 --- a/.xata/migrations/.ledger +++ b/.xata/migrations/.ledger @@ -1 +1 @@ - +mig_cso99kp32vjpfm25jl70_e4b0e013 diff --git a/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json b/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json new file mode 100644 index 000000000..a55dc1e38 --- /dev/null +++ b/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json @@ -0,0 +1,92 @@ +{ + "id": "mig_cso99kp32vjpfm25jl70", + "checksum": "1:e4b0e01303c68771fd0139e5dc9ad466c630d7ab59ad428b8fc570348284df40", + "operations": [ + { + "addTable": { + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "type", + "type": "text" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "lon", + "type": "float" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "lat", + "type": "float" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "osmType", + "type": "text" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "osmId", + "type": "int" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "osmVersion", + "type": "int" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "name", + "type": "text" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "count", + "type": "int" + }, + "table": "climbing_tiles" + } + }, + { + "addColumn": { + "column": { + "name": "json", + "type": "json" + }, + "table": "climbing_tiles" + } + } + ] +} diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts index 742811b0b..f37c3143f 100644 --- a/src/server/db/xata-generated.ts +++ b/src/server/db/xata-generated.ts @@ -6,12 +6,32 @@ import type { XataRecord, } from '@xata.io/client'; -const tables = [] as const; +const tables = [ + { + name: 'climbing_tiles', + columns: [ + { name: 'type', type: 'text' }, + { name: 'lon', type: 'float' }, + { name: 'lat', type: 'float' }, + { name: 'osmType', type: 'text' }, + { name: 'osmId', type: 'int' }, + { name: 'osmVersion', type: 'int' }, + { name: 'name', type: 'text' }, + { name: 'count', type: 'int' }, + { name: 'json', type: 'json' }, + ], + }, +] as const; export type SchemaTables = typeof tables; export type InferredTypes = SchemaInference; -export type DatabaseSchema = {}; +export type ClimbingTiles = InferredTypes['climbing_tiles']; +export type ClimbingTilesRecord = ClimbingTiles & XataRecord; + +export type DatabaseSchema = { + climbing_tiles: ClimbingTilesRecord; +}; const DatabaseClient = buildClient(); From b51906044f32e4503b070d9e19c402c9b953ba3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Sun, 10 Nov 2024 17:51:47 +0100 Subject: [PATCH 03/22] fetchAll by single call (8 minutes= 3000 records) --- pages/api/climbing-tile.ts | 15 + src/server/climbing-tiles/algo.ts | 265 ++++++++++++++++++ .../overpass/__tests__/basic.test.ts | 0 .../overpass/overpassToGeojsons.ts | 238 ++++++++++++++++ src/services/getCenter.ts | 17 +- src/services/types.ts | 4 +- 6 files changed, 529 insertions(+), 10 deletions(-) create mode 100644 pages/api/climbing-tile.ts create mode 100644 src/server/climbing-tiles/algo.ts create mode 100644 src/server/climbing-tiles/overpass/__tests__/basic.test.ts create mode 100644 src/server/climbing-tiles/overpass/overpassToGeojsons.ts diff --git a/pages/api/climbing-tile.ts b/pages/api/climbing-tile.ts new file mode 100644 index 000000000..4ce937674 --- /dev/null +++ b/pages/api/climbing-tile.ts @@ -0,0 +1,15 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { serverFetchOsmUser } from '../../src/services/osmApiAuthServer'; +import { fetchAll } from '../../src/server/climbing-tiles/algo'; + +// TODO upgrade Nextjs and use export async function POST(request: NextRequest) { +export default async (req: NextApiRequest, res: NextApiResponse) => { + try { + const json = await fetchAll(); + + res.status(200).json(json); + } catch (err) { + console.error(err); // eslint-disable-line no-console + res.status(err.code ?? 400).send(String(err)); + } +}; diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts new file mode 100644 index 000000000..365f454b4 --- /dev/null +++ b/src/server/climbing-tiles/algo.ts @@ -0,0 +1,265 @@ +// import {encodeUrl} from "@/utils"; +// import * as fs from "node:fs/promises"; +// const query = `[out:json][timeout:80];(nwr["climbing"];nwr["sport"="climbing"];);>>;out qt;`; +// const res = await fetch("https://overpass-api.de/api/interpreter", { +// "body": encodeUrl`data=${query}`, +// "method": "POST" +// }); +// const data = await res.json(); +// fs.writeFile("data.json", JSON.stringify(data, null, 2)); + +import { xata } from '../db/db'; +import { overpassToGeojsons } from './overpass/overpassToGeojsons'; + +type OsmType = 'node' | 'way' | 'relation'; +type OsmNode = { + type: 'node'; + id: number; + lat: number; + lon: number; + tags: Record; +}; +type OsmWay = { + type: 'way'; + id: number; + nodes: number[]; + tags: Record; +}; +type OsmRelation = { + type: 'relation'; + id: number; + members: { + type: OsmType; + ref: number; + role: string; + }[]; + tags: Record; +}; +type OsmItem = OsmNode | OsmWay | OsmRelation; + +const elements: OsmItem[] = [ + // { + // "type": "node", + // "id": 313822575, + // "lat": 50.0547464, + // "lon": 14.4056821, + // "tags": { + // "climbing:boulder": "yes", + // "climbing:toprope": "yes", + // "leisure": "sports_centre", + // "name": "SmíchOFF", + // "opening_hours": "Mo 07:00-23:00; Tu-Th 07:00-23:30; Fr 07:00-23:00; Sa,Su 08:00-23:00", + // "sport": "climbing", + // "website": "https://www.lezeckecentrum.cz/" + // } + // }, + { + type: 'node', + id: 11580052710, + lat: 49.6600391, + lon: 14.2573987, + tags: { + climbing: 'route_bottom', + 'climbing:grade:uiaa': '9-', + name: 'Lída', + sport: 'climbing', + wikimedia_commons: 'File:Roviště - Hafty2.jpg', + 'wikimedia_commons:2': 'File:Roviště - Hafty3.jpg', + 'wikimedia_commons:2:path': + '0.273,0.904|0.229,0.566B|0.317,0.427B|0.433,0.329B|0.515,0.21B|0.526,0.126B|0.495,0.075A', + 'wikimedia_commons:path': + '0.67,0.601|0.66,0.442B|0.682,0.336B|0.739,0.236B|0.733,0.16B|0.72,0.1B|0.688,0.054A', + }, + }, + { + type: 'relation', + id: 17130663, + members: [ + { + type: 'node', + ref: 11580052710, + role: '', + }, + ], + tags: { + climbing: 'crag', + name: 'Yosemite (Hafty)', + site: 'climbing', + sport: 'climbing', + type: 'site', + wikimedia_commons: 'File:Roviště - Hafty.jpg', + 'wikimedia_commons:10': 'File:Roviště - Hafty10.jpg', + 'wikimedia_commons:2': 'File:Roviště - Hafty2.jpg', + 'wikimedia_commons:3': 'File:Roviště - Hafty3.jpg', + 'wikimedia_commons:4': 'File:Roviště - Hafty4.jpg', + 'wikimedia_commons:5': 'File:Roviště - Hafty5.jpg', + 'wikimedia_commons:6': 'File:Roviště - Hafty6.jpg', + 'wikimedia_commons:7': 'File:Roviště - Hafty7.jpg', + 'wikimedia_commons:8': 'File:Roviště - Hafty8.jpg', + 'wikimedia_commons:9': 'File:Roviště - Hafty9.jpg', + }, + }, + { + type: 'relation', + id: 17130099, + members: [ + { + type: 'relation', + ref: 17130663, + role: '', + }, + ], + tags: { + climbing: 'area', + description: + 'Roviště je klasická vltavská žula. Jedná se o velmi vyhlášenou oblast. Nabízí cesty prakticky všech obtížností, zpravidla dobře odjištěné.', + name: 'Roviště', + site: 'climbing', + type: 'site', + website: 'https://www.horosvaz.cz/skaly-sektor-289/', + 'website:2': 'https://www.lezec.cz/pruvodcx.php?key=5', + }, + }, + + // two nodes and a climbing=route way + { + type: 'node', + id: 1, + lat: 50, + lon: 14, + tags: {}, + }, + { + type: 'node', + id: 2, + lat: 51, + lon: 15, + tags: {}, + }, + { + type: 'way', + id: 3, + nodes: [1, 2], + tags: { + climbing: 'route', + name: 'Route of type way starting on 14,50', + }, + }, + + // two nodes and natural=cliff way ("crag") + { + type: 'node', + id: 4, + lat: 52, + lon: 16, + tags: {}, + }, + { + type: 'node', + id: 5, + lat: 53, + lon: 17, + tags: {}, + }, + { + type: 'way', + id: 6, + nodes: [4, 5], + tags: { + natural: 'cliff', + name: 'Cliff of type way at 16.5,52.5', + }, + }, +]; + +import * as fs from 'fs/promises'; + +export const fetchAll = async () => { + const file = await fs.readFile('data3_42s_25mb.json', 'utf8'); + const data = JSON.parse(file); // 200 ms + + const start0 = performance.now(); + const geojsons = overpassToGeojsons(data); + const start1 = performance.now(); + console.log('overpassToGeojsons', start1 - start0); + + // TODO start postgre transaction ?? + + await xata.sql`DELETE FROM climbing_tiles`; + const start2 = performance.now(); + console.log('delete', start2 - start1); + + for (const node of geojsons.node) { + if (node.tags?.climbing === 'route_bottom') { + await xata.db.climbing_tiles.create({ + type: 'route', + osmType: 'node', + osmId: node.osmMeta.id, + lon: node.geometry.coordinates[0], + lat: node.geometry.coordinates[1], + name: node.tags.name, + json: node, + }); + } + if (node.tags?.climbing === 'crag') { + await xata.db.climbing_tiles.create({ + type: 'group', + osmType: 'node', + osmId: node.osmMeta.id, + lon: node.geometry.coordinates[0], + lat: node.geometry.coordinates[1], + name: node.tags.name, + count: node.properties.osmappRouteCount, + json: node, + }); + } + } + const start3 = performance.now(); + console.log('node', start3 - start2); + + for (const way of geojsons.way) { + if (way.tags.climbing === 'route') { + const start = way.geometry.coordinates[0]; + await xata.db.climbing_tiles.create({ + type: 'route', + osmType: 'way', + osmId: way.osmMeta.id, + lon: start[0], + lat: start[1], + name: way.tags.name, + json: { ...way, geometry: { type: 'Point', coordinates: start } }, + }); + } else { + await xata.db.climbing_tiles.create({ + type: 'group', + osmType: 'way', + osmId: way.osmMeta.id, + lon: way.center[0], + lat: way.center[1], + name: way.tags.name, + count: way.properties.osmappRouteCount, + json: way, + }); + } + } + const start4 = performance.now(); + console.log('way', start4 - start3); + + for (const relation of geojsons.relation) { + await xata.db.climbing_tiles.create({ + type: 'group', + osmType: 'relation', + osmId: relation.osmMeta.id, + lon: relation.center[0], + lat: relation.center[1], + name: relation.tags.name, + count: relation.properties.osmappRouteCount, + json: relation, + }); + } + const start5 = performance.now(); + console.log('relation', start5 - start4); + console.log('fetchAll', start5 - start0); + + return []; +}; diff --git a/src/server/climbing-tiles/overpass/__tests__/basic.test.ts b/src/server/climbing-tiles/overpass/__tests__/basic.test.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts new file mode 100644 index 000000000..dd7290c01 --- /dev/null +++ b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts @@ -0,0 +1,238 @@ +import { + FeatureGeometry, + FeatureTags, + GeometryCollection, + LineString, + OsmId, + Point, +} from '../../../services/types'; +import { join } from '../../../utils'; +import { getCenter } from '../../../services/getCenter'; +import { getPoiClass } from '../../../services/getPoiClass'; + +type OsmType = 'node' | 'way' | 'relation'; +type OsmNode = { + type: 'node'; + id: number; + lat: number; + lon: number; + tags?: Record; +}; +type OsmWay = { + type: 'way'; + id: number; + nodes: number[]; + tags?: Record; +}; +type OsmRelation = { + type: 'relation'; + id: number; + members: { + type: OsmType; + ref: number; + role: string; + }[]; + tags?: Record; + center?: { lat: number; lon: number }; // only for overpass `out center` queries +}; +type OsmItem = OsmNode | OsmWay | OsmRelation; +type OsmResponse = { + elements: OsmItem[]; +}; + +type Feature = { + type: 'Feature'; + id: number; + osmMeta: OsmId; + tags: FeatureTags; + properties: { + class: string; + subclass: string; + [key: string]: string | number | boolean; + osmappRouteCount?: number; + osmappHasImages?: boolean; + osmappType?: 'node' | 'way' | 'relation'; + osmappLabel?: string; + }; + geometry: T; + center?: number[]; + members?: OsmRelation['members']; +}; + +type Lookup = { + node: Record>; + way: Record>; + relation: Record>; +}; + +const convertOsmIdToMapId = (apiId: OsmId) => { + const osmToMapType = { node: 0, way: 1, relation: 4 }; + return parseInt(`${apiId.id}${osmToMapType[apiId.type]}`, 10); +}; + +const getItems = (elements: OsmItem[]) => { + const nodes: OsmNode[] = []; + const ways: OsmWay[] = []; + const relations: OsmRelation[] = []; + elements.forEach((element) => { + if (element.type === 'node') { + nodes.push(element); + } else if (element.type === 'way') { + ways.push(element); + } else if (element.type === 'relation') { + relations.push(element); + } + }); + return { nodes, ways, relations }; +}; + +const numberToSuperScript = (number?: number) => + number?.toString().replace(/\d/g, (d) => '⁰¹²³⁴⁵⁶⁷⁸⁹'[+d]); + +const getLabel = (tags: FeatureTags, osmappRouteCount: number) => + join(tags?.name, '\n', numberToSuperScript(osmappRouteCount)); + +const convert = ( + element: T, + geometryFn: (element: T) => TGeometry, +): Feature => { + const { type, id, tags = {} } = element; + const geometry = geometryFn(element); + const center = getCenter(geometry) ?? undefined; + const osmappRouteCount = + element.tags?.climbing === 'crag' + ? Math.max( + element.type === 'relation' + ? element.members.filter((member) => member.role === '').length + : 0, + parseFloat(element.tags['climbing:sport'] ?? '0'), // TODO sum all types + ) + : undefined; + const properties = { + ...getPoiClass(tags), + ...tags, + osmappType: type, + osmappRouteCount, + osmappLabel: getLabel(tags, osmappRouteCount), + osmappHasImages: Object.keys(tags).some((key) => + key.startsWith('wikimedia_commons'), + ), + }; + + return { + type: 'Feature', + id: convertOsmIdToMapId({ type, id }), + osmMeta: { type, id }, + tags, + properties, + geometry, + center, + members: element.type === 'relation' ? element.members : undefined, + }; +}; + +const getNodeGeomFn = + () => + (node: any): Point => ({ + type: 'Point', + coordinates: [node.lon, node.lat], + }); + +const getWayGeomFn = + (lookup: Lookup) => + ({ nodes }: OsmWay): LineString => ({ + type: 'LineString' as const, + coordinates: nodes + .map((ref) => lookup.node[ref]?.geometry?.coordinates) + .filter(Boolean), // some nodes may be missing + }); + +const getRelationGeomFn = + (lookup: Lookup) => + ({ members, center }: OsmRelation): FeatureGeometry => { + const geometries = members + .map(({ type, ref }) => lookup[type][ref]?.geometry) + .filter(Boolean); // some members may be undefined in first pass + + return geometries.length + ? { + type: 'GeometryCollection', + geometries, + } + : center + ? { type: 'Point', coordinates: [center.lon, center.lat] } + : undefined; + }; + +const addToLookup = ( + items: Feature[], + lookup: Lookup, +) => { + items.forEach((item) => { + // @ts-ignore + lookup[item.osmMeta.type][item.osmMeta.id] = item; // eslint-disable-line no-param-reassign + }); +}; + +const getRelationWithAreaCount = ( + relations: Feature[], + lookup: Record>, +) => + relations.map((relation) => { + if (relation.tags?.climbing === 'area') { + const members = relation.members.map( + ({ type, ref }) => lookup[type][ref]?.properties, + ); + const osmappRouteCount = members + .map((member) => member?.osmappRouteCount ?? 0) + .reduce((acc, count) => acc + count); + const osmappHasImages = members + .map((member) => member?.osmappHasImages) + .some((value) => value === true); + + return { + ...relation, + properties: { + ...relation.properties, + osmappRouteCount, + osmappHasImages, + osmappLabel: getLabel(relation.tags, osmappRouteCount), + }, + }; + } + + return relation; + }); + +export const overpassToGeojsons = (response: OsmResponse) => { + const { nodes, ways, relations } = getItems(response.elements); + + const lookup = { node: {}, way: {}, relation: {} } as Lookup; + + const NODE_GEOM = getNodeGeomFn(); + const nodesOut = nodes.map((node) => convert(node, NODE_GEOM)); + addToLookup(nodesOut, lookup); + + const WAY_GEOM = getWayGeomFn(lookup); + const waysOut = ways.map((way) => convert(way, WAY_GEOM)); + addToLookup(waysOut, lookup); + + // first pass + const RELATION_GEOM1 = getRelationGeomFn(lookup); + const relationsOut1 = relations.map((relation) => + convert(relation, RELATION_GEOM1), + ); + addToLookup(relationsOut1, lookup); + + // second pass for climbing=area geometries + // TODO: loop while number of geometries changes + // TODO: update only geometries (?) + const RELATION_GEOM2 = getRelationGeomFn(lookup); + const relationsOut2 = relations.map((relation) => + convert(relation, RELATION_GEOM2), + ); + + const relationsOut3 = getRelationWithAreaCount(relationsOut2, lookup); + + return { node: nodesOut, way: waysOut, relation: relationsOut3 }; +}; diff --git a/src/services/getCenter.ts b/src/services/getCenter.ts index 836490aae..9ecb0d29d 100644 --- a/src/services/getCenter.ts +++ b/src/services/getCenter.ts @@ -5,7 +5,7 @@ import { isLineString, isPoint, isPolygon, - Position, + LonLat, } from './types'; export type NamedBbox = { @@ -15,7 +15,7 @@ export type NamedBbox = { n: number; }; -export const getBbox = (coordinates: Position[]): NamedBbox => { +export const getBbox = (coordinates: LonLat[]): NamedBbox => { const [firstX, firstY] = coordinates[0]; const initialBbox = { w: firstX, s: firstY, e: firstX, n: firstY }; @@ -30,16 +30,17 @@ export const getBbox = (coordinates: Position[]): NamedBbox => { ); }; -const getCenterOfBbox = (points: Position[]) => { +const getCenterOfBbox = (points: LonLat[]): LonLat | undefined => { if (!points.length) return undefined; - const { w, s, e, n } = getBbox(points); // [WSEN] - const lon = (w + e) / 2; // flat earth rulezz - const lat = (s + n) / 2; + const { w, s, e, n } = getBbox(points); + const lon = w + (e - w) / 2; // flat earth rulezz + const lat = s + (n - s) / 2; + return [lon, lat]; }; -const getPointsRecursive = (geometry: GeometryCollection): Position[] => +const getPointsRecursive = (geometry: GeometryCollection): LonLat[] => geometry.geometries.flatMap((subGeometry) => { if (isGeometryCollection(subGeometry)) { return getPointsRecursive(subGeometry); @@ -53,7 +54,7 @@ const getPointsRecursive = (geometry: GeometryCollection): Position[] => return []; }); -export const getCenter = (geometry: FeatureGeometry): Position => { +export const getCenter = (geometry: FeatureGeometry): LonLat => { if (isPoint(geometry)) { return geometry.coordinates; } diff --git a/src/services/types.ts b/src/services/types.ts index 9ccb13c51..f3b537832 100644 --- a/src/services/types.ts +++ b/src/services/types.ts @@ -79,7 +79,7 @@ export type RelationMember = { }; // TODO split in two types /extend/ -export interface Feature { +export type Feature = { point?: boolean; // TODO rename to isMarker or isCoords type: 'Feature'; id?: number; // for map hover effect @@ -125,7 +125,7 @@ export interface Feature { state?: { hover: boolean }; skeleton?: boolean; // that means loading is in progress nonOsmObject?: boolean; -} +}; export type MessagesType = typeof Vocabulary; export type TranslationId = keyof MessagesType; From 45a06123cb54340ddc9112027a040df804cb9289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 12:01:57 +0100 Subject: [PATCH 04/22] rework to transaction --- pages/api/climbing-tile.ts | 5 +- src/server/climbing-tiles/algo.ts | 76 ++++++++++++++++++++----------- src/services/fetchCrags.tsx | 6 +++ 3 files changed, 58 insertions(+), 29 deletions(-) diff --git a/pages/api/climbing-tile.ts b/pages/api/climbing-tile.ts index 4ce937674..7afa63853 100644 --- a/pages/api/climbing-tile.ts +++ b/pages/api/climbing-tile.ts @@ -1,11 +1,10 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { serverFetchOsmUser } from '../../src/services/osmApiAuthServer'; -import { fetchAll } from '../../src/server/climbing-tiles/algo'; +import { climbingTile } from '../../src/server/climbing-tiles/algo'; // TODO upgrade Nextjs and use export async function POST(request: NextRequest) { export default async (req: NextApiRequest, res: NextApiResponse) => { try { - const json = await fetchAll(); + const json = await climbingTile(); res.status(200).json(json); } catch (err) { diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts index 365f454b4..e46660235 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/algo.ts @@ -10,6 +10,8 @@ import { xata } from '../db/db'; import { overpassToGeojsons } from './overpass/overpassToGeojsons'; +import * as fs from 'fs/promises'; +import { chunk } from 'lodash'; type OsmType = 'node' | 'way' | 'relation'; type OsmNode = { @@ -172,26 +174,16 @@ const elements: OsmItem[] = [ }, ]; -import * as fs from 'fs/promises'; - -export const fetchAll = async () => { +const getInserts = async () => { const file = await fs.readFile('data3_42s_25mb.json', 'utf8'); const data = JSON.parse(file); // 200 ms + const geojsons = overpassToGeojsons(data); // 700 ms - const start0 = performance.now(); - const geojsons = overpassToGeojsons(data); - const start1 = performance.now(); - console.log('overpassToGeojsons', start1 - start0); - - // TODO start postgre transaction ?? - - await xata.sql`DELETE FROM climbing_tiles`; - const start2 = performance.now(); - console.log('delete', start2 - start1); + const inserts = []; for (const node of geojsons.node) { if (node.tags?.climbing === 'route_bottom') { - await xata.db.climbing_tiles.create({ + inserts.push({ type: 'route', osmType: 'node', osmId: node.osmMeta.id, @@ -202,7 +194,7 @@ export const fetchAll = async () => { }); } if (node.tags?.climbing === 'crag') { - await xata.db.climbing_tiles.create({ + inserts.push({ type: 'group', osmType: 'node', osmId: node.osmMeta.id, @@ -214,13 +206,11 @@ export const fetchAll = async () => { }); } } - const start3 = performance.now(); - console.log('node', start3 - start2); for (const way of geojsons.way) { if (way.tags.climbing === 'route') { const start = way.geometry.coordinates[0]; - await xata.db.climbing_tiles.create({ + inserts.push({ type: 'route', osmType: 'way', osmId: way.osmMeta.id, @@ -230,7 +220,7 @@ export const fetchAll = async () => { json: { ...way, geometry: { type: 'Point', coordinates: start } }, }); } else { - await xata.db.climbing_tiles.create({ + inserts.push({ type: 'group', osmType: 'way', osmId: way.osmMeta.id, @@ -242,11 +232,9 @@ export const fetchAll = async () => { }); } } - const start4 = performance.now(); - console.log('way', start4 - start3); for (const relation of geojsons.relation) { - await xata.db.climbing_tiles.create({ + inserts.push({ type: 'group', osmType: 'relation', osmId: relation.osmMeta.id, @@ -257,9 +245,45 @@ export const fetchAll = async () => { json: relation, }); } - const start5 = performance.now(); - console.log('relation', start5 - start4); - console.log('fetchAll', start5 - start0); - return []; + return inserts; +}; + +export const fetchAll = async () => { + return; + + await xata.sql`DELETE FROM climbing_tiles`; + + const inserts = await getInserts(); // 16k inserts + + const chunks = chunk(inserts, 1000); + + for (const chunk of chunks) { + await xata.transactions.run( + chunk.map((record) => ({ + insert: { + table: 'climbing_tiles', + record: { ...record, json: JSON.stringify(record.json, null, 2) }, + }, + })), + ); + + // avg 700kb per 1000 records, takes 5 secs + } + + // total 69 seconds +}; + +// all 8 secs +export const climbingTile = async () => { + const start = performance.now(); + const alldata = await xata.db.climbing_tiles + .filter({ + type: 'route', + }) + .getAll(); + + console.log('climbingTile', performance.now() - start); + + return alldata.map((record) => record.json); }; diff --git a/src/services/fetchCrags.tsx b/src/services/fetchCrags.tsx index 8ae9088bd..3210a140c 100644 --- a/src/services/fetchCrags.tsx +++ b/src/services/fetchCrags.tsx @@ -183,6 +183,12 @@ export const cragsToGeojson = (response: any): Feature[] => { // on CZ 48,11,51,19 makes 12 MB (only crags is 700kB) export const fetchCrags = async () => { + const tile = await fetchJson('/api/climbing-tile'); + return { + type: 'FeatureCollection', + features: tile, + } as GeoJSON.FeatureCollection; + const query = `[out:json][timeout:25]; ( nwr["climbing"](49.64474,14.21855,49.67273,14.28025); From 238cd0f98ebbe25ac486587fa7ad76a85d3746a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 16:29:21 +0100 Subject: [PATCH 05/22] column change to geojson + tweak size --- .gitignore | 1 + .xata/migrations/.ledger | 2 + pages/api/climbing-tiles/refresh.ts | 13 +++ .../tile.ts} | 3 +- .../Map/styles/layers/climbingLayers.ts | 5 +- src/server/climbing-tiles/algo.ts | 82 ++++++++++++------- .../overpass/overpassToGeojsons.ts | 15 ++-- src/server/db/xata-generated.ts | 19 ++++- src/services/fetchCrags.tsx | 3 +- 9 files changed, 101 insertions(+), 42 deletions(-) create mode 100644 pages/api/climbing-tiles/refresh.ts rename pages/api/{climbing-tile.ts => climbing-tiles/tile.ts} (69%) diff --git a/.gitignore b/.gitignore index ed84fc933..c1a864a70 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ yarn-error.log* .env.local .env.sentry-build-plugin /public/icons-indoor +/.xata/migrations/*.json diff --git a/.xata/migrations/.ledger b/.xata/migrations/.ledger index 6d0fcbf79..3f0cc2200 100644 --- a/.xata/migrations/.ledger +++ b/.xata/migrations/.ledger @@ -1 +1,3 @@ mig_cso99kp32vjpfm25jl70_e4b0e013 +mig_csp0306qn9e0obu1aakg_1515dd60 +mig_csp2358b12tq4cm03vtg_67247356 diff --git a/pages/api/climbing-tiles/refresh.ts b/pages/api/climbing-tiles/refresh.ts new file mode 100644 index 000000000..ec0d66d53 --- /dev/null +++ b/pages/api/climbing-tiles/refresh.ts @@ -0,0 +1,13 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import { climbingTile, refresh } from '../../../src/server/climbing-tiles/algo'; + +export default async (req: NextApiRequest, res: NextApiResponse) => { + try { + const json = await refresh(); + + res.status(200).json(json); + } catch (err) { + console.error(err); // eslint-disable-line no-console + res.status(err.code ?? 400).send(String(err)); + } +}; diff --git a/pages/api/climbing-tile.ts b/pages/api/climbing-tiles/tile.ts similarity index 69% rename from pages/api/climbing-tile.ts rename to pages/api/climbing-tiles/tile.ts index 7afa63853..31c1bfdb8 100644 --- a/pages/api/climbing-tile.ts +++ b/pages/api/climbing-tiles/tile.ts @@ -1,7 +1,6 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { climbingTile } from '../../src/server/climbing-tiles/algo'; +import { climbingTile } from '../../../src/server/climbing-tiles/algo'; -// TODO upgrade Nextjs and use export async function POST(request: NextRequest) { export default async (req: NextApiRequest, res: NextApiResponse) => { try { const json = await climbingTile(); diff --git a/src/components/Map/styles/layers/climbingLayers.ts b/src/components/Map/styles/layers/climbingLayers.ts index a625fbad9..a6475770d 100644 --- a/src/components/Map/styles/layers/climbingLayers.ts +++ b/src/components/Map/styles/layers/climbingLayers.ts @@ -181,8 +181,9 @@ const mixed: LayerSpecification = { maxzoom: 20, filter: [ 'all', - ['==', 'osmappType', 'relationPoint'], - ['in', 'climbing', 'area', 'crag'], + ['==', 'type', 'group'], + // ['==', 'osmappType', 'relationPoint'], + // ['in', 'climbing', 'area', 'crag'], ], layout: { 'icon-image': ifCrag( diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts index e46660235..2b4513f16 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/algo.ts @@ -9,9 +9,14 @@ // fs.writeFile("data.json", JSON.stringify(data, null, 2)); import { xata } from '../db/db'; -import { overpassToGeojsons } from './overpass/overpassToGeojsons'; +import { + geometryToPoint, + overpassToGeojsons, +} from './overpass/overpassToGeojsons'; import * as fs from 'fs/promises'; import { chunk } from 'lodash'; +import { EditableData, TransactionOperation } from '@xata.io/client'; +import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; type OsmType = 'node' | 'way' | 'relation'; type OsmNode = { @@ -174,27 +179,27 @@ const elements: OsmItem[] = [ }, ]; -const getInserts = async () => { +const getNewRecords = async () => { const file = await fs.readFile('data3_42s_25mb.json', 'utf8'); const data = JSON.parse(file); // 200 ms const geojsons = overpassToGeojsons(data); // 700 ms - const inserts = []; + const records: Partial>[] = []; for (const node of geojsons.node) { if (node.tags?.climbing === 'route_bottom') { - inserts.push({ + records.push({ type: 'route', osmType: 'node', osmId: node.osmMeta.id, lon: node.geometry.coordinates[0], lat: node.geometry.coordinates[1], name: node.tags.name, - json: node, + geojson: node, }); } if (node.tags?.climbing === 'crag') { - inserts.push({ + records.push({ type: 'group', osmType: 'node', osmId: node.osmMeta.id, @@ -202,7 +207,7 @@ const getInserts = async () => { lat: node.geometry.coordinates[1], name: node.tags.name, count: node.properties.osmappRouteCount, - json: node, + geojson: node, }); } } @@ -210,17 +215,17 @@ const getInserts = async () => { for (const way of geojsons.way) { if (way.tags.climbing === 'route') { const start = way.geometry.coordinates[0]; - inserts.push({ + records.push({ type: 'route', osmType: 'way', osmId: way.osmMeta.id, lon: start[0], lat: start[1], name: way.tags.name, - json: { ...way, geometry: { type: 'Point', coordinates: start } }, + geojson: { ...way, geometry: { type: 'Point', coordinates: start } }, }); } else { - inserts.push({ + records.push({ type: 'group', osmType: 'way', osmId: way.osmMeta.id, @@ -228,13 +233,13 @@ const getInserts = async () => { lat: way.center[1], name: way.tags.name, count: way.properties.osmappRouteCount, - json: way, + geojson: geometryToPoint(way), }); } } for (const relation of geojsons.relation) { - inserts.push({ + records.push({ type: 'group', osmType: 'relation', osmId: relation.osmMeta.id, @@ -242,36 +247,38 @@ const getInserts = async () => { lat: relation.center[1], name: relation.tags.name, count: relation.properties.osmappRouteCount, - json: relation, + geojson: geometryToPoint(relation), }); } - return inserts; + return records; }; -export const fetchAll = async () => { - return; - +export const refresh = async () => { + const start = performance.now(); await xata.sql`DELETE FROM climbing_tiles`; - const inserts = await getInserts(); // 16k inserts - + const inserts = await getNewRecords(); // ~ 16k records const chunks = chunk(inserts, 1000); - for (const chunk of chunks) { await xata.transactions.run( - chunk.map((record) => ({ - insert: { - table: 'climbing_tiles', - record: { ...record, json: JSON.stringify(record.json, null, 2) }, - }, - })), + // avg 700kb per 1000 records, takes ~5 secs + chunk.map( + (record) => + ({ + insert: { + table: 'climbing_tiles', + record: { ...record, json: JSON.stringify(record.geojson) }, + }, + }) as TransactionOperation, + ), ); - - // avg 700kb per 1000 records, takes 5 secs } - // total 69 seconds + return { + createdRecords: inserts.length, + durationMs: Math.round(performance.now() - start), // total 69 seconds + }; }; // all 8 secs @@ -279,11 +286,24 @@ export const climbingTile = async () => { const start = performance.now(); const alldata = await xata.db.climbing_tiles .filter({ - type: 'route', + type: 'group', }) .getAll(); + // .getMany({ + // pagination: { size: 1000, offset: 0 }, + // }); console.log('climbingTile', performance.now() - start); - return alldata.map((record) => record.json); + return alldata.map((record) => ({ + ...record.geojson, + properties: { + ...record.geojson.properties, + type: record.type, + }, + geometry: { + type: 'Point', + coordinates: [record.lon, record.lat], + }, + })); }; diff --git a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts index dd7290c01..6e401afbb 100644 --- a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts +++ b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts @@ -46,9 +46,7 @@ type Feature = { osmMeta: OsmId; tags: FeatureTags; properties: { - class: string; - subclass: string; - [key: string]: string | number | boolean; + climbing?: string; osmappRouteCount?: number; osmappHasImages?: boolean; osmappType?: 'node' | 'way' | 'relation'; @@ -109,8 +107,7 @@ const convert = ( ) : undefined; const properties = { - ...getPoiClass(tags), - ...tags, + climbing: tags?.climbing, osmappType: type, osmappRouteCount, osmappLabel: getLabel(tags, osmappRouteCount), @@ -204,6 +201,14 @@ const getRelationWithAreaCount = ( return relation; }); +export const geometryToPoint = (feature: Feature) => ({ + ...feature, + geometry: { + type: 'Point', + coordinates: feature.center, + }, +}); + export const overpassToGeojsons = (response: OsmResponse) => { const { nodes, ways, relations } = getItems(response.elements); diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts index f37c3143f..f513debaf 100644 --- a/src/server/db/xata-generated.ts +++ b/src/server/db/xata-generated.ts @@ -8,7 +8,7 @@ import type { const tables = [ { - name: 'climbing_tiles', + name: 'climbing_tiles-oldsunday', columns: [ { name: 'type', type: 'text' }, { name: 'lon', type: 'float' }, @@ -21,15 +21,32 @@ const tables = [ { name: 'json', type: 'json' }, ], }, + { + name: 'climbing_tiles', + columns: [ + { name: 'type', type: 'text' }, + { name: 'lon', type: 'float' }, + { name: 'lat', type: 'float' }, + { name: 'osmType', type: 'text' }, + { name: 'osmId', type: 'int' }, + { name: 'name', type: 'text' }, + { name: 'count', type: 'int' }, + { name: 'geojson', type: 'json' }, + ], + }, ] as const; export type SchemaTables = typeof tables; export type InferredTypes = SchemaInference; +export type ClimbingTilesOldsunday = InferredTypes['climbing_tiles-oldsunday']; +export type ClimbingTilesOldsundayRecord = ClimbingTilesOldsunday & XataRecord; + export type ClimbingTiles = InferredTypes['climbing_tiles']; export type ClimbingTilesRecord = ClimbingTiles & XataRecord; export type DatabaseSchema = { + 'climbing_tiles-oldsunday': ClimbingTilesOldsundayRecord; climbing_tiles: ClimbingTilesRecord; }; diff --git a/src/services/fetchCrags.tsx b/src/services/fetchCrags.tsx index 3210a140c..daab551a3 100644 --- a/src/services/fetchCrags.tsx +++ b/src/services/fetchCrags.tsx @@ -183,7 +183,8 @@ export const cragsToGeojson = (response: any): Feature[] => { // on CZ 48,11,51,19 makes 12 MB (only crags is 700kB) export const fetchCrags = async () => { - const tile = await fetchJson('/api/climbing-tile'); + const tile = await fetchJson('/api/climbing-tiles/tile', { nocache: true }); + publishDbgObject('fetchCrags', tile); return { type: 'FeatureCollection', features: tile, From 59cba7b40eb18659f0618264970bbba5910651cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 16:58:28 +0100 Subject: [PATCH 06/22] smaller geojson --- src/server/climbing-tiles/algo.ts | 202 +++--------------- .../overpass/__tests__/basic.test.ts | 134 ++++++++++++ .../overpass/overpassToGeojsons.ts | 25 +-- 3 files changed, 169 insertions(+), 192 deletions(-) diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts index 2b4513f16..5bb2d74f3 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/algo.ts @@ -10,7 +10,7 @@ import { xata } from '../db/db'; import { - geometryToPoint, + GeojsonFeature, overpassToGeojsons, } from './overpass/overpassToGeojsons'; import * as fs from 'fs/promises'; @@ -18,166 +18,21 @@ import { chunk } from 'lodash'; import { EditableData, TransactionOperation } from '@xata.io/client'; import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; -type OsmType = 'node' | 'way' | 'relation'; -type OsmNode = { - type: 'node'; - id: number; - lat: number; - lon: number; - tags: Record; -}; -type OsmWay = { - type: 'way'; - id: number; - nodes: number[]; - tags: Record; -}; -type OsmRelation = { - type: 'relation'; - id: number; - members: { - type: OsmType; - ref: number; - role: string; - }[]; - tags: Record; -}; -type OsmItem = OsmNode | OsmWay | OsmRelation; - -const elements: OsmItem[] = [ - // { - // "type": "node", - // "id": 313822575, - // "lat": 50.0547464, - // "lon": 14.4056821, - // "tags": { - // "climbing:boulder": "yes", - // "climbing:toprope": "yes", - // "leisure": "sports_centre", - // "name": "SmíchOFF", - // "opening_hours": "Mo 07:00-23:00; Tu-Th 07:00-23:30; Fr 07:00-23:00; Sa,Su 08:00-23:00", - // "sport": "climbing", - // "website": "https://www.lezeckecentrum.cz/" - // } - // }, - { - type: 'node', - id: 11580052710, - lat: 49.6600391, - lon: 14.2573987, - tags: { - climbing: 'route_bottom', - 'climbing:grade:uiaa': '9-', - name: 'Lída', - sport: 'climbing', - wikimedia_commons: 'File:Roviště - Hafty2.jpg', - 'wikimedia_commons:2': 'File:Roviště - Hafty3.jpg', - 'wikimedia_commons:2:path': - '0.273,0.904|0.229,0.566B|0.317,0.427B|0.433,0.329B|0.515,0.21B|0.526,0.126B|0.495,0.075A', - 'wikimedia_commons:path': - '0.67,0.601|0.66,0.442B|0.682,0.336B|0.739,0.236B|0.733,0.16B|0.72,0.1B|0.688,0.054A', - }, - }, - { - type: 'relation', - id: 17130663, - members: [ - { - type: 'node', - ref: 11580052710, - role: '', - }, - ], - tags: { - climbing: 'crag', - name: 'Yosemite (Hafty)', - site: 'climbing', - sport: 'climbing', - type: 'site', - wikimedia_commons: 'File:Roviště - Hafty.jpg', - 'wikimedia_commons:10': 'File:Roviště - Hafty10.jpg', - 'wikimedia_commons:2': 'File:Roviště - Hafty2.jpg', - 'wikimedia_commons:3': 'File:Roviště - Hafty3.jpg', - 'wikimedia_commons:4': 'File:Roviště - Hafty4.jpg', - 'wikimedia_commons:5': 'File:Roviště - Hafty5.jpg', - 'wikimedia_commons:6': 'File:Roviště - Hafty6.jpg', - 'wikimedia_commons:7': 'File:Roviště - Hafty7.jpg', - 'wikimedia_commons:8': 'File:Roviště - Hafty8.jpg', - 'wikimedia_commons:9': 'File:Roviště - Hafty9.jpg', - }, - }, - { - type: 'relation', - id: 17130099, - members: [ - { - type: 'relation', - ref: 17130663, - role: '', - }, - ], - tags: { - climbing: 'area', - description: - 'Roviště je klasická vltavská žula. Jedná se o velmi vyhlášenou oblast. Nabízí cesty prakticky všech obtížností, zpravidla dobře odjištěné.', - name: 'Roviště', - site: 'climbing', - type: 'site', - website: 'https://www.horosvaz.cz/skaly-sektor-289/', - 'website:2': 'https://www.lezec.cz/pruvodcx.php?key=5', - }, +export const geometryToPoint = (feature: GeojsonFeature): GeojsonFeature => ({ + ...feature, + geometry: { + type: 'Point', + coordinates: feature.center, }, +}); - // two nodes and a climbing=route way - { - type: 'node', - id: 1, - lat: 50, - lon: 14, - tags: {}, - }, - { - type: 'node', - id: 2, - lat: 51, - lon: 15, - tags: {}, - }, - { - type: 'way', - id: 3, - nodes: [1, 2], - tags: { - climbing: 'route', - name: 'Route of type way starting on 14,50', - }, - }, - - // two nodes and natural=cliff way ("crag") - { - type: 'node', - id: 4, - lat: 52, - lon: 16, - tags: {}, - }, - { - type: 'node', - id: 5, - lat: 53, - lon: 17, - tags: {}, - }, - { - type: 'way', - id: 6, - nodes: [4, 5], - tags: { - natural: 'cliff', - name: 'Cliff of type way at 16.5,52.5', - }, - }, -]; +const prepareGeojson = ({ id, geometry, properties }: GeojsonFeature) => + JSON.stringify({ + type: 'Feature', + id, + geometry, + properties, + }); const getNewRecords = async () => { const file = await fs.readFile('data3_42s_25mb.json', 'utf8'); @@ -195,7 +50,7 @@ const getNewRecords = async () => { lon: node.geometry.coordinates[0], lat: node.geometry.coordinates[1], name: node.tags.name, - geojson: node, + geojson: prepareGeojson(node), }); } if (node.tags?.climbing === 'crag') { @@ -207,7 +62,7 @@ const getNewRecords = async () => { lat: node.geometry.coordinates[1], name: node.tags.name, count: node.properties.osmappRouteCount, - geojson: node, + geojson: prepareGeojson(node), }); } } @@ -222,7 +77,10 @@ const getNewRecords = async () => { lon: start[0], lat: start[1], name: way.tags.name, - geojson: { ...way, geometry: { type: 'Point', coordinates: start } }, + geojson: prepareGeojson({ + ...way, + geometry: { type: 'Point', coordinates: start }, + }), }); } else { records.push({ @@ -233,7 +91,7 @@ const getNewRecords = async () => { lat: way.center[1], name: way.tags.name, count: way.properties.osmappRouteCount, - geojson: geometryToPoint(way), + geojson: prepareGeojson(geometryToPoint(way)), }); } } @@ -247,7 +105,7 @@ const getNewRecords = async () => { lat: relation.center[1], name: relation.tags.name, count: relation.properties.osmappRouteCount, - geojson: geometryToPoint(relation), + geojson: prepareGeojson(geometryToPoint(relation)), }); } @@ -258,26 +116,24 @@ export const refresh = async () => { const start = performance.now(); await xata.sql`DELETE FROM climbing_tiles`; - const inserts = await getNewRecords(); // ~ 16k records - const chunks = chunk(inserts, 1000); + const records = await getNewRecords(); // ~ 16k records + const chunks = chunk(records, 1000); for (const chunk of chunks) { await xata.transactions.run( // avg 700kb per 1000 records, takes ~5 secs chunk.map( (record) => ({ - insert: { - table: 'climbing_tiles', - record: { ...record, json: JSON.stringify(record.geojson) }, - }, + insert: { table: 'climbing_tiles', record }, }) as TransactionOperation, ), ); } return { - createdRecords: inserts.length, + createdRecords: records.length, durationMs: Math.round(performance.now() - start), // total 69 seconds + sizeKb: Math.round(JSON.stringify(records).length / 1000), }; }; @@ -301,9 +157,5 @@ export const climbingTile = async () => { ...record.geojson.properties, type: record.type, }, - geometry: { - type: 'Point', - coordinates: [record.lon, record.lat], - }, })); }; diff --git a/src/server/climbing-tiles/overpass/__tests__/basic.test.ts b/src/server/climbing-tiles/overpass/__tests__/basic.test.ts index e69de29bb..fd2e63fa5 100644 --- a/src/server/climbing-tiles/overpass/__tests__/basic.test.ts +++ b/src/server/climbing-tiles/overpass/__tests__/basic.test.ts @@ -0,0 +1,134 @@ +const elements: OsmItem[] = [ + // { + // "type": "node", + // "id": 313822575, + // "lat": 50.0547464, + // "lon": 14.4056821, + // "tags": { + // "climbing:boulder": "yes", + // "climbing:toprope": "yes", + // "leisure": "sports_centre", + // "name": "SmíchOFF", + // "opening_hours": "Mo 07:00-23:00; Tu-Th 07:00-23:30; Fr 07:00-23:00; Sa,Su 08:00-23:00", + // "sport": "climbing", + // "website": "https://www.lezeckecentrum.cz/" + // } + // }, + { + type: 'node', + id: 11580052710, + lat: 49.6600391, + lon: 14.2573987, + tags: { + climbing: 'route_bottom', + 'climbing:grade:uiaa': '9-', + name: 'Lída', + sport: 'climbing', + wikimedia_commons: 'File:Roviště - Hafty2.jpg', + 'wikimedia_commons:2': 'File:Roviště - Hafty3.jpg', + 'wikimedia_commons:2:path': + '0.273,0.904|0.229,0.566B|0.317,0.427B|0.433,0.329B|0.515,0.21B|0.526,0.126B|0.495,0.075A', + 'wikimedia_commons:path': + '0.67,0.601|0.66,0.442B|0.682,0.336B|0.739,0.236B|0.733,0.16B|0.72,0.1B|0.688,0.054A', + }, + }, + { + type: 'relation', + id: 17130663, + members: [ + { + type: 'node', + ref: 11580052710, + role: '', + }, + ], + tags: { + climbing: 'crag', + name: 'Yosemite (Hafty)', + site: 'climbing', + sport: 'climbing', + type: 'site', + wikimedia_commons: 'File:Roviště - Hafty.jpg', + 'wikimedia_commons:10': 'File:Roviště - Hafty10.jpg', + 'wikimedia_commons:2': 'File:Roviště - Hafty2.jpg', + 'wikimedia_commons:3': 'File:Roviště - Hafty3.jpg', + 'wikimedia_commons:4': 'File:Roviště - Hafty4.jpg', + 'wikimedia_commons:5': 'File:Roviště - Hafty5.jpg', + 'wikimedia_commons:6': 'File:Roviště - Hafty6.jpg', + 'wikimedia_commons:7': 'File:Roviště - Hafty7.jpg', + 'wikimedia_commons:8': 'File:Roviště - Hafty8.jpg', + 'wikimedia_commons:9': 'File:Roviště - Hafty9.jpg', + }, + }, + { + type: 'relation', + id: 17130099, + members: [ + { + type: 'relation', + ref: 17130663, + role: '', + }, + ], + tags: { + climbing: 'area', + description: + 'Roviště je klasická vltavská žula. Jedná se o velmi vyhlášenou oblast. Nabízí cesty prakticky všech obtížností, zpravidla dobře odjištěné.', + name: 'Roviště', + site: 'climbing', + type: 'site', + website: 'https://www.horosvaz.cz/skaly-sektor-289/', + 'website:2': 'https://www.lezec.cz/pruvodcx.php?key=5', + }, + }, + + // two nodes and a climbing=route way + { + type: 'node', + id: 1, + lat: 50, + lon: 14, + tags: {}, + }, + { + type: 'node', + id: 2, + lat: 51, + lon: 15, + tags: {}, + }, + { + type: 'way', + id: 3, + nodes: [1, 2], + tags: { + climbing: 'route', + name: 'Route of type way starting on 14,50', + }, + }, + + // two nodes and natural=cliff way ("crag") + { + type: 'node', + id: 4, + lat: 52, + lon: 16, + tags: {}, + }, + { + type: 'node', + id: 5, + lat: 53, + lon: 17, + tags: {}, + }, + { + type: 'way', + id: 6, + nodes: [4, 5], + tags: { + natural: 'cliff', + name: 'Cliff of type way at 16.5,52.5', + }, + }, +]; diff --git a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts index 6e401afbb..27eba1ae5 100644 --- a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts +++ b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts @@ -8,7 +8,6 @@ import { } from '../../../services/types'; import { join } from '../../../utils'; import { getCenter } from '../../../services/getCenter'; -import { getPoiClass } from '../../../services/getPoiClass'; type OsmType = 'node' | 'way' | 'relation'; type OsmNode = { @@ -40,7 +39,7 @@ type OsmResponse = { elements: OsmItem[]; }; -type Feature = { +export type GeojsonFeature = { type: 'Feature'; id: number; osmMeta: OsmId; @@ -58,9 +57,9 @@ type Feature = { }; type Lookup = { - node: Record>; - way: Record>; - relation: Record>; + node: Record>; + way: Record>; + relation: Record>; }; const convertOsmIdToMapId = (apiId: OsmId) => { @@ -93,7 +92,7 @@ const getLabel = (tags: FeatureTags, osmappRouteCount: number) => const convert = ( element: T, geometryFn: (element: T) => TGeometry, -): Feature => { +): GeojsonFeature => { const { type, id, tags = {} } = element; const geometry = geometryFn(element); const center = getCenter(geometry) ?? undefined; @@ -162,7 +161,7 @@ const getRelationGeomFn = }; const addToLookup = ( - items: Feature[], + items: GeojsonFeature[], lookup: Lookup, ) => { items.forEach((item) => { @@ -172,8 +171,8 @@ const addToLookup = ( }; const getRelationWithAreaCount = ( - relations: Feature[], - lookup: Record>, + relations: GeojsonFeature[], + lookup: Record>, ) => relations.map((relation) => { if (relation.tags?.climbing === 'area') { @@ -201,14 +200,6 @@ const getRelationWithAreaCount = ( return relation; }); -export const geometryToPoint = (feature: Feature) => ({ - ...feature, - geometry: { - type: 'Point', - coordinates: feature.center, - }, -}); - export const overpassToGeojsons = (response: OsmResponse) => { const { nodes, ways, relations } = getItems(response.elements); From df87089e3b71d4e7dcbdc09178de02c1f94d3880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 17:18:10 +0100 Subject: [PATCH 07/22] add `name` because of skeleton --- src/server/climbing-tiles/overpass/overpassToGeojsons.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts index 27eba1ae5..0a8f00fd5 100644 --- a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts +++ b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts @@ -107,6 +107,7 @@ const convert = ( : undefined; const properties = { climbing: tags?.climbing, + name: tags?.name, osmappType: type, osmappRouteCount, osmappLabel: getLabel(tags, osmappRouteCount), From 7455e7d45afa9a88c72b6be4152bbe46abc00766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 17:19:18 +0100 Subject: [PATCH 08/22] resolve route of type relation (via ferrata eg relation/10621005 ) --- .../Map/styles/layers/climbingLayers.ts | 5 ++- src/server/climbing-tiles/algo.ts | 39 ++++++++++++------- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/components/Map/styles/layers/climbingLayers.ts b/src/components/Map/styles/layers/climbingLayers.ts index a6475770d..4ef49fc5b 100644 --- a/src/components/Map/styles/layers/climbingLayers.ts +++ b/src/components/Map/styles/layers/climbingLayers.ts @@ -37,7 +37,7 @@ const linearByRouteCount = ( ): ExpressionSpecification => [ 'interpolate', ['linear'], - ['get', 'osmappRouteCount'], + ['coalesce', ['get', 'osmappRouteCount'], 0], from, a, to, @@ -75,8 +75,9 @@ const sortKey = [ -1, [ '+', - ['get', 'osmappRouteCount'], + ['coalesce', ['get', 'osmappRouteCount'], 0], ['case', ['get', 'osmappHasImages'], 99999, 0], // preference for items with images + ['case', ['get', 'name'], 2, 0], // prefer items with name ], ] as DataDrivenPropertyValueSpecification; diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts index 5bb2d74f3..841a6da17 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/algo.ts @@ -18,7 +18,7 @@ import { chunk } from 'lodash'; import { EditableData, TransactionOperation } from '@xata.io/client'; import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; -export const geometryToPoint = (feature: GeojsonFeature): GeojsonFeature => ({ +export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, geometry: { type: 'Point', @@ -91,22 +91,35 @@ const getNewRecords = async () => { lat: way.center[1], name: way.tags.name, count: way.properties.osmappRouteCount, - geojson: prepareGeojson(geometryToPoint(way)), + geojson: prepareGeojson(centerGeometry(way)), }); } } for (const relation of geojsons.relation) { - records.push({ - type: 'group', - osmType: 'relation', - osmId: relation.osmMeta.id, - lon: relation.center[0], - lat: relation.center[1], - name: relation.tags.name, - count: relation.properties.osmappRouteCount, - geojson: prepareGeojson(geometryToPoint(relation)), - }); + if (relation.tags.climbing === 'route') { + records.push({ + type: 'route', + osmType: 'relation', + osmId: relation.osmMeta.id, + lon: relation.center[0], // TODO maybe use first point in future + lat: relation.center[1], + name: relation.tags.name, + count: relation.properties.osmappRouteCount, + geojson: prepareGeojson(centerGeometry(relation)), + }); + } else { + records.push({ + type: 'group', + osmType: 'relation', + osmId: relation.osmMeta.id, + lon: relation.center[0], + lat: relation.center[1], + name: relation.tags.name, + count: relation.properties.osmappRouteCount, + geojson: prepareGeojson(centerGeometry(relation)), + }); + } } return records; @@ -133,7 +146,7 @@ export const refresh = async () => { return { createdRecords: records.length, durationMs: Math.round(performance.now() - start), // total 69 seconds - sizeKb: Math.round(JSON.stringify(records).length / 1000), + sizeBytes: JSON.stringify(records).length, }; }; From b29f048a74abbfebd207e4046f95f50cc1857102 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 17:53:10 +0100 Subject: [PATCH 09/22] comment --- src/server/climbing-tiles/algo.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/algo.ts index 841a6da17..5fc2a9474 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/algo.ts @@ -150,7 +150,7 @@ export const refresh = async () => { }; }; -// all 8 secs +// all 3 secs, 2MB export const climbingTile = async () => { const start = performance.now(); const alldata = await xata.db.climbing_tiles @@ -168,7 +168,7 @@ export const climbingTile = async () => { ...record.geojson, properties: { ...record.geojson.properties, - type: record.type, + type: record.type, // TODO add to main geojson? }, })); }; From 401f252cc49c4204b8fc577a5ee65ccfd4a0972b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Mon, 11 Nov 2024 18:03:49 +0100 Subject: [PATCH 10/22] fix test --- src/server/climbing-tiles/overpass/__tests__/basic.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/server/climbing-tiles/overpass/__tests__/basic.test.ts b/src/server/climbing-tiles/overpass/__tests__/basic.test.ts index fd2e63fa5..cd2221ccd 100644 --- a/src/server/climbing-tiles/overpass/__tests__/basic.test.ts +++ b/src/server/climbing-tiles/overpass/__tests__/basic.test.ts @@ -132,3 +132,5 @@ const elements: OsmItem[] = [ }, }, ]; + +test('noop', () => {}); From dbe039ff1aaa3bb6a737df4ecfbaedc39d019d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 12 Nov 2024 17:29:37 +0100 Subject: [PATCH 11/22] fetch from overpass --- pages/api/climbing-tiles/refresh.ts | 17 +++- pages/api/climbing-tiles/tile.ts | 2 +- src/server/climbing-tiles/climbing-tile.ts | 14 +++ .../overpass/overpassToGeojsons.ts | 2 +- .../climbing-tiles/{algo.ts => refresh.ts} | 93 ++++++++----------- 5 files changed, 69 insertions(+), 59 deletions(-) create mode 100644 src/server/climbing-tiles/climbing-tile.ts rename src/server/climbing-tiles/{algo.ts => refresh.ts} (63%) diff --git a/pages/api/climbing-tiles/refresh.ts b/pages/api/climbing-tiles/refresh.ts index ec0d66d53..e7e941825 100644 --- a/pages/api/climbing-tiles/refresh.ts +++ b/pages/api/climbing-tiles/refresh.ts @@ -1,11 +1,22 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { climbingTile, refresh } from '../../../src/server/climbing-tiles/algo'; +import { climbingTile } from '../../../src/server/climbing-tiles/climbing-tile'; +import { refresh } from '../../../src/server/climbing-tiles/refresh'; export default async (req: NextApiRequest, res: NextApiResponse) => { try { - const json = await refresh(); + res.writeHead(200, { + Connection: 'keep-alive', + 'Content-Type': 'text/plain; charset=utf-8', + 'Transfer-Encoding': 'chunked', + }); - res.status(200).json(json); + const writeCallback = (line: string) => { + res.write(line + '\n'); + }; + + await refresh(writeCallback); + + res.end(); } catch (err) { console.error(err); // eslint-disable-line no-console res.status(err.code ?? 400).send(String(err)); diff --git a/pages/api/climbing-tiles/tile.ts b/pages/api/climbing-tiles/tile.ts index 31c1bfdb8..aa7802033 100644 --- a/pages/api/climbing-tiles/tile.ts +++ b/pages/api/climbing-tiles/tile.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { climbingTile } from '../../../src/server/climbing-tiles/algo'; +import { climbingTile } from '../../../src/server/climbing-tiles/climbing-tile'; export default async (req: NextApiRequest, res: NextApiResponse) => { try { diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts new file mode 100644 index 000000000..40078800a --- /dev/null +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -0,0 +1,14 @@ +import { xata } from '../db/db'; + +export const climbingTile = async () => { + const start = performance.now(); + const alldata = await xata.db.climbing_tiles + .filter({ + type: 'group', + }) + .getAll(); + + console.log('climbingTile', performance.now() - start); + + return alldata.map((record) => record.geojson); +}; diff --git a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts index 0a8f00fd5..67d4a4a7e 100644 --- a/src/server/climbing-tiles/overpass/overpassToGeojsons.ts +++ b/src/server/climbing-tiles/overpass/overpassToGeojsons.ts @@ -35,7 +35,7 @@ type OsmRelation = { center?: { lat: number; lon: number }; // only for overpass `out center` queries }; type OsmItem = OsmNode | OsmWay | OsmRelation; -type OsmResponse = { +export type OsmResponse = { elements: OsmItem[]; }; diff --git a/src/server/climbing-tiles/algo.ts b/src/server/climbing-tiles/refresh.ts similarity index 63% rename from src/server/climbing-tiles/algo.ts rename to src/server/climbing-tiles/refresh.ts index 5fc2a9474..bd04cce9e 100644 --- a/src/server/climbing-tiles/algo.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -1,22 +1,13 @@ -// import {encodeUrl} from "@/utils"; -// import * as fs from "node:fs/promises"; -// const query = `[out:json][timeout:80];(nwr["climbing"];nwr["sport"="climbing"];);>>;out qt;`; -// const res = await fetch("https://overpass-api.de/api/interpreter", { -// "body": encodeUrl`data=${query}`, -// "method": "POST" -// }); -// const data = await res.json(); -// fs.writeFile("data.json", JSON.stringify(data, null, 2)); - -import { xata } from '../db/db'; import { GeojsonFeature, + OsmResponse, overpassToGeojsons, } from './overpass/overpassToGeojsons'; -import * as fs from 'fs/promises'; +import { xata } from '../db/db'; import { chunk } from 'lodash'; import { EditableData, TransactionOperation } from '@xata.io/client'; import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; +import { encodeUrl } from '../../helpers/utils'; export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, @@ -26,17 +17,29 @@ export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ }, }); -const prepareGeojson = ({ id, geometry, properties }: GeojsonFeature) => +const prepareGeojson = ( + type: string, + { id, geometry, properties }: GeojsonFeature, +) => JSON.stringify({ type: 'Feature', id, geometry, - properties, + properties: { ...properties, type }, }); -const getNewRecords = async () => { - const file = await fs.readFile('data3_42s_25mb.json', 'utf8'); - const data = JSON.parse(file); // 200 ms +const fetchFromOverpass = async (): Promise => { + // takes about 42 secs, 25MB + const query = `[out:json][timeout:80];(nwr["climbing"];nwr["sport"="climbing"];);>>;out qt;`; + const res = await fetch('https://overpass-api.de/api/interpreter', { + body: encodeUrl`data=${query}`, + method: 'POST', + }); + + return await res.json(); +}; + +const getNewRecords = async (data: OsmResponse) => { const geojsons = overpassToGeojsons(data); // 700 ms const records: Partial>[] = []; @@ -50,7 +53,7 @@ const getNewRecords = async () => { lon: node.geometry.coordinates[0], lat: node.geometry.coordinates[1], name: node.tags.name, - geojson: prepareGeojson(node), + geojson: prepareGeojson('route', node), }); } if (node.tags?.climbing === 'crag') { @@ -62,7 +65,7 @@ const getNewRecords = async () => { lat: node.geometry.coordinates[1], name: node.tags.name, count: node.properties.osmappRouteCount, - geojson: prepareGeojson(node), + geojson: prepareGeojson('group', node), }); } } @@ -77,7 +80,7 @@ const getNewRecords = async () => { lon: start[0], lat: start[1], name: way.tags.name, - geojson: prepareGeojson({ + geojson: prepareGeojson('route', { ...way, geometry: { type: 'Point', coordinates: start }, }), @@ -91,7 +94,7 @@ const getNewRecords = async () => { lat: way.center[1], name: way.tags.name, count: way.properties.osmappRouteCount, - geojson: prepareGeojson(centerGeometry(way)), + geojson: prepareGeojson('group', centerGeometry(way)), }); } } @@ -106,7 +109,7 @@ const getNewRecords = async () => { lat: relation.center[1], name: relation.tags.name, count: relation.properties.osmappRouteCount, - geojson: prepareGeojson(centerGeometry(relation)), + geojson: prepareGeojson('route', centerGeometry(relation)), }); } else { records.push({ @@ -117,7 +120,7 @@ const getNewRecords = async () => { lat: relation.center[1], name: relation.tags.name, count: relation.properties.osmappRouteCount, - geojson: prepareGeojson(centerGeometry(relation)), + geojson: prepareGeojson('group', centerGeometry(relation)), }); } } @@ -125,15 +128,20 @@ const getNewRecords = async () => { return records; }; -export const refresh = async () => { +export const refresh = async (writeCallback: (line: string) => void) => { const start = performance.now(); + writeCallback('Deleting old records...'); await xata.sql`DELETE FROM climbing_tiles`; - const records = await getNewRecords(); // ~ 16k records + writeCallback('Fetching data from Overpass...'); + const data = await fetchFromOverpass(); + const records = await getNewRecords(data); // ~ 16k records + + writeCallback(`Inserting new records (${records.length})...`); const chunks = chunk(records, 1000); for (const chunk of chunks) { await xata.transactions.run( - // avg 700kb per 1000 records, takes ~5 secs + // avg takes ~5 secs chunk.map( (record) => ({ @@ -141,34 +149,11 @@ export const refresh = async () => { }) as TransactionOperation, ), ); + writeCallback(`Inserted ${chunk.length} records.`); } - return { - createdRecords: records.length, - durationMs: Math.round(performance.now() - start), // total 69 seconds - sizeBytes: JSON.stringify(records).length, - }; -}; - -// all 3 secs, 2MB -export const climbingTile = async () => { - const start = performance.now(); - const alldata = await xata.db.climbing_tiles - .filter({ - type: 'group', - }) - .getAll(); - // .getMany({ - // pagination: { size: 1000, offset: 0 }, - // }); - - console.log('climbingTile', performance.now() - start); - - return alldata.map((record) => ({ - ...record.geojson, - properties: { - ...record.geojson.properties, - type: record.type, // TODO add to main geojson? - }, - })); + writeCallback('Done.'); + writeCallback(`Created records: ${records.length}`); + writeCallback(`Duration: ${Math.round(performance.now() - start)} ms`); + writeCallback(`Records size: ${JSON.stringify(records).length} bytes`); }; From a8b653049406095a532feca9462efa1b63b963fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 12 Nov 2024 17:48:28 +0100 Subject: [PATCH 12/22] use fetchJson --- pages/api/climbing-tiles/refresh.ts | 1 + src/server/climbing-tiles/refresh.ts | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pages/api/climbing-tiles/refresh.ts b/pages/api/climbing-tiles/refresh.ts index e7e941825..8193ed63e 100644 --- a/pages/api/climbing-tiles/refresh.ts +++ b/pages/api/climbing-tiles/refresh.ts @@ -11,6 +11,7 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { }); const writeCallback = (line: string) => { + console.log(line); // eslint-disable-line no-console res.write(line + '\n'); }; diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index bd04cce9e..addba0e7e 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -8,6 +8,7 @@ import { chunk } from 'lodash'; import { EditableData, TransactionOperation } from '@xata.io/client'; import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; import { encodeUrl } from '../../helpers/utils'; +import { fetchJson } from '../../services/fetch'; export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, @@ -28,15 +29,25 @@ const prepareGeojson = ( properties: { ...properties, type }, }); -const fetchFromOverpass = async (): Promise => { +const fetchFromOverpass = async () => { // takes about 42 secs, 25MB const query = `[out:json][timeout:80];(nwr["climbing"];nwr["sport"="climbing"];);>>;out qt;`; - const res = await fetch('https://overpass-api.de/api/interpreter', { - body: encodeUrl`data=${query}`, - method: 'POST', - }); + const data = await fetchJson( + 'https://overpass-api.de/api/interpreter', + { + body: encodeUrl`data=${query}`, + method: 'POST', + nocache: true, + }, + ); + + if (data.elements.length < 1000) { + throw new Error( + `Overpass returned too few elements. Data:${JSON.stringify(data).substring(0, 200)}`, + ); + } - return await res.json(); + return data; }; const getNewRecords = async (data: OsmResponse) => { From 7f3d9b3547635ca9abfe589b136b5cea0eb91247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Thu, 14 Nov 2024 17:00:13 +0100 Subject: [PATCH 13/22] xata-postgre + more cases in refresh --- .xata/migrations/.ledger | 5 +- .../mig_cso99kp32vjpfm25jl70_e4b0e013.json | 92 --------- .xatarc | 2 +- package.json | 2 +- .../Map/styles/layers/climbingLayers.ts | 33 +-- src/server/climbing-tiles/climbing-tile.ts | 4 +- src/server/climbing-tiles/refresh.ts | 194 +++++++++++------- src/server/db/xata-generated.ts | 105 +++++++--- yarn.lock | 8 +- 9 files changed, 227 insertions(+), 218 deletions(-) delete mode 100644 .xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json diff --git a/.xata/migrations/.ledger b/.xata/migrations/.ledger index 3f0cc2200..81fe7ced3 100644 --- a/.xata/migrations/.ledger +++ b/.xata/migrations/.ledger @@ -1,3 +1,2 @@ -mig_cso99kp32vjpfm25jl70_e4b0e013 -mig_csp0306qn9e0obu1aakg_1515dd60 -mig_csp2358b12tq4cm03vtg_67247356 +mig_csqvnk4icce8859die00 +mig_csqvvssicce8859die5g diff --git a/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json b/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json deleted file mode 100644 index a55dc1e38..000000000 --- a/.xata/migrations/mig_cso99kp32vjpfm25jl70_e4b0e013.json +++ /dev/null @@ -1,92 +0,0 @@ -{ - "id": "mig_cso99kp32vjpfm25jl70", - "checksum": "1:e4b0e01303c68771fd0139e5dc9ad466c630d7ab59ad428b8fc570348284df40", - "operations": [ - { - "addTable": { - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "type", - "type": "text" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "lon", - "type": "float" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "lat", - "type": "float" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "osmType", - "type": "text" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "osmId", - "type": "int" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "osmVersion", - "type": "int" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "name", - "type": "text" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "count", - "type": "int" - }, - "table": "climbing_tiles" - } - }, - { - "addColumn": { - "column": { - "name": "json", - "type": "json" - }, - "table": "climbing_tiles" - } - } - ] -} diff --git a/.xatarc b/.xatarc index 48f846e4e..0d565b7d9 100644 --- a/.xatarc +++ b/.xatarc @@ -1,5 +1,5 @@ { - "databaseURL": "https://osmapp-tvgiad.us-east-1.xata.sh/db/osmapp", + "databaseURL": "https://osmapp-tvgiad.us-east-1.xata.sh/db/db_with_direct_access", "codegen": { "output": "src/server/db/xata-generated.ts" } diff --git a/package.json b/package.json index 14293ed3a..9bdfa77a8 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@openstreetmap/id-tagging-schema": "^6.8.1", "@sentry/nextjs": "^8.34.0", "@teritorio/openmaptiles-gl-language": "^1.5.4", - "@xata.io/client": "^0.30.1", + "@xata.io/client": "^0.0.0-next.va121e4207b94bfe0a3c025fc00b247b923880930", "@xmldom/xmldom": "^0.9.3", "accept-language-parser": "^1.5.0", "autosuggest-highlight": "^3.3.4", diff --git a/src/components/Map/styles/layers/climbingLayers.ts b/src/components/Map/styles/layers/climbingLayers.ts index 4ef49fc5b..ae04024d3 100644 --- a/src/components/Map/styles/layers/climbingLayers.ts +++ b/src/components/Map/styles/layers/climbingLayers.ts @@ -78,6 +78,12 @@ const sortKey = [ ['coalesce', ['get', 'osmappRouteCount'], 0], ['case', ['get', 'osmappHasImages'], 99999, 0], // preference for items with images ['case', ['get', 'name'], 2, 0], // prefer items with name + // [ + // 'case', + // ['all', ['>', ['zoom'], 16], ['==', ['get', 'climbing'], 'crag']], + // 999, + // 0, + // ], // prefer areas on low zoom ], ] as DataDrivenPropertyValueSpecification; @@ -100,16 +106,20 @@ const step = ( ]; export const routes: LayerSpecification[] = [ + { + id: 'climbing-3-routes-line', + type: 'line', + source: 'climbing', + minzoom: 16, + filter: ['all', ['==', 'type', 'route']], + }, + { id: 'climbing-3-routes-circle', type: 'circle', source: 'climbing', minzoom: 16, - filter: [ - 'all', - ['==', 'osmappType', 'node'], - ['==', 'climbing', 'route_bottom'], - ], + filter: ['all', ['==', 'type', 'route']], paint: { 'circle-color': [ 'case', @@ -126,11 +136,7 @@ export const routes: LayerSpecification[] = [ type: 'symbol', source: 'climbing', minzoom: 19, - filter: [ - 'all', - ['==', 'osmappType', 'node'], - ['==', 'climbing', 'route_bottom'], - ], + filter: ['all', ['==', 'type', 'route']], layout: { 'text-padding': 2, 'text-font': ['Noto Sans Medium'], @@ -180,12 +186,7 @@ const mixed: LayerSpecification = { type: 'symbol', source: 'climbing', maxzoom: 20, - filter: [ - 'all', - ['==', 'type', 'group'], - // ['==', 'osmappType', 'relationPoint'], - // ['in', 'climbing', 'area', 'crag'], - ], + filter: ['all', ['==', 'type', 'group']], layout: { 'icon-image': ifCrag( byHasImages(CRAG, 'IMAGE'), diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts index 40078800a..e48f1b6b0 100644 --- a/src/server/climbing-tiles/climbing-tile.ts +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -4,7 +4,9 @@ export const climbingTile = async () => { const start = performance.now(); const alldata = await xata.db.climbing_tiles .filter({ - type: 'group', + // type: '_otherWays', + lat: { $gt: 48, $lt: 51 }, + lon: { $gt: 14, $lt: 19 }, }) .getAll(); diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index addba0e7e..0f4b9c45f 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -9,8 +9,10 @@ import { EditableData, TransactionOperation } from '@xata.io/client'; import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; import { encodeUrl } from '../../helpers/utils'; import { fetchJson } from '../../services/fetch'; +import { LineString, LonLat, Point } from '../../services/types'; +import * as fs from 'node:fs'; -export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ +const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, geometry: { type: 'Point', @@ -18,6 +20,16 @@ export const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ }, }); +const firstPointGeometry = ( + feature: GeojsonFeature, +): GeojsonFeature => ({ + ...feature, + geometry: { + type: 'Point', + coordinates: feature.geometry.coordinates[0], + }, +}); + const prepareGeojson = ( type: string, { id, geometry, properties }: GeojsonFeature, @@ -50,90 +62,119 @@ const fetchFromOverpass = async () => { return data; }; -const getNewRecords = async (data: OsmResponse) => { - const geojsons = overpassToGeojsons(data); // 700 ms +type Records = Partial>[]; - const records: Partial>[] = []; +const recordsFactory = () => { + const records: Records = []; + const addRecordRaw = ( + type: string, + coordinates: LonLat, + feature: GeojsonFeature, + ) => + records.push({ + type, + osmType: feature.osmMeta.type, + osmId: feature.osmMeta.id, + name: feature.tags.name, + count: feature.properties.osmappRouteCount, + lon: coordinates[0], + lat: coordinates[1], + geojson: prepareGeojson(type, feature), + }); + + const addRecord = (type: string, feature: GeojsonFeature) => { + addRecordRaw(type, feature.geometry.coordinates, feature); + }; + + const addRecordWithLine = (type: string, way: GeojsonFeature) => { + addRecord(type, firstPointGeometry(way)); + addRecordRaw(type, way.center, way); + }; + + return { records, addRecord, addRecordWithLine }; +}; + +const getNewRecords = async (data: OsmResponse) => { + const geojsons = overpassToGeojsons(data); // 700 ms on 16k items + const { records, addRecord, addRecordWithLine } = recordsFactory(); for (const node of geojsons.node) { - if (node.tags?.climbing === 'route_bottom') { - records.push({ - type: 'route', - osmType: 'node', - osmId: node.osmMeta.id, - lon: node.geometry.coordinates[0], - lat: node.geometry.coordinates[1], - name: node.tags.name, - geojson: prepareGeojson('route', node), - }); + if (!node.tags) continue; + if ( + node.tags.climbing === 'area' || + node.tags.climbing === 'boulder' || + node.tags.climbing === 'crag' || + node.tags.natural === 'peak' + ) { + addRecord('group', node); } - if (node.tags?.climbing === 'crag') { - records.push({ - type: 'group', - osmType: 'node', - osmId: node.osmMeta.id, - lon: node.geometry.coordinates[0], - lat: node.geometry.coordinates[1], - name: node.tags.name, - count: node.properties.osmappRouteCount, - geojson: prepareGeojson('group', node), - }); + + // + else if ( + node.tags.climbing === 'route' || + node.tags.climbing === 'route_bottom' + ) { + addRecord('route', node); + } + + // + else if (node.tags.climbing === 'route_top') { + // later + update climbingLayer + } + + // 120 k nodes ??? + else { + //addRecord('_otherNodes', node); } } for (const way of geojsons.way) { - if (way.tags.climbing === 'route') { - const start = way.geometry.coordinates[0]; - records.push({ - type: 'route', - osmType: 'way', - osmId: way.osmMeta.id, - lon: start[0], - lat: start[1], - name: way.tags.name, - geojson: prepareGeojson('route', { - ...way, - geometry: { type: 'Point', coordinates: start }, - }), - }); - } else { - records.push({ - type: 'group', - osmType: 'way', - osmId: way.osmMeta.id, - lon: way.center[0], - lat: way.center[1], - name: way.tags.name, - count: way.properties.osmappRouteCount, - geojson: prepareGeojson('group', centerGeometry(way)), - }); + // climbing=route -> route + line + // highway=via_ferrata -> route + line + if (way.tags.climbing === 'route' || way.tags.highway === 'via_ferrata') { + addRecordWithLine('route', way); + } + + // natural=cliff + sport=climbing -> group + // natural=rock + sport=climbing -> group + else if ( + way.tags.sport === 'climbing' && + (way.tags.natural === 'cliff' || way.tags.natural === 'rock') + ) { + addRecord('group', centerGeometry(way)); + } + + // _otherWays to debug + else { + addRecord('_otherWays', centerGeometry(way)); + // TODO way/167416816 is natural=cliff with parent relation type=site } } for (const relation of geojsons.relation) { - if (relation.tags.climbing === 'route') { - records.push({ - type: 'route', - osmType: 'relation', - osmId: relation.osmMeta.id, - lon: relation.center[0], // TODO maybe use first point in future - lat: relation.center[1], - name: relation.tags.name, - count: relation.properties.osmappRouteCount, - geojson: prepareGeojson('route', centerGeometry(relation)), - }); - } else { - records.push({ - type: 'group', - osmType: 'relation', - osmId: relation.osmMeta.id, - lon: relation.center[0], - lat: relation.center[1], - name: relation.tags.name, - count: relation.properties.osmappRouteCount, - geojson: prepareGeojson('group', centerGeometry(relation)), - }); + // climbing=area -> group + // climbing=boulder -> group + // climbing=crag -> group + // climbing=route -> group // multipitch or via_ferrata + // type=site -> group + // type=multipolygon -> group + delete nodes + if ( + relation.tags.climbing === 'area' || + relation.tags.type === 'boulder' || + relation.tags.type === 'crag' || + relation.tags.climbing === 'route' || + relation.tags.type === 'site' || + relation.tags.type === 'multipolygon' + ) { + addRecord('group', centerGeometry(relation)); + } + + // _otherRelations to debug + else { + addRecord('group', centerGeometry(relation)); } + + // TODO no center -> write to log } return records; @@ -145,7 +186,14 @@ export const refresh = async (writeCallback: (line: string) => void) => { await xata.sql`DELETE FROM climbing_tiles`; writeCallback('Fetching data from Overpass...'); - const data = await fetchFromOverpass(); + //const data = await fetchFromOverpass(); + //fs.writeFileSync('../overpass.json', JSON.stringify(data)); + const data = JSON.parse( + fs.readFileSync('../overpass.json').toString(), + ) as OsmResponse; + + console.log('Data:', data.elements.length); + const records = await getNewRecords(data); // ~ 16k records writeCallback(`Inserting new records (${records.length})...`); diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts index f513debaf..0f6bd6ec3 100644 --- a/src/server/db/xata-generated.ts +++ b/src/server/db/xata-generated.ts @@ -7,31 +7,85 @@ import type { } from '@xata.io/client'; const tables = [ - { - name: 'climbing_tiles-oldsunday', - columns: [ - { name: 'type', type: 'text' }, - { name: 'lon', type: 'float' }, - { name: 'lat', type: 'float' }, - { name: 'osmType', type: 'text' }, - { name: 'osmId', type: 'int' }, - { name: 'osmVersion', type: 'int' }, - { name: 'name', type: 'text' }, - { name: 'count', type: 'int' }, - { name: 'json', type: 'json' }, - ], - }, { name: 'climbing_tiles', + checkConstraints: {}, + foreignKeys: {}, + primaryKey: ['xata_id'], + uniqueConstraints: {}, columns: [ - { name: 'type', type: 'text' }, - { name: 'lon', type: 'float' }, - { name: 'lat', type: 'float' }, - { name: 'osmType', type: 'text' }, - { name: 'osmId', type: 'int' }, - { name: 'name', type: 'text' }, - { name: 'count', type: 'int' }, - { name: 'geojson', type: 'json' }, + { + name: 'count', + type: 'int', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'geojson', + type: 'json', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'lat', + type: 'float', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'lon', + type: 'float', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'name', + type: 'text', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'osmId', + type: 'int', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'osmType', + type: 'text', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'type', + type: 'text', + notNull: true, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'xata_id', + type: 'int', + notNull: true, + unique: true, + defaultValue: null, + comment: '', + }, ], }, ] as const; @@ -39,21 +93,18 @@ const tables = [ export type SchemaTables = typeof tables; export type InferredTypes = SchemaInference; -export type ClimbingTilesOldsunday = InferredTypes['climbing_tiles-oldsunday']; -export type ClimbingTilesOldsundayRecord = ClimbingTilesOldsunday & XataRecord; - export type ClimbingTiles = InferredTypes['climbing_tiles']; export type ClimbingTilesRecord = ClimbingTiles & XataRecord; export type DatabaseSchema = { - 'climbing_tiles-oldsunday': ClimbingTilesOldsundayRecord; climbing_tiles: ClimbingTilesRecord; }; const DatabaseClient = buildClient(); const defaultOptions = { - databaseURL: 'https://osmapp-tvgiad.us-east-1.xata.sh/db/osmapp', + databaseURL: + 'https://osmapp-tvgiad.us-east-1.xata.sh/db/db_with_direct_access', }; export class XataClient extends DatabaseClient { diff --git a/yarn.lock b/yarn.lock index 8f6ddf620..7e4b2077b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2276,10 +2276,10 @@ resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@xata.io/client@^0.30.1": - version "0.30.1" - resolved "https://registry.yarnpkg.com/@xata.io/client/-/client-0.30.1.tgz#a7fe9729d7241f903d94d3b16c293816bea7d04b" - integrity sha512-dAzDPHmIfenVIpF39m1elmW5ngjWu2mO8ZqJBN7dmYdXr98uhPANfLdVZnc3mUNG+NH37LqY1dSO862hIo2oRw== +"@xata.io/client@^0.0.0-next.va121e4207b94bfe0a3c025fc00b247b923880930": + version "0.0.0-next.va121e4207b94bfe0a3c025fc00b247b923880930" + resolved "https://registry.yarnpkg.com/@xata.io/client/-/client-0.0.0-next.va121e4207b94bfe0a3c025fc00b247b923880930.tgz#c3feb0f27a11c1418540fe4c0f219ecf6804e77d" + integrity sha512-tRhsrV6vSJk4r+Cn7WkE1hyV2WfnHOoMzpQeLujqrbemPtVj82R1uhqHM2fPFuUKMfD+19PVF+Gist0ePlX5SQ== "@xmldom/xmldom@^0.9.3": version "0.9.3" From f75a8a4852d1da0766bf4e4b515b72be1018bff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Fri, 15 Nov 2024 09:16:16 +0100 Subject: [PATCH 14/22] change to pg lib + other db --- package.json | 1 + src/server/climbing-tiles/refresh.ts | 34 +++++++++++++++++++++++----- src/server/db/xata-generated.ts | 2 ++ 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 9bdfa77a8..6be6ffd49 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "open-location-code": "^1.0.3", "opening_hours": "^3.8.0", "osm-auth": "^2.5.0", + "pg": "^8.13.1", "react": "^18.3.1", "react-custom-scrollbars": "^4.2.1", "react-dom": "^18.3.1", diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index 0f4b9c45f..fff72e400 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -3,14 +3,13 @@ import { OsmResponse, overpassToGeojsons, } from './overpass/overpassToGeojsons'; -import { xata } from '../db/db'; -import { chunk } from 'lodash'; -import { EditableData, TransactionOperation } from '@xata.io/client'; -import { ClimbingTilesRecord, DatabaseSchema } from '../db/xata-generated'; +import { EditableData } from '@xata.io/client'; +import { ClimbingTilesRecord } from '../db/xata-generated'; import { encodeUrl } from '../../helpers/utils'; import { fetchJson } from '../../services/fetch'; import { LineString, LonLat, Point } from '../../services/types'; import * as fs from 'node:fs'; +import { Client } from 'pg'; const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, @@ -76,7 +75,7 @@ const recordsFactory = () => { osmType: feature.osmMeta.type, osmId: feature.osmMeta.id, name: feature.tags.name, - count: feature.properties.osmappRouteCount, + count: feature.properties.osmappRouteCount || 0, lon: coordinates[0], lat: coordinates[1], geojson: prepareGeojson(type, feature), @@ -181,9 +180,22 @@ const getNewRecords = async (data: OsmResponse) => { }; export const refresh = async (writeCallback: (line: string) => void) => { + const client = new Client({ + user: 'tvgiad', + password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', + host: 'us-east-1.sql.xata.sh', + port: 5432, + database: 'db_with_direct_access:main', + ssl: { + rejectUnauthorized: false, + }, + }); + + await client.connect(); + const start = performance.now(); writeCallback('Deleting old records...'); - await xata.sql`DELETE FROM climbing_tiles`; + await client.query('DELETE FROM climbing_tiles'); writeCallback('Fetching data from Overpass...'); //const data = await fetchFromOverpass(); @@ -197,6 +209,14 @@ export const refresh = async (writeCallback: (line: string) => void) => { const records = await getNewRecords(data); // ~ 16k records writeCallback(`Inserting new records (${records.length})...`); + + for (const record of records) { + await client.query({ + text: 'INSERT INTO climbing_tiles(type, "osmType", "osmId", name, count, lon, lat, geojson) VALUES($1, $2, $3, $4, $5, $6, $7, $8)', + values: Object.values(record), + }); + } + const chunks = chunk(records, 1000); for (const chunk of chunks) { await xata.transactions.run( @@ -211,6 +231,8 @@ export const refresh = async (writeCallback: (line: string) => void) => { writeCallback(`Inserted ${chunk.length} records.`); } + await client.end(); + writeCallback('Done.'); writeCallback(`Created records: ${records.length}`); writeCallback(`Duration: ${Math.round(performance.now() - start)} ms`); diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts index 0f6bd6ec3..000172e69 100644 --- a/src/server/db/xata-generated.ts +++ b/src/server/db/xata-generated.ts @@ -105,6 +105,8 @@ const DatabaseClient = buildClient(); const defaultOptions = { databaseURL: 'https://osmapp-tvgiad.us-east-1.xata.sh/db/db_with_direct_access', + apiKey: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', + branch: 'main', }; export class XataClient extends DatabaseClient { From f9d7dc146719d63a0fc2b0d08cec7b9f82b71d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 26 Nov 2024 08:54:36 +0100 Subject: [PATCH 15/22] add vercel.json --- vercel.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 vercel.json diff --git a/vercel.json b/vercel.json new file mode 100644 index 000000000..83d1ecde7 --- /dev/null +++ b/vercel.json @@ -0,0 +1,8 @@ +{ + "functions": { + "pages/api/climbing-tiles/refresh.ts": { + "memory": 3009, + "maxDuration": 300 + } + } +} From 64e72d5bfb93bc4c130e83320ca6ae830c8c562a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 26 Nov 2024 09:35:12 +0100 Subject: [PATCH 16/22] use pg-format to format query --- package.json | 1 + src/server/climbing-tiles/refresh.ts | 56 +++++++++++----------------- 2 files changed, 22 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index 6be6ffd49..78fda2f5e 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "opening_hours": "^3.8.0", "osm-auth": "^2.5.0", "pg": "^8.13.1", + "pg-format": "^1.0.4", "react": "^18.3.1", "react-custom-scrollbars": "^4.2.1", "react-dom": "^18.3.1", diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index fff72e400..a050d5764 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -10,6 +10,7 @@ import { fetchJson } from '../../services/fetch'; import { LineString, LonLat, Point } from '../../services/types'; import * as fs from 'node:fs'; import { Client } from 'pg'; +import format from 'pg-format'; const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, @@ -93,7 +94,7 @@ const recordsFactory = () => { return { records, addRecord, addRecordWithLine }; }; -const getNewRecords = async (data: OsmResponse) => { +const getNewRecords = (data: OsmResponse) => { const geojsons = overpassToGeojsons(data); // 700 ms on 16k items const { records, addRecord, addRecordWithLine } = recordsFactory(); @@ -179,7 +180,7 @@ const getNewRecords = async (data: OsmResponse) => { return records; }; -export const refresh = async (writeCallback: (line: string) => void) => { +export const refresh = async (log: (line: string) => void) => { const client = new Client({ user: 'tvgiad', password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', @@ -194,47 +195,32 @@ export const refresh = async (writeCallback: (line: string) => void) => { await client.connect(); const start = performance.now(); - writeCallback('Deleting old records...'); + log('Deleting old records...'); await client.query('DELETE FROM climbing_tiles'); - writeCallback('Fetching data from Overpass...'); - //const data = await fetchFromOverpass(); - //fs.writeFileSync('../overpass.json', JSON.stringify(data)); + log('Fetching data from Overpass...'); + // const data = await fetchFromOverpass(); + // fs.writeFileSync('../overpass.json', JSON.stringify(data)); const data = JSON.parse( fs.readFileSync('../overpass.json').toString(), ) as OsmResponse; + log(`Overpass elements: ${data.elements.length}`); - console.log('Data:', data.elements.length); + const records = getNewRecords(data); // ~ 16k records + log(`Records: (${records.length})`); - const records = await getNewRecords(data); // ~ 16k records - - writeCallback(`Inserting new records (${records.length})...`); - - for (const record of records) { - await client.query({ - text: 'INSERT INTO climbing_tiles(type, "osmType", "osmId", name, count, lon, lat, geojson) VALUES($1, $2, $3, $4, $5, $6, $7, $8)', - values: Object.values(record), - }); - } - - const chunks = chunk(records, 1000); - for (const chunk of chunks) { - await xata.transactions.run( - // avg takes ~5 secs - chunk.map( - (record) => - ({ - insert: { table: 'climbing_tiles', record }, - }) as TransactionOperation, - ), - ); - writeCallback(`Inserted ${chunk.length} records.`); - } + const columns = Object.keys(records[0]); + const values = records.map((record) => Object.values(record)); + const query = format( + `INSERT INTO climbing_tiles(%I) VALUES %L`, + columns, + values, + ); + log(`SQL Query length: ${query.length} chars`); + await client.query(query); await client.end(); - writeCallback('Done.'); - writeCallback(`Created records: ${records.length}`); - writeCallback(`Duration: ${Math.round(performance.now() - start)} ms`); - writeCallback(`Records size: ${JSON.stringify(records).length} bytes`); + log('Done.'); + log(`Duration: ${Math.round(performance.now() - start)} ms`); }; From bef7e5af0a51efec378a404c160f270b434022d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 26 Nov 2024 16:41:00 +0100 Subject: [PATCH 17/22] geohashing + pg --- .xata/migrations/.ledger | 4 + package.json | 1 + pages/api/climbing-tiles/tile.ts | 7 +- src/server/climbing-tiles/climbing-tile.ts | 35 ++++++- src/server/climbing-tiles/refresh.ts | 15 ++- src/server/db/xata-generated.ts | 109 ++++++++++++++++++++- yarn.lock | 5 + 7 files changed, 161 insertions(+), 15 deletions(-) diff --git a/.xata/migrations/.ledger b/.xata/migrations/.ledger index 81fe7ced3..a54052c88 100644 --- a/.xata/migrations/.ledger +++ b/.xata/migrations/.ledger @@ -1,2 +1,6 @@ mig_csqvnk4icce8859die00 mig_csqvvssicce8859die5g +mig_ct2scmdc21vap51ejf8g +sql_28451505e1dd5d +sql_c642d1b1707c7a +sql_8f7a225a10aa7d diff --git a/package.json b/package.json index 78fda2f5e..8614819f2 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "canvg": "^4.0.2", "date-fns": "^4.1.0", "dice-coefficient": "^2.1.1", + "geohashing": "^2.0.1", "image-size": "^1.1.1", "isomorphic-unfetch": "^4.0.2", "isomorphic-xml2js": "^0.1.3", diff --git a/pages/api/climbing-tiles/tile.ts b/pages/api/climbing-tiles/tile.ts index aa7802033..744f2bed4 100644 --- a/pages/api/climbing-tiles/tile.ts +++ b/pages/api/climbing-tiles/tile.ts @@ -1,9 +1,12 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { climbingTile } from '../../../src/server/climbing-tiles/climbing-tile'; +import { + climbingTile, + climbingTilePg, +} from '../../../src/server/climbing-tiles/climbing-tile'; export default async (req: NextApiRequest, res: NextApiResponse) => { try { - const json = await climbingTile(); + const json = req.query.pg ? await climbingTilePg() : await climbingTile(); res.status(200).json(json); } catch (err) { diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts index e48f1b6b0..2f778b705 100644 --- a/src/server/climbing-tiles/climbing-tile.ts +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -1,16 +1,45 @@ import { xata } from '../db/db'; +import { Client } from 'pg'; export const climbingTile = async () => { const start = performance.now(); const alldata = await xata.db.climbing_tiles + .select(['geojson']) .filter({ // type: '_otherWays', - lat: { $gt: 48, $lt: 51 }, - lon: { $gt: 14, $lt: 19 }, + // lat: { $gt: 48, $lt: 51 }, + // lon: { $gt: 14, $lt: 19 }, + type: 'group', + geohash: 'u2', }) .getAll(); - console.log('climbingTile', performance.now() - start); + console.log('climbingTileXata', performance.now() - start, alldata.length); return alldata.map((record) => record.geojson); }; + +export const climbingTilePg = async () => { + const start = performance.now(); + + const client = new Client({ + user: 'tvgiad', + password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', + host: 'us-east-1.sql.xata.sh', + port: 5432, + database: 'db_with_direct_access:main', + ssl: { + rejectUnauthorized: false, + }, + }); + + await client.connect(); + + const result = await client.query( + "SELECT geojson FROM climbing_tiles WHERE type='group' AND geohash = 'u2'", + ); + + console.log('climbingTilePg', performance.now() - start, result.rows.length); + + return result.rows.map((record) => record.geojson); +}; diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index a050d5764..3a5d6b3da 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -11,6 +11,7 @@ import { LineString, LonLat, Point } from '../../services/types'; import * as fs from 'node:fs'; import { Client } from 'pg'; import format from 'pg-format'; +import { encodeBase32, decodeBase32 } from 'geohashing'; const centerGeometry = (feature: GeojsonFeature): GeojsonFeature => ({ ...feature, @@ -70,17 +71,21 @@ const recordsFactory = () => { type: string, coordinates: LonLat, feature: GeojsonFeature, - ) => - records.push({ + ) => { + const lon = coordinates[0]; + const lat = coordinates[1]; + return records.push({ type, osmType: feature.osmMeta.type, osmId: feature.osmMeta.id, name: feature.tags.name, count: feature.properties.osmappRouteCount || 0, - lon: coordinates[0], - lat: coordinates[1], + lon, + lat, + geohash: encodeBase32(lat, lon, 2), geojson: prepareGeojson(type, feature), }); + }; const addRecord = (type: string, feature: GeojsonFeature) => { addRecordRaw(type, feature.geometry.coordinates, feature); @@ -207,7 +212,7 @@ export const refresh = async (log: (line: string) => void) => { log(`Overpass elements: ${data.elements.length}`); const records = getNewRecords(data); // ~ 16k records - log(`Records: (${records.length})`); + log(`Records: ${records.length}`); const columns = Object.keys(records[0]); const values = records.map((record) => Object.values(record)); diff --git a/src/server/db/xata-generated.ts b/src/server/db/xata-generated.ts index 000172e69..d97d49ce2 100644 --- a/src/server/db/xata-generated.ts +++ b/src/server/db/xata-generated.ts @@ -9,14 +9,33 @@ import type { const tables = [ { name: 'climbing_tiles', - checkConstraints: {}, + checkConstraints: { + climbing_tiles_xata_id_length_xata_id: { + name: 'climbing_tiles_xata_id_length_xata_id', + columns: ['xata_id'], + definition: 'CHECK ((length(xata_id) < 256))', + }, + }, foreignKeys: {}, - primaryKey: ['xata_id'], - uniqueConstraints: {}, + primaryKey: [], + uniqueConstraints: { + _pgroll_new_climbing_tiles_xata_id_key: { + name: '_pgroll_new_climbing_tiles_xata_id_key', + columns: ['xata_id'], + }, + }, columns: [ { name: 'count', type: 'int', + notNull: false, + unique: false, + defaultValue: null, + comment: '', + }, + { + name: 'geohash', + type: 'text', notNull: true, unique: false, defaultValue: null, @@ -49,7 +68,7 @@ const tables = [ { name: 'name', type: 'text', - notNull: true, + notNull: false, unique: false, defaultValue: null, comment: '', @@ -78,12 +97,88 @@ const tables = [ defaultValue: null, comment: '', }, + { + name: 'xata_createdat', + type: 'datetime', + notNull: true, + unique: false, + defaultValue: 'now()', + comment: '', + }, { name: 'xata_id', + type: 'text', + notNull: true, + unique: true, + defaultValue: "('rec_'::text || (xata_private.xid())::text)", + comment: '', + }, + { + name: 'xata_updatedat', + type: 'datetime', + notNull: true, + unique: false, + defaultValue: 'now()', + comment: '', + }, + { + name: 'xata_version', type: 'int', notNull: true, + unique: false, + defaultValue: '0', + comment: '', + }, + ], + }, + { + name: 'test_xata_cols', + checkConstraints: { + test_xata_cols_xata_id_length_xata_id: { + name: 'test_xata_cols_xata_id_length_xata_id', + columns: ['xata_id'], + definition: 'CHECK ((length(xata_id) < 256))', + }, + }, + foreignKeys: {}, + primaryKey: [], + uniqueConstraints: { + _pgroll_new_test_xata_cols_xata_id_key: { + name: '_pgroll_new_test_xata_cols_xata_id_key', + columns: ['xata_id'], + }, + }, + columns: [ + { + name: 'xata_createdat', + type: 'datetime', + notNull: true, + unique: false, + defaultValue: 'now()', + comment: '', + }, + { + name: 'xata_id', + type: 'text', + notNull: true, unique: true, - defaultValue: null, + defaultValue: "('rec_'::text || (xata_private.xid())::text)", + comment: '', + }, + { + name: 'xata_updatedat', + type: 'datetime', + notNull: true, + unique: false, + defaultValue: 'now()', + comment: '', + }, + { + name: 'xata_version', + type: 'int', + notNull: true, + unique: false, + defaultValue: '0', comment: '', }, ], @@ -96,8 +191,12 @@ export type InferredTypes = SchemaInference; export type ClimbingTiles = InferredTypes['climbing_tiles']; export type ClimbingTilesRecord = ClimbingTiles & XataRecord; +export type TestXataCols = InferredTypes['test_xata_cols']; +export type TestXataColsRecord = TestXataCols & XataRecord; + export type DatabaseSchema = { climbing_tiles: ClimbingTilesRecord; + test_xata_cols: TestXataColsRecord; }; const DatabaseClient = buildClient(); diff --git a/yarn.lock b/yarn.lock index 7e4b2077b..39d0a4b57 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4140,6 +4140,11 @@ gensync@^1.0.0-beta.2: resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" integrity sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg== +geohashing@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/geohashing/-/geohashing-2.0.1.tgz#c081070a2b6ca551930f75ce8586afe0dd179944" + integrity sha512-u0H29yhqHEyBBgwv1GJrvqbD33AnQm2lbWOLNbXLZEWVrCKNoblyiWWnVGeYGaM4NNM7rBBTqvJZhZ/CDfrBVw== + geojson-vt@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/geojson-vt/-/geojson-vt-4.0.2.tgz#1162f6c7d61a0ba305b1030621e6e111f847828a" From f5e0875a4ee287ef2f83b3225a0b429e134db34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Tue, 26 Nov 2024 16:56:23 +0100 Subject: [PATCH 18/22] from overpass --- src/server/climbing-tiles/refresh.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/server/climbing-tiles/refresh.ts b/src/server/climbing-tiles/refresh.ts index 3a5d6b3da..d3ce8a14e 100644 --- a/src/server/climbing-tiles/refresh.ts +++ b/src/server/climbing-tiles/refresh.ts @@ -200,15 +200,13 @@ export const refresh = async (log: (line: string) => void) => { await client.connect(); const start = performance.now(); - log('Deleting old records...'); - await client.query('DELETE FROM climbing_tiles'); log('Fetching data from Overpass...'); - // const data = await fetchFromOverpass(); + const data = await fetchFromOverpass(); // fs.writeFileSync('../overpass.json', JSON.stringify(data)); - const data = JSON.parse( - fs.readFileSync('../overpass.json').toString(), - ) as OsmResponse; + // const data = JSON.parse( + // fs.readFileSync('../overpass.json').toString(), + // ) as OsmResponse; log(`Overpass elements: ${data.elements.length}`); const records = getNewRecords(data); // ~ 16k records @@ -217,7 +215,7 @@ export const refresh = async (log: (line: string) => void) => { const columns = Object.keys(records[0]); const values = records.map((record) => Object.values(record)); const query = format( - `INSERT INTO climbing_tiles(%I) VALUES %L`, + `DELETE FROM climbing_tiles;INSERT INTO climbing_tiles(%I) VALUES %L`, columns, values, ); From 209d851173e75d0747cde41f8113a278947d91f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Thu, 5 Dec 2024 10:35:51 +0100 Subject: [PATCH 19/22] fix sortKey for groups without `name` --- src/components/Map/styles/layers/climbingLayers.ts | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/components/Map/styles/layers/climbingLayers.ts b/src/components/Map/styles/layers/climbingLayers.ts index ae04024d3..6a81563d9 100644 --- a/src/components/Map/styles/layers/climbingLayers.ts +++ b/src/components/Map/styles/layers/climbingLayers.ts @@ -75,15 +75,9 @@ const sortKey = [ -1, [ '+', - ['coalesce', ['get', 'osmappRouteCount'], 0], - ['case', ['get', 'osmappHasImages'], 99999, 0], // preference for items with images - ['case', ['get', 'name'], 2, 0], // prefer items with name - // [ - // 'case', - // ['all', ['>', ['zoom'], 16], ['==', ['get', 'climbing'], 'crag']], - // 999, - // 0, - // ], // prefer areas on low zoom + ['to-number', ['get', 'osmappRouteCount']], + ['case', ['get', 'osmappHasImages'], 10000, 0], // preference for items with images + ['case', ['to-boolean', ['get', 'name']], 2, 0], // prefer items with name ], ] as DataDrivenPropertyValueSpecification; @@ -113,7 +107,6 @@ export const routes: LayerSpecification[] = [ minzoom: 16, filter: ['all', ['==', 'type', 'route']], }, - { id: 'climbing-3-routes-circle', type: 'circle', From a1c577db2e4b3d931e54cfd2845cb59285ed1d80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Sat, 7 Dec 2024 07:58:49 +0100 Subject: [PATCH 20/22] MVT --- package.json | 1 + pages/api/climbing-tiles/tile.ts | 10 +++-- src/server/climbing-tiles/climbing-tile.ts | 48 +++++++++++----------- tsconfig.json | 2 +- yarn.lock | 5 +++ 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/package.json b/package.json index 8614819f2..14fde42eb 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@emotion/react": "^11.13.3", "@emotion/server": "^11.11.0", "@emotion/styled": "^11.13.0", + "@mapbox/tilebelt": "^2.0.2", "@mui/icons-material": "^6.1.3", "@mui/lab": "^6.0.0-beta.11", "@mui/material": "^6.1.3", diff --git a/pages/api/climbing-tiles/tile.ts b/pages/api/climbing-tiles/tile.ts index 744f2bed4..6e13290b3 100644 --- a/pages/api/climbing-tiles/tile.ts +++ b/pages/api/climbing-tiles/tile.ts @@ -1,14 +1,18 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import { climbingTile, - climbingTilePg, + TileNumber, } from '../../../src/server/climbing-tiles/climbing-tile'; export default async (req: NextApiRequest, res: NextApiResponse) => { try { - const json = req.query.pg ? await climbingTilePg() : await climbingTile(); + const tileNumber = [req.query.z, req.query.x, req.query.y].map( + Number, + ) as TileNumber; - res.status(200).json(json); + const buffer = await climbingTile(tileNumber); + + res.status(200).send(buffer); } catch (err) { console.error(err); // eslint-disable-line no-console res.status(err.code ?? 400).send(String(err)); diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts index 2f778b705..527fcf0b0 100644 --- a/src/server/climbing-tiles/climbing-tile.ts +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -1,25 +1,11 @@ -import { xata } from '../db/db'; import { Client } from 'pg'; +import vtpbf from 'vt-pbf'; +import geojsonVt from 'geojson-vt'; +import * as tilebelt from '@mapbox/tilebelt'; -export const climbingTile = async () => { - const start = performance.now(); - const alldata = await xata.db.climbing_tiles - .select(['geojson']) - .filter({ - // type: '_otherWays', - // lat: { $gt: 48, $lt: 51 }, - // lon: { $gt: 14, $lt: 19 }, - type: 'group', - geohash: 'u2', - }) - .getAll(); - - console.log('climbingTileXata', performance.now() - start, alldata.length); - - return alldata.map((record) => record.geojson); -}; +export type TileNumber = [number, number, number]; -export const climbingTilePg = async () => { +const fetchFromDb = async ([z, x, y]: TileNumber) => { const start = performance.now(); const client = new Client({ @@ -35,11 +21,27 @@ export const climbingTilePg = async () => { await client.connect(); - const result = await client.query( - "SELECT geojson FROM climbing_tiles WHERE type='group' AND geohash = 'u2'", - ); + const bbox = tilebelt.tileToBBOX([z, x, y]); + + const query = + z < 12 + ? `SELECT geojson FROM climbing_tiles WHERE type='group' AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}` + : `SELECT geojson FROM climbing_tiles WHERE type IN ('group', 'route') AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}`; + const result = await client.query(query); + const geojson = { + type: 'FeatureCollection', + features: result.rows.map((record) => record.geojson), + } as GeoJSON.FeatureCollection; console.log('climbingTilePg', performance.now() - start, result.rows.length); - return result.rows.map((record) => record.geojson); + return geojson; +}; + +export const climbingTile = async ([z, x, y]: TileNumber) => { + const orig = await fetchFromDb([z, x, y]); + const tileindex = geojsonVt(orig, {}); + const tile = tileindex.getTile(z, x, y); + + return vtpbf.fromGeojsonVt({ groups: tile }); }; diff --git a/tsconfig.json b/tsconfig.json index cb770944d..39c8fa219 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -10,7 +10,7 @@ "noEmit": true, "esModuleInterop": true, "module": "esnext", - "moduleResolution": "node", + "moduleResolution": "node16", "resolveJsonModule": true, "isolatedModules": true, "jsx": "preserve", diff --git a/yarn.lock b/yarn.lock index 39d0a4b57..7f9fc5c22 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1001,6 +1001,11 @@ resolved "https://registry.yarnpkg.com/@mapbox/point-geometry/-/point-geometry-0.1.0.tgz#8a83f9335c7860effa2eeeca254332aa0aeed8f2" integrity sha1-ioP5M1x4YO/6Lu7KJUMyqgru2PI= +"@mapbox/tilebelt@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@mapbox/tilebelt/-/tilebelt-2.0.2.tgz#8176d82a05541acee8104fe1d280981ebd8d4b7a" + integrity sha512-pzFgrmsCUjOAAeYns1MKJ1dJICYdEC+sYwtWLdxO7iJwmDlepGrEXDqP9ejApO/YUVctA/SYKDS4Oc0mBNNLZw== + "@mapbox/tiny-sdf@^2.0.6": version "2.0.6" resolved "https://registry.yarnpkg.com/@mapbox/tiny-sdf/-/tiny-sdf-2.0.6.tgz#9a1d33e5018093e88f6a4df2343e886056287282" From 31ad8221455bbe63df54bac84af55fa4216b9252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Fri, 10 Jan 2025 16:25:16 +0100 Subject: [PATCH 21/22] MVT works http://localhost:3000/api/climbing-tiles/tile?z=10&x=553&y=346 --- pages/api/climbing-tiles/refresh.ts | 3 +- pages/api/climbing-tiles/tile.ts | 8 ++-- src/server/climbing-tiles/climbing-tile.ts | 31 +++++++++++-- yarn.lock | 52 +++++++++++++++++++++- 4 files changed, 84 insertions(+), 10 deletions(-) diff --git a/pages/api/climbing-tiles/refresh.ts b/pages/api/climbing-tiles/refresh.ts index 8193ed63e..87e4718d9 100644 --- a/pages/api/climbing-tiles/refresh.ts +++ b/pages/api/climbing-tiles/refresh.ts @@ -1,6 +1,5 @@ import type { NextApiRequest, NextApiResponse } from 'next'; -import { climbingTile } from '../../../src/server/climbing-tiles/climbing-tile'; -import { refresh } from '../../../src/server/climbing-tiles/refresh'; +import { refresh } from '../../../src/server/climbing-tiles/refresh.js'; export default async (req: NextApiRequest, res: NextApiResponse) => { try { diff --git a/pages/api/climbing-tiles/tile.ts b/pages/api/climbing-tiles/tile.ts index 6e13290b3..133448acd 100644 --- a/pages/api/climbing-tiles/tile.ts +++ b/pages/api/climbing-tiles/tile.ts @@ -10,9 +10,11 @@ export default async (req: NextApiRequest, res: NextApiResponse) => { Number, ) as TileNumber; - const buffer = await climbingTile(tileNumber); - - res.status(200).send(buffer); + const buffer = await climbingTile(tileNumber, req.query.type); + res + .setHeader('Content-Type', 'application/x-protobuf') + .status(200) + .end(buffer); } catch (err) { console.error(err); // eslint-disable-line no-console res.status(err.code ?? 400).send(String(err)); diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts index 527fcf0b0..aa85fae97 100644 --- a/src/server/climbing-tiles/climbing-tile.ts +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -1,9 +1,27 @@ import { Client } from 'pg'; import vtpbf from 'vt-pbf'; import geojsonVt from 'geojson-vt'; -import * as tilebelt from '@mapbox/tilebelt'; +import { BBox } from 'geojson'; -export type TileNumber = [number, number, number]; +const r2d = 180 / Math.PI; + +function tile2lon(x: number, z: number): number { + return (x / Math.pow(2, z)) * 360 - 180; +} + +function tile2lat(y: number, z: number): number { + const n = Math.PI - (2 * Math.PI * y) / Math.pow(2, z); + return r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n))); +} +export function tileToBBOX([z, x, y]: TileNumber): BBox { + const e = tile2lon(x + 1, z); + const w = tile2lon(x, z); + const s = tile2lat(y + 1, z); + const n = tile2lat(y, z); + return [w, s, e, n]; +} + +export type TileNumber = [number, number, number]; // z,x,y const fetchFromDb = async ([z, x, y]: TileNumber) => { const start = performance.now(); @@ -21,7 +39,7 @@ const fetchFromDb = async ([z, x, y]: TileNumber) => { await client.connect(); - const bbox = tilebelt.tileToBBOX([z, x, y]); + const bbox = tileToBBOX([z, x, y]); const query = z < 12 @@ -38,7 +56,12 @@ const fetchFromDb = async ([z, x, y]: TileNumber) => { return geojson; }; -export const climbingTile = async ([z, x, y]: TileNumber) => { +export const climbingTile = async ([z, x, y]: TileNumber, type: string) => { + if (type === 'json') { + const orig = await fetchFromDb([z, x, y]); + return JSON.stringify(orig); + } + const orig = await fetchFromDb([z, x, y]); const tileindex = geojsonVt(orig, {}); const tile = tileindex.getTile(z, x, y); diff --git a/yarn.lock b/yarn.lock index 7f9fc5c22..5a3c3b8c7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6461,6 +6461,21 @@ performance-now@^2.1.0: resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== +pg-cloudflare@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz#e6d5833015b170e23ae819e8c5d7eaedb472ca98" + integrity sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q== + +pg-connection-string@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.7.0.tgz#f1d3489e427c62ece022dba98d5262efcb168b37" + integrity sha512-PI2W9mv53rXJQEOb8xNR8lH7Hr+EKa6oJa38zsK0S/ky2er16ios1wLKhZyxzD7jUReiWokc9WK5nxSnC7W1TA== + +pg-format@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/pg-format/-/pg-format-1.0.4.tgz#27734236c2ad3f4e5064915a59334e20040a828e" + integrity sha512-YyKEF78pEA6wwTAqOUaHIN/rWpfzzIuMh9KdAhc3rSLQ/7zkRFcCgYBAEGatDstLyZw4g0s9SNICmaTGnBVeyw== + pg-int8@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" @@ -6471,12 +6486,22 @@ pg-numeric@1.0.2: resolved "https://registry.yarnpkg.com/pg-numeric/-/pg-numeric-1.0.2.tgz#816d9a44026086ae8ae74839acd6a09b0636aa3a" integrity sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw== +pg-pool@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.7.0.tgz#d4d3c7ad640f8c6a2245adc369bafde4ebb8cbec" + integrity sha512-ZOBQForurqh4zZWjrgSwwAtzJ7QiRX0ovFkZr2klsen3Nm0aoh33Ls0fzfv3imeH/nw/O27cjdz5kzYJfeGp/g== + pg-protocol@*: version "1.6.1" resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.6.1.tgz#21333e6d83b01faaebfe7a33a7ad6bfd9ed38cb3" integrity sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg== -pg-types@^2.2.0: +pg-protocol@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.7.0.tgz#ec037c87c20515372692edac8b63cf4405448a93" + integrity sha512-hTK/mE36i8fDDhgDFjy6xNOG+LCorxLG3WO17tku+ij6sVHXh1jQUJ8hYAnRhNla4QVD2H8er/FOjc/+EgC6yQ== + +pg-types@^2.1.0, pg-types@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== @@ -6500,6 +6525,26 @@ pg-types@^4.0.1: postgres-interval "^3.0.0" postgres-range "^1.1.1" +pg@^8.13.1: + version "8.13.1" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.13.1.tgz#6498d8b0a87ff76c2df7a32160309d3168c0c080" + integrity sha512-OUir1A0rPNZlX//c7ksiu7crsGZTKSOXJPgtNiHGIlC9H0lO+NC6ZDYksSgBYY/thSWhnSRBv8w1lieNNGATNQ== + dependencies: + pg-connection-string "^2.7.0" + pg-pool "^3.7.0" + pg-protocol "^1.7.0" + pg-types "^2.1.0" + pgpass "1.x" + optionalDependencies: + pg-cloudflare "^1.1.1" + +pgpass@1.x: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -7302,6 +7347,11 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.1: resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.6.1.tgz#74722af32e9614e9c287a8d0bbde48b5e2f1a263" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" From b2c3cfccc6a146136bb27b192090896851d6cc85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pavel=20Zbytovsk=C3=BD?= Date: Sat, 11 Jan 2025 15:22:31 +0100 Subject: [PATCH 22/22] MVT clientside + pg pool --- .../Map/behaviour/useUpdateStyle.tsx | 32 +++++++++++-------- src/components/Map/consts.ts | 5 +++ src/server/climbing-tiles/climbing-tile.ts | 24 ++++---------- src/server/climbing-tiles/db.ts | 27 ++++++++++++++++ 4 files changed, 58 insertions(+), 30 deletions(-) create mode 100644 src/server/climbing-tiles/db.ts diff --git a/src/components/Map/behaviour/useUpdateStyle.tsx b/src/components/Map/behaviour/useUpdateStyle.tsx index c2dc66459..6316402b0 100644 --- a/src/components/Map/behaviour/useUpdateStyle.tsx +++ b/src/components/Map/behaviour/useUpdateStyle.tsx @@ -64,19 +64,25 @@ const addRasterOverlay = ( }; const addClimbingOverlay = (style: StyleSpecification, map: Map) => { - style.sources.climbing = EMPTY_GEOJSON_SOURCE; - style.layers.push(...climbingLayers); // must be also in `layersWithOsmId` because of hover effect - style.sprite = [...OSMAPP_SPRITE, CLIMBING_SPRITE]; - - fetchCrags().then( - (geojson) => { - const geojsonSource = map.getSource('climbing') as GeoJSONSource; - geojsonSource?.setData(geojson); // TODO can be undefined at first map render - }, - (error) => { - console.warn('Climbing Layer failed to fetch.', error); // eslint-disable-line no-console - }, - ); + // style.sources.climbing = EMPTY_GEOJSON_SOURCE; + style.layers.push( + ...climbingLayers.map((x) => ({ + ...x, + source: 'climbingTiles', + 'source-layer': 'groups', + })), + ); // must be also in `layersWithOsmId` because of hover effect + // style.sprite = [...OSMAPP_SPRITE, CLIMBING_SPRITE]; + + // fetchCrags().then( + // (geojson) => { + // const geojsonSource = map.getSource('climbing') as GeoJSONSource; + // geojsonSource?.setData(geojson); // TODO can be undefined at first map render + // }, + // (error) => { + // console.warn('Climbing Layer failed to fetch.', error); // eslint-disable-line no-console + // }, + // ); }; const addOverlaysToStyle = ( diff --git a/src/components/Map/consts.ts b/src/components/Map/consts.ts index 838e779c7..1165d0124 100644 --- a/src/components/Map/consts.ts +++ b/src/components/Map/consts.ts @@ -50,6 +50,11 @@ export const OSMAPP_SOURCES: Record = { type: 'vector' as const, }, overpass: EMPTY_GEOJSON_SOURCE, + climbingTiles: { + type: 'vector' as const, + tiles: ['http://localhost:3000/api/climbing-tiles/tile?z={z}&x={x}&y={y}'], + maxzoom: 10, + }, }; export const BACKGROUND = [ diff --git a/src/server/climbing-tiles/climbing-tile.ts b/src/server/climbing-tiles/climbing-tile.ts index aa85fae97..b5b99cd1e 100644 --- a/src/server/climbing-tiles/climbing-tile.ts +++ b/src/server/climbing-tiles/climbing-tile.ts @@ -1,7 +1,7 @@ -import { Client } from 'pg'; import vtpbf from 'vt-pbf'; import geojsonVt from 'geojson-vt'; import { BBox } from 'geojson'; +import { getClient } from './db'; const r2d = 180 / Math.PI; @@ -26,23 +26,12 @@ export type TileNumber = [number, number, number]; // z,x,y const fetchFromDb = async ([z, x, y]: TileNumber) => { const start = performance.now(); - const client = new Client({ - user: 'tvgiad', - password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', - host: 'us-east-1.sql.xata.sh', - port: 5432, - database: 'db_with_direct_access:main', - ssl: { - rejectUnauthorized: false, - }, - }); - - await client.connect(); + const client = await getClient(); const bbox = tileToBBOX([z, x, y]); const query = - z < 12 + z < 10 ? `SELECT geojson FROM climbing_tiles WHERE type='group' AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}` : `SELECT geojson FROM climbing_tiles WHERE type IN ('group', 'route') AND lon >= ${bbox[0]} AND lon <= ${bbox[2]} AND lat >= ${bbox[1]} AND lat <= ${bbox[3]}`; const result = await client.query(query); @@ -50,7 +39,6 @@ const fetchFromDb = async ([z, x, y]: TileNumber) => { type: 'FeatureCollection', features: result.rows.map((record) => record.geojson), } as GeoJSON.FeatureCollection; - console.log('climbingTilePg', performance.now() - start, result.rows.length); return geojson; @@ -63,8 +51,10 @@ export const climbingTile = async ([z, x, y]: TileNumber, type: string) => { } const orig = await fetchFromDb([z, x, y]); - const tileindex = geojsonVt(orig, {}); + + const tileindex = geojsonVt(orig, { tolerance: 0 }); const tile = tileindex.getTile(z, x, y); + console.log(tile); - return vtpbf.fromGeojsonVt({ groups: tile }); + return tile ? vtpbf.fromGeojsonVt({ groups: tile }) : null; }; diff --git a/src/server/climbing-tiles/db.ts b/src/server/climbing-tiles/db.ts new file mode 100644 index 000000000..ada0345d5 --- /dev/null +++ b/src/server/climbing-tiles/db.ts @@ -0,0 +1,27 @@ +import { Client } from 'pg'; + +if (!global.db) { + global.db = { pool: false }; +} + +export async function getClient() { + if (!global.db.pool) { + console.log('No pool available, creating new pool.'); + + const client = new Client({ + user: 'tvgiad', + password: 'xau_E0h76BAWwiiGCOqEYZsRoCUQqXEQ3jpM', + host: 'us-east-1.sql.xata.sh', + port: 5432, + database: 'db_with_direct_access:main', + ssl: { + rejectUnauthorized: false, + }, + }); + + await client.connect(); + + global.db.pool = client; + } + return global.db.pool; +}