diff --git a/.env.example b/.env.example index e16c7fa..b8cf8d7 100644 --- a/.env.example +++ b/.env.example @@ -48,6 +48,35 @@ ADMIN_EMAIL=admin@example.com # if you want to use a different database for the location data. # DB_LOCATION_TABLE=location_history + +### +### Geocode configuration +### + +# Specify the geocode provider to use. The default is none (empty). +# If set, then, upon ingestion, any points with missing address metadata +# (address, locality, country, postal code) will be geocoded using the Google +# Maps API or the Nominatim API and the metadata will be updated in the +# database. +# The available options are: +# - "nominatim" +# - "google" (requires GOOGLE_API_KEY to be set) +# GEOCODE_PROVIDER=nominatim + +# Specify the Nominatim API URL to use for geocoding and reverse geocoding +# if you have set GEOCODE_PROVIDER to "nominatim". The default one +# (https://nominatim.openstreetmap.org) will be used if not set, but keep in +# mind that it is rate-limited to 1 request per second. +# NOMINATIM_API_URL=https://nominatim.openstreetmap.org + +# User agent to use for the Nominatim API. The default one is +# "Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)" +# NOMINATIM_USER_AGENT=YourUserAgent + +# Specify a Google API key to use the Google Maps API for geocoding and reverse +# geocoding, if you have set GEOCODE_PROVIDER to "google". +# GOOGLE_API_KEY=your_google_api_key + ### ### Location history table column mappings. ### The following settings are only taken into account when you use a different database diff --git a/README.md b/README.md index 5ef1bce..cb16e87 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ - [Usage](#usage) * [Initial setup](#initial-setup) * [Ingestion](#ingestion) + + [Enriching location data](#enriching-location-data) * [External data sources](#external-data-sources) - [Development](#development) * [Compile and Hot-Reload for Development](#compile-and-hot-reload-for-development) @@ -173,6 +174,33 @@ Or, for more advanced use cases, you can use a general-purpose application like to send data to the endpoint, or decouple the ingestion from the frontend by using an intermediate MQTT or Kafka broker. +#### Enriching location data + +If the ingested location data does not contain the `address`, `locality`, +`country` or `postalCode` fields, and you have set the `GEOCODE_PROVIDER` +environment variable, then the application will try to geocode the location +upon ingestion using the configured geocoding provider. Supported providers: + +- [`nominatim`](https://nominatim.org/) + - It uses OpenStreetMap data ([usage + policy](https://operations.osmfoundation.org/policies/nominatim/)). + - It doesn't require an API key, but it is rate-limited to 1 request per + second (not advised if you are ingesting bulk data if you use the + default `NOMINATIM_URL` instance). + - It supports a custom `NOMINATIM_URL` environment variable to use a custom + Nominatim instance. +- [`google`](https://developers.google.com/maps/documentation/geocoding/start) + - It requires a Google Maps API key, set in the `GOOGLE_API_KEY` environment + variable. + - See [additional usage + limits](https://developers.google.com/maps/documentation/geocoding/usage-and-billing) + for details. You can set your own usage limits in the Google Cloud + console, but keep in mind that above a certain threshold you will be + charged. + +If `GEOCODE_PROVIDER` is not set, the application will not attempt to geocode +the location data upon ingestion. + ### External data sources By default, the application will store the GPS data under the configured diff --git a/src/Secrets.ts b/src/Secrets.ts index 384c2a3..71d896d 100644 --- a/src/Secrets.ts +++ b/src/Secrets.ts @@ -2,15 +2,18 @@ class Secrets { public readonly serverKey: string; public readonly adminPassword: string; public readonly adminEmail: string; + public readonly googleApiKey?: string; - private constructor({ - serverKey, - adminPassword, - adminEmail, - }: any) { - this.serverKey = serverKey; - this.adminPassword = adminPassword; - this.adminEmail = adminEmail; + private constructor(args: { + serverKey: string; + adminPassword: string; + adminEmail: string; + googleApiKey?: string; + }) { + this.serverKey = args.serverKey; + this.adminPassword = args.adminPassword; + this.adminEmail = args.adminEmail; + this.googleApiKey = args.googleApiKey; } public static fromEnv(): Secrets { @@ -33,6 +36,7 @@ class Secrets { serverKey: process.env.SERVER_KEY, adminPassword: process.env.ADMIN_PASSWORD, adminEmail: process.env.ADMIN_EMAIL, + googleApiKey: process.env.GOOGLE_API_KEY, }); } } diff --git a/src/config/Geocode.ts b/src/config/Geocode.ts new file mode 100644 index 0000000..663c4bd --- /dev/null +++ b/src/config/Geocode.ts @@ -0,0 +1,71 @@ +import { ValidationError } from '../errors'; + +type GeocodeProvider = 'google' | 'nominatim'; + +class NominatimConfig { + public readonly url: string; + public readonly userAgent: string; + + private constructor(args: { url: string; userAgent: string }) { + this.url = args.url; + this.userAgent = args.userAgent; + } + + public static fromEnv(): NominatimConfig { + return new NominatimConfig({ + url: process.env.NOMINATIM_URL || 'https://nominatim.openstreetmap.org', + userAgent: process.env.NOMINATIM_USER_AGENT || 'Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)', + }); + } +} + +class GoogleConfig { + public readonly url: string; + public readonly apiKey: string; + + private constructor(args: { apiKey: string }) { + this.url = 'https://maps.googleapis.com/maps/api/geocode/json' + this.apiKey = args.apiKey; + } + + public static fromEnv(): GoogleConfig { + return new GoogleConfig({ + apiKey: process.env.GOOGLE_API_KEY || '', + }); + } +} + +class Geocode { + public readonly provider?: GeocodeProvider; + public readonly nominatim: NominatimConfig; + public readonly google: GoogleConfig; + + private constructor(args: { + provider?: GeocodeProvider; + nominatim: NominatimConfig; + google: GoogleConfig; + }) { + this.provider = args.provider; + this.nominatim = args.nominatim; + this.google = args.google; + + if (this.provider === 'google' && !this.google.apiKey) { + throw new ValidationError('Google API key is required when using Google geocoding.'); + } + } + + public static fromEnv(): Geocode { + const provider = process.env.GEOCODE_PROVIDER as GeocodeProvider | undefined; + if (provider?.length && provider !== 'google' && provider !== 'nominatim') { + throw new ValidationError('GEOCODE_PROVIDER must be either "google" or "nominatim".'); + } + + return new Geocode({ + provider, + nominatim: NominatimConfig.fromEnv(), + google: GoogleConfig.fromEnv(), + }); + } +} + +export default Geocode; diff --git a/src/ext/location/GoogleLocationInfoProvider.ts b/src/ext/location/GoogleLocationInfoProvider.ts new file mode 100644 index 0000000..2eba948 --- /dev/null +++ b/src/ext/location/GoogleLocationInfoProvider.ts @@ -0,0 +1,105 @@ +import { GPSPoint } from "../../models"; + +class GoogleLocationInfoProvider { + private apiKey: string; + private apiUrl: string; + + constructor() { + this.apiKey = $geocode.google.apiKey + this.apiUrl = $geocode.google.url + } + + private parseAddressComponents(response: { + results: { + address_components: { + long_name: string; + short_name: string; + types: string[]; + }[]; + }[] + }): { + address?: string; + locality?: string; + postalCode?: string; + country?: string; + description?: string; + } { + const result = { + address: undefined, + locality: undefined, + postalCode: undefined, + country: undefined, + description: undefined, + } as { + address?: string; + locality?: string; + postalCode?: string; + country?: string; + description?: string; + }; + + if (!response.results?.length) { + return result; + } + + const addressComponents = response.results[0].address_components.reduce( + (acc: any, component: any) => { + ['street_number', 'route', 'locality', 'postal_code'].forEach((type) => { + if (component.types.includes(type)) { + acc[type] = component.long_name; + } + }); + + if (component.types.includes('country')) { + acc.country = component.short_name.toLowerCase(); + } + + return acc; + }, + {}, + ); + + if (addressComponents.route) { + result.address = ( + (addressComponents.route || '') + + (addressComponents.street_number ? ' ' + addressComponents.street_number : '') + ).trim(); + + if (!result.address?.length) { + result.address = undefined; + } + } + + ['locality', 'postal_code', 'country'].forEach((key) => { + if (addressComponents[key]) { + // @ts-expect-error + result[key] = addressComponents[key]; + } + }); + + return result; + } + + async getLocationInfo(location: GPSPoint): Promise<GPSPoint> { + const response = await fetch( + `${this.apiUrl}?latlng=${location.latitude},${location.longitude}&key=${this.apiKey}`, + ); + + if (!response.ok) { + throw new Error(`Error fetching location info: ${response.statusText}`); + } + + const data = await response.json(); + const addressComponents = this.parseAddressComponents(data); + + return new GPSPoint({ + ...location, + address: location.address || addressComponents.address, + locality: location.locality || addressComponents.locality, + postalCode: location.postalCode || addressComponents.postalCode, + country: location.country || addressComponents.country, + }) + } +} + +export default GoogleLocationInfoProvider; diff --git a/src/ext/location/LocationInfoProvider.ts b/src/ext/location/LocationInfoProvider.ts new file mode 100644 index 0000000..b54a180 --- /dev/null +++ b/src/ext/location/LocationInfoProvider.ts @@ -0,0 +1,24 @@ +import { GPSPoint } from "~/models"; +import { useGlobals } from '../../globals'; +import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider"; +import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider"; + +useGlobals(); + +abstract class LocationInfoProvider { + // TODO Cache location info + abstract getLocationInfo: (location: GPSPoint) => Promise<GPSPoint>; + + static get(): LocationInfoProvider | undefined { + switch ($geocode.provider) { + case 'nominatim': + return new NominatimLocationInfoProvider(); + case 'google': + return new GoogleLocationInfoProvider(); + } + + return undefined; + }; +} + +export default LocationInfoProvider; diff --git a/src/ext/location/NominatimLocationInfoProvider.ts b/src/ext/location/NominatimLocationInfoProvider.ts new file mode 100644 index 0000000..760e0d4 --- /dev/null +++ b/src/ext/location/NominatimLocationInfoProvider.ts @@ -0,0 +1,54 @@ +import { GPSPoint } from "../../models"; + +class NominatimLocationInfoProvider { + private apiUrl: string; + private userAgent: string; + + constructor() { + this.apiUrl = $geocode.nominatim.url; + this.userAgent = $geocode.nominatim.userAgent; + } + + async getLocationInfo(location: GPSPoint): Promise<GPSPoint> { + const response = await fetch( + `${this.apiUrl}/reverse?lat=${location.latitude}&lon=${location.longitude}&format=json`, + { + headers: { + 'User-Agent': this.userAgent, + }, + } + ); + + if (!response.ok) { + throw new Error(`Error fetching location info: ${response.statusText}`); + } + + const data = await response.json(); + const address = data.address || {}; + + if (Object.keys(address).length > 0) { + let addressString: string | undefined = ( + (address.road || '') + ( + address.house_number ? (' ' + address.house_number) : '' + ) + ).trim(); + + if (!addressString?.length) { + addressString = undefined; + } + + return new GPSPoint({ + ...location, + description: location.description || address.amenity, + address: addressString, + locality: address.city || address.town || address.village, + postalCode: address.postcode, + country: address.country_code, + }) + } + + return location; + } +} + +export default NominatimLocationInfoProvider; diff --git a/src/ext/location/index.ts b/src/ext/location/index.ts new file mode 100644 index 0000000..5fe2592 --- /dev/null +++ b/src/ext/location/index.ts @@ -0,0 +1,9 @@ +import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider"; +import LocationInfoProvider from "./LocationInfoProvider"; +import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider"; + +export { + GoogleLocationInfoProvider, + LocationInfoProvider, + NominatimLocationInfoProvider, +}; diff --git a/src/globals.ts b/src/globals.ts index 2ddb3eb..2e4d64f 100644 --- a/src/globals.ts +++ b/src/globals.ts @@ -1,6 +1,7 @@ import dotenv from 'dotenv'; import { Db } from './db'; +import Geocode from './config/Geocode'; import Secrets from './Secrets'; import Repositories from './repos'; @@ -10,10 +11,12 @@ declare global { var $db: Db; var $repos: Repositories; var $secrets: Secrets; + var $geocode: Geocode; } export function useGlobals() { globalThis.$secrets = Secrets.fromEnv(); globalThis.$db = Db.fromEnv(); + globalThis.$geocode = Geocode.fromEnv(); globalThis.$repos = new Repositories(); } diff --git a/src/routes/api/v1/GPSData.ts b/src/routes/api/v1/GPSData.ts index 85f5e31..0a1c45c 100644 --- a/src/routes/api/v1/GPSData.ts +++ b/src/routes/api/v1/GPSData.ts @@ -2,6 +2,7 @@ import { Request, Response } from 'express'; import { authenticate } from '../../../auth'; import { AuthInfo } from '../../../auth'; +import { LocationInfoProvider } from '../../../ext/location'; import { LocationRequest } from '../../../requests'; import { Optional } from '../../../types'; import { GPSPoint, RoleName } from '../../../models'; @@ -22,6 +23,30 @@ class GPSData extends ApiV1Route { } }; + private enrichWithLocationInfo = async (gpsData: GPSPoint[]) => { + const provider = LocationInfoProvider.get(); + if (!provider) { + return gpsData; + } + + return await Promise.all( + gpsData + .map(async (point) => { + // Only enrich points that have latitude and longitude, but no + // location info + if ( + !(point.latitude && point.longitude) || + (point.country && point.locality && point.address) + ) { + return point; + } + + const locationInfo = await provider.getLocationInfo(point); + return { ...point, ...locationInfo }; + }) + ); + } + @authenticate() get = async (req: Request, res: Response, auth: Optional<AuthInfo>) => { let query: LocationRequest @@ -44,7 +69,10 @@ class GPSData extends ApiV1Route { post = async (req: Request, res: Response, auth: Optional<AuthInfo>) => { const deviceIds = req.body.map((p: any) => p.deviceId).filter((d: any) => !!d); this.validateOwnership(deviceIds, auth!); - await $repos.location.createPoints(req.body); + + const points = await this.enrichWithLocationInfo(req.body as GPSPoint[]); + console.log(`Storing ${points.length} location point${points.length > 1 ? 's' : ''}`); + await $repos.location.createPoints(points); res.status(201).send(); } diff --git a/src/routes/api/v1/LocationInfo.ts b/src/routes/api/v1/LocationInfo.ts new file mode 100644 index 0000000..bf5144d --- /dev/null +++ b/src/routes/api/v1/LocationInfo.ts @@ -0,0 +1,43 @@ +import { Request, Response } from 'express'; + +import { authenticate } from '../../../auth'; +import { GPSPoint } from '../../../models'; +import { LocationInfoProvider } from '../../../ext/location'; +import ApiV1Route from './Route'; + +class LocationInfo extends ApiV1Route { + private provider: LocationInfoProvider | undefined; + + constructor() { + super('/location-info'); + this.provider = LocationInfoProvider.get(); + } + + @authenticate() + get = async (req: Request, res: Response) => { + if (!this.provider) { + res.status(500).send('Location info provider not configured'); + return; + } + + let location: GPSPoint; + + try { + location = new GPSPoint(req.query); + if (!(location?.latitude && location?.longitude)) { + res.status(400).send('Invalid GPS coordinates'); + return; + } + } catch (error) { + const e = `Error parsing location request: ${error}`; + console.warn(e); + res.status(400).send(e); + return; + } + + location = await this.provider.getLocationInfo(location); + res.json(location); + } +} + +export default LocationInfo; diff --git a/src/routes/api/v1/index.ts b/src/routes/api/v1/index.ts index a80c806..b2a0751 100644 --- a/src/routes/api/v1/index.ts +++ b/src/routes/api/v1/index.ts @@ -2,6 +2,7 @@ import Auth from "./Auth"; import Devices from "./Devices"; import DevicesById from "./DevicesById"; import GPSData from "./GPSData"; +import LocationInfo from "./LocationInfo"; import Routes from "../../Routes"; import Stats from "./Stats"; import Tokens from "./Tokens"; @@ -14,6 +15,7 @@ class ApiV1Routes extends Routes { new Devices(), new DevicesById(), new GPSData(), + new LocationInfo(), new Stats(), new Tokens(), new TokensById(),