Added location info enrichment.

- Adds `GET /location-info?latitude=<...>&longitude=<...>` endpoint to
  get information about a point at the given coordinates.

- Adds `nominatim` and `google` as reverse geocode providers.

- Adds support for automatic enrichment upon ingestion.
This commit is contained in:
Fabio Manganiello 2025-04-12 23:35:18 +02:00
parent d52fed5a57
commit 9ce57b23cd
Signed by: blacklight
GPG key ID: D90FBA7F76362774
12 changed files with 409 additions and 9 deletions

View file

@ -48,6 +48,35 @@ ADMIN_EMAIL=admin@example.com
# if you want to use a different database for the location data.
# DB_LOCATION_TABLE=location_history
###
### Geocode configuration
###
# Specify the geocode provider to use. The default is none (empty).
# If set, then, upon ingestion, any points with missing address metadata
# (address, locality, country, postal code) will be geocoded using the Google
# Maps API or the Nominatim API and the metadata will be updated in the
# database.
# The available options are:
# - "nominatim"
# - "google" (requires GOOGLE_API_KEY to be set)
# GEOCODE_PROVIDER=nominatim
# Specify the Nominatim API URL to use for geocoding and reverse geocoding
# if you have set GEOCODE_PROVIDER to "nominatim". The default one
# (https://nominatim.openstreetmap.org) will be used if not set, but keep in
# mind that it is rate-limited to 1 request per second.
# NOMINATIM_API_URL=https://nominatim.openstreetmap.org
# User agent to use for the Nominatim API. The default one is
# "Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)"
# NOMINATIM_USER_AGENT=YourUserAgent
# Specify a Google API key to use the Google Maps API for geocoding and reverse
# geocoding, if you have set GEOCODE_PROVIDER to "google".
# GOOGLE_API_KEY=your_google_api_key
###
### Location history table column mappings.
### The following settings are only taken into account when you use a different database

View file

@ -18,6 +18,7 @@
- [Usage](#usage)
* [Initial setup](#initial-setup)
* [Ingestion](#ingestion)
+ [Enriching location data](#enriching-location-data)
* [External data sources](#external-data-sources)
- [Development](#development)
* [Compile and Hot-Reload for Development](#compile-and-hot-reload-for-development)
@ -173,6 +174,33 @@ Or, for more advanced use cases, you can use a general-purpose application like
to send data to the endpoint, or decouple the ingestion from the frontend by
using an intermediate MQTT or Kafka broker.
#### Enriching location data
If the ingested location data does not contain the `address`, `locality`,
`country` or `postalCode` fields, and you have set the `GEOCODE_PROVIDER`
environment variable, then the application will try to geocode the location
upon ingestion using the configured geocoding provider. Supported providers:
- [`nominatim`](https://nominatim.org/)
- It uses OpenStreetMap data ([usage
policy](https://operations.osmfoundation.org/policies/nominatim/)).
- It doesn't require an API key, but it is rate-limited to 1 request per
second (not advised if you are ingesting bulk data if you use the
default `NOMINATIM_URL` instance).
- It supports a custom `NOMINATIM_URL` environment variable to use a custom
Nominatim instance.
- [`google`](https://developers.google.com/maps/documentation/geocoding/start)
- It requires a Google Maps API key, set in the `GOOGLE_API_KEY` environment
variable.
- See [additional usage
limits](https://developers.google.com/maps/documentation/geocoding/usage-and-billing)
for details. You can set your own usage limits in the Google Cloud
console, but keep in mind that above a certain threshold you will be
charged.
If `GEOCODE_PROVIDER` is not set, the application will not attempt to geocode
the location data upon ingestion.
### External data sources
By default, the application will store the GPS data under the configured

View file

@ -2,15 +2,18 @@ class Secrets {
public readonly serverKey: string;
public readonly adminPassword: string;
public readonly adminEmail: string;
public readonly googleApiKey?: string;
private constructor({
serverKey,
adminPassword,
adminEmail,
}: any) {
this.serverKey = serverKey;
this.adminPassword = adminPassword;
this.adminEmail = adminEmail;
private constructor(args: {
serverKey: string;
adminPassword: string;
adminEmail: string;
googleApiKey?: string;
}) {
this.serverKey = args.serverKey;
this.adminPassword = args.adminPassword;
this.adminEmail = args.adminEmail;
this.googleApiKey = args.googleApiKey;
}
public static fromEnv(): Secrets {
@ -33,6 +36,7 @@ class Secrets {
serverKey: process.env.SERVER_KEY,
adminPassword: process.env.ADMIN_PASSWORD,
adminEmail: process.env.ADMIN_EMAIL,
googleApiKey: process.env.GOOGLE_API_KEY,
});
}
}

71
src/config/Geocode.ts Normal file
View file

@ -0,0 +1,71 @@
import { ValidationError } from '../errors';
type GeocodeProvider = 'google' | 'nominatim';
class NominatimConfig {
public readonly url: string;
public readonly userAgent: string;
private constructor(args: { url: string; userAgent: string }) {
this.url = args.url;
this.userAgent = args.userAgent;
}
public static fromEnv(): NominatimConfig {
return new NominatimConfig({
url: process.env.NOMINATIM_URL || 'https://nominatim.openstreetmap.org',
userAgent: process.env.NOMINATIM_USER_AGENT || 'Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)',
});
}
}
class GoogleConfig {
public readonly url: string;
public readonly apiKey: string;
private constructor(args: { apiKey: string }) {
this.url = 'https://maps.googleapis.com/maps/api/geocode/json'
this.apiKey = args.apiKey;
}
public static fromEnv(): GoogleConfig {
return new GoogleConfig({
apiKey: process.env.GOOGLE_API_KEY || '',
});
}
}
class Geocode {
public readonly provider?: GeocodeProvider;
public readonly nominatim: NominatimConfig;
public readonly google: GoogleConfig;
private constructor(args: {
provider?: GeocodeProvider;
nominatim: NominatimConfig;
google: GoogleConfig;
}) {
this.provider = args.provider;
this.nominatim = args.nominatim;
this.google = args.google;
if (this.provider === 'google' && !this.google.apiKey) {
throw new ValidationError('Google API key is required when using Google geocoding.');
}
}
public static fromEnv(): Geocode {
const provider = process.env.GEOCODE_PROVIDER as GeocodeProvider | undefined;
if (provider?.length && provider !== 'google' && provider !== 'nominatim') {
throw new ValidationError('GEOCODE_PROVIDER must be either "google" or "nominatim".');
}
return new Geocode({
provider,
nominatim: NominatimConfig.fromEnv(),
google: GoogleConfig.fromEnv(),
});
}
}
export default Geocode;

View file

@ -0,0 +1,105 @@
import { GPSPoint } from "../../models";
class GoogleLocationInfoProvider {
private apiKey: string;
private apiUrl: string;
constructor() {
this.apiKey = $geocode.google.apiKey
this.apiUrl = $geocode.google.url
}
private parseAddressComponents(response: {
results: {
address_components: {
long_name: string;
short_name: string;
types: string[];
}[];
}[]
}): {
address?: string;
locality?: string;
postalCode?: string;
country?: string;
description?: string;
} {
const result = {
address: undefined,
locality: undefined,
postalCode: undefined,
country: undefined,
description: undefined,
} as {
address?: string;
locality?: string;
postalCode?: string;
country?: string;
description?: string;
};
if (!response.results?.length) {
return result;
}
const addressComponents = response.results[0].address_components.reduce(
(acc: any, component: any) => {
['street_number', 'route', 'locality', 'postal_code'].forEach((type) => {
if (component.types.includes(type)) {
acc[type] = component.long_name;
}
});
if (component.types.includes('country')) {
acc.country = component.short_name.toLowerCase();
}
return acc;
},
{},
);
if (addressComponents.route) {
result.address = (
(addressComponents.route || '') +
(addressComponents.street_number ? ' ' + addressComponents.street_number : '')
).trim();
if (!result.address?.length) {
result.address = undefined;
}
}
['locality', 'postal_code', 'country'].forEach((key) => {
if (addressComponents[key]) {
// @ts-expect-error
result[key] = addressComponents[key];
}
});
return result;
}
async getLocationInfo(location: GPSPoint): Promise<GPSPoint> {
const response = await fetch(
`${this.apiUrl}?latlng=${location.latitude},${location.longitude}&key=${this.apiKey}`,
);
if (!response.ok) {
throw new Error(`Error fetching location info: ${response.statusText}`);
}
const data = await response.json();
const addressComponents = this.parseAddressComponents(data);
return new GPSPoint({
...location,
address: location.address || addressComponents.address,
locality: location.locality || addressComponents.locality,
postalCode: location.postalCode || addressComponents.postalCode,
country: location.country || addressComponents.country,
})
}
}
export default GoogleLocationInfoProvider;

View file

@ -0,0 +1,24 @@
import { GPSPoint } from "~/models";
import { useGlobals } from '../../globals';
import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider";
import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider";
useGlobals();
abstract class LocationInfoProvider {
// TODO Cache location info
abstract getLocationInfo: (location: GPSPoint) => Promise<GPSPoint>;
static get(): LocationInfoProvider | undefined {
switch ($geocode.provider) {
case 'nominatim':
return new NominatimLocationInfoProvider();
case 'google':
return new GoogleLocationInfoProvider();
}
return undefined;
};
}
export default LocationInfoProvider;

View file

@ -0,0 +1,54 @@
import { GPSPoint } from "../../models";
class NominatimLocationInfoProvider {
private apiUrl: string;
private userAgent: string;
constructor() {
this.apiUrl = $geocode.nominatim.url;
this.userAgent = $geocode.nominatim.userAgent;
}
async getLocationInfo(location: GPSPoint): Promise<GPSPoint> {
const response = await fetch(
`${this.apiUrl}/reverse?lat=${location.latitude}&lon=${location.longitude}&format=json`,
{
headers: {
'User-Agent': this.userAgent,
},
}
);
if (!response.ok) {
throw new Error(`Error fetching location info: ${response.statusText}`);
}
const data = await response.json();
const address = data.address || {};
if (Object.keys(address).length > 0) {
let addressString: string | undefined = (
(address.road || '') + (
address.house_number ? (' ' + address.house_number) : ''
)
).trim();
if (!addressString?.length) {
addressString = undefined;
}
return new GPSPoint({
...location,
description: location.description || address.amenity,
address: addressString,
locality: address.city || address.town || address.village,
postalCode: address.postcode,
country: address.country_code,
})
}
return location;
}
}
export default NominatimLocationInfoProvider;

View file

@ -0,0 +1,9 @@
import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider";
import LocationInfoProvider from "./LocationInfoProvider";
import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider";
export {
GoogleLocationInfoProvider,
LocationInfoProvider,
NominatimLocationInfoProvider,
};

View file

@ -1,6 +1,7 @@
import dotenv from 'dotenv';
import { Db } from './db';
import Geocode from './config/Geocode';
import Secrets from './Secrets';
import Repositories from './repos';
@ -10,10 +11,12 @@ declare global {
var $db: Db;
var $repos: Repositories;
var $secrets: Secrets;
var $geocode: Geocode;
}
export function useGlobals() {
globalThis.$secrets = Secrets.fromEnv();
globalThis.$db = Db.fromEnv();
globalThis.$geocode = Geocode.fromEnv();
globalThis.$repos = new Repositories();
}

View file

@ -2,6 +2,7 @@ import { Request, Response } from 'express';
import { authenticate } from '../../../auth';
import { AuthInfo } from '../../../auth';
import { LocationInfoProvider } from '../../../ext/location';
import { LocationRequest } from '../../../requests';
import { Optional } from '../../../types';
import { GPSPoint, RoleName } from '../../../models';
@ -22,6 +23,30 @@ class GPSData extends ApiV1Route {
}
};
private enrichWithLocationInfo = async (gpsData: GPSPoint[]) => {
const provider = LocationInfoProvider.get();
if (!provider) {
return gpsData;
}
return await Promise.all(
gpsData
.map(async (point) => {
// Only enrich points that have latitude and longitude, but no
// location info
if (
!(point.latitude && point.longitude) ||
(point.country && point.locality && point.address)
) {
return point;
}
const locationInfo = await provider.getLocationInfo(point);
return { ...point, ...locationInfo };
})
);
}
@authenticate()
get = async (req: Request, res: Response, auth: Optional<AuthInfo>) => {
let query: LocationRequest
@ -44,7 +69,10 @@ class GPSData extends ApiV1Route {
post = async (req: Request, res: Response, auth: Optional<AuthInfo>) => {
const deviceIds = req.body.map((p: any) => p.deviceId).filter((d: any) => !!d);
this.validateOwnership(deviceIds, auth!);
await $repos.location.createPoints(req.body);
const points = await this.enrichWithLocationInfo(req.body as GPSPoint[]);
console.log(`Storing ${points.length} location point${points.length > 1 ? 's' : ''}`);
await $repos.location.createPoints(points);
res.status(201).send();
}

View file

@ -0,0 +1,43 @@
import { Request, Response } from 'express';
import { authenticate } from '../../../auth';
import { GPSPoint } from '../../../models';
import { LocationInfoProvider } from '../../../ext/location';
import ApiV1Route from './Route';
class LocationInfo extends ApiV1Route {
private provider: LocationInfoProvider | undefined;
constructor() {
super('/location-info');
this.provider = LocationInfoProvider.get();
}
@authenticate()
get = async (req: Request, res: Response) => {
if (!this.provider) {
res.status(500).send('Location info provider not configured');
return;
}
let location: GPSPoint;
try {
location = new GPSPoint(req.query);
if (!(location?.latitude && location?.longitude)) {
res.status(400).send('Invalid GPS coordinates');
return;
}
} catch (error) {
const e = `Error parsing location request: ${error}`;
console.warn(e);
res.status(400).send(e);
return;
}
location = await this.provider.getLocationInfo(location);
res.json(location);
}
}
export default LocationInfo;

View file

@ -2,6 +2,7 @@ import Auth from "./Auth";
import Devices from "./Devices";
import DevicesById from "./DevicesById";
import GPSData from "./GPSData";
import LocationInfo from "./LocationInfo";
import Routes from "../../Routes";
import Stats from "./Stats";
import Tokens from "./Tokens";
@ -14,6 +15,7 @@ class ApiV1Routes extends Routes {
new Devices(),
new DevicesById(),
new GPSData(),
new LocationInfo(),
new Stats(),
new Tokens(),
new TokensById(),