- Adds `GET /location-info?latitude=<...>&longitude=<...>` endpoint to get information about a point at the given coordinates. - Adds `nominatim` and `google` as reverse geocode providers. - Adds support for automatic enrichment upon ingestion.
This commit is contained in:
parent
d52fed5a57
commit
9ce57b23cd
12 changed files with 409 additions and 9 deletions
29
.env.example
29
.env.example
|
@ -48,6 +48,35 @@ ADMIN_EMAIL=admin@example.com
|
|||
# if you want to use a different database for the location data.
|
||||
# DB_LOCATION_TABLE=location_history
|
||||
|
||||
|
||||
###
|
||||
### Geocode configuration
|
||||
###
|
||||
|
||||
# Specify the geocode provider to use. The default is none (empty).
|
||||
# If set, then, upon ingestion, any points with missing address metadata
|
||||
# (address, locality, country, postal code) will be geocoded using the Google
|
||||
# Maps API or the Nominatim API and the metadata will be updated in the
|
||||
# database.
|
||||
# The available options are:
|
||||
# - "nominatim"
|
||||
# - "google" (requires GOOGLE_API_KEY to be set)
|
||||
# GEOCODE_PROVIDER=nominatim
|
||||
|
||||
# Specify the Nominatim API URL to use for geocoding and reverse geocoding
|
||||
# if you have set GEOCODE_PROVIDER to "nominatim". The default one
|
||||
# (https://nominatim.openstreetmap.org) will be used if not set, but keep in
|
||||
# mind that it is rate-limited to 1 request per second.
|
||||
# NOMINATIM_API_URL=https://nominatim.openstreetmap.org
|
||||
|
||||
# User agent to use for the Nominatim API. The default one is
|
||||
# "Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)"
|
||||
# NOMINATIM_USER_AGENT=YourUserAgent
|
||||
|
||||
# Specify a Google API key to use the Google Maps API for geocoding and reverse
|
||||
# geocoding, if you have set GEOCODE_PROVIDER to "google".
|
||||
# GOOGLE_API_KEY=your_google_api_key
|
||||
|
||||
###
|
||||
### Location history table column mappings.
|
||||
### The following settings are only taken into account when you use a different database
|
||||
|
|
28
README.md
28
README.md
|
@ -18,6 +18,7 @@
|
|||
- [Usage](#usage)
|
||||
* [Initial setup](#initial-setup)
|
||||
* [Ingestion](#ingestion)
|
||||
+ [Enriching location data](#enriching-location-data)
|
||||
* [External data sources](#external-data-sources)
|
||||
- [Development](#development)
|
||||
* [Compile and Hot-Reload for Development](#compile-and-hot-reload-for-development)
|
||||
|
@ -173,6 +174,33 @@ Or, for more advanced use cases, you can use a general-purpose application like
|
|||
to send data to the endpoint, or decouple the ingestion from the frontend by
|
||||
using an intermediate MQTT or Kafka broker.
|
||||
|
||||
#### Enriching location data
|
||||
|
||||
If the ingested location data does not contain the `address`, `locality`,
|
||||
`country` or `postalCode` fields, and you have set the `GEOCODE_PROVIDER`
|
||||
environment variable, then the application will try to geocode the location
|
||||
upon ingestion using the configured geocoding provider. Supported providers:
|
||||
|
||||
- [`nominatim`](https://nominatim.org/)
|
||||
- It uses OpenStreetMap data ([usage
|
||||
policy](https://operations.osmfoundation.org/policies/nominatim/)).
|
||||
- It doesn't require an API key, but it is rate-limited to 1 request per
|
||||
second (not advised if you are ingesting bulk data if you use the
|
||||
default `NOMINATIM_URL` instance).
|
||||
- It supports a custom `NOMINATIM_URL` environment variable to use a custom
|
||||
Nominatim instance.
|
||||
- [`google`](https://developers.google.com/maps/documentation/geocoding/start)
|
||||
- It requires a Google Maps API key, set in the `GOOGLE_API_KEY` environment
|
||||
variable.
|
||||
- See [additional usage
|
||||
limits](https://developers.google.com/maps/documentation/geocoding/usage-and-billing)
|
||||
for details. You can set your own usage limits in the Google Cloud
|
||||
console, but keep in mind that above a certain threshold you will be
|
||||
charged.
|
||||
|
||||
If `GEOCODE_PROVIDER` is not set, the application will not attempt to geocode
|
||||
the location data upon ingestion.
|
||||
|
||||
### External data sources
|
||||
|
||||
By default, the application will store the GPS data under the configured
|
||||
|
|
|
@ -2,15 +2,18 @@ class Secrets {
|
|||
public readonly serverKey: string;
|
||||
public readonly adminPassword: string;
|
||||
public readonly adminEmail: string;
|
||||
public readonly googleApiKey?: string;
|
||||
|
||||
private constructor({
|
||||
serverKey,
|
||||
adminPassword,
|
||||
adminEmail,
|
||||
}: any) {
|
||||
this.serverKey = serverKey;
|
||||
this.adminPassword = adminPassword;
|
||||
this.adminEmail = adminEmail;
|
||||
private constructor(args: {
|
||||
serverKey: string;
|
||||
adminPassword: string;
|
||||
adminEmail: string;
|
||||
googleApiKey?: string;
|
||||
}) {
|
||||
this.serverKey = args.serverKey;
|
||||
this.adminPassword = args.adminPassword;
|
||||
this.adminEmail = args.adminEmail;
|
||||
this.googleApiKey = args.googleApiKey;
|
||||
}
|
||||
|
||||
public static fromEnv(): Secrets {
|
||||
|
@ -33,6 +36,7 @@ class Secrets {
|
|||
serverKey: process.env.SERVER_KEY,
|
||||
adminPassword: process.env.ADMIN_PASSWORD,
|
||||
adminEmail: process.env.ADMIN_EMAIL,
|
||||
googleApiKey: process.env.GOOGLE_API_KEY,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
71
src/config/Geocode.ts
Normal file
71
src/config/Geocode.ts
Normal file
|
@ -0,0 +1,71 @@
|
|||
import { ValidationError } from '../errors';
|
||||
|
||||
type GeocodeProvider = 'google' | 'nominatim';
|
||||
|
||||
class NominatimConfig {
|
||||
public readonly url: string;
|
||||
public readonly userAgent: string;
|
||||
|
||||
private constructor(args: { url: string; userAgent: string }) {
|
||||
this.url = args.url;
|
||||
this.userAgent = args.userAgent;
|
||||
}
|
||||
|
||||
public static fromEnv(): NominatimConfig {
|
||||
return new NominatimConfig({
|
||||
url: process.env.NOMINATIM_URL || 'https://nominatim.openstreetmap.org',
|
||||
userAgent: process.env.NOMINATIM_USER_AGENT || 'Mozilla/5.0 (compatible; gpstracker/1.0; +https://github.com/blacklight/gpstracker)',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class GoogleConfig {
|
||||
public readonly url: string;
|
||||
public readonly apiKey: string;
|
||||
|
||||
private constructor(args: { apiKey: string }) {
|
||||
this.url = 'https://maps.googleapis.com/maps/api/geocode/json'
|
||||
this.apiKey = args.apiKey;
|
||||
}
|
||||
|
||||
public static fromEnv(): GoogleConfig {
|
||||
return new GoogleConfig({
|
||||
apiKey: process.env.GOOGLE_API_KEY || '',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class Geocode {
|
||||
public readonly provider?: GeocodeProvider;
|
||||
public readonly nominatim: NominatimConfig;
|
||||
public readonly google: GoogleConfig;
|
||||
|
||||
private constructor(args: {
|
||||
provider?: GeocodeProvider;
|
||||
nominatim: NominatimConfig;
|
||||
google: GoogleConfig;
|
||||
}) {
|
||||
this.provider = args.provider;
|
||||
this.nominatim = args.nominatim;
|
||||
this.google = args.google;
|
||||
|
||||
if (this.provider === 'google' && !this.google.apiKey) {
|
||||
throw new ValidationError('Google API key is required when using Google geocoding.');
|
||||
}
|
||||
}
|
||||
|
||||
public static fromEnv(): Geocode {
|
||||
const provider = process.env.GEOCODE_PROVIDER as GeocodeProvider | undefined;
|
||||
if (provider?.length && provider !== 'google' && provider !== 'nominatim') {
|
||||
throw new ValidationError('GEOCODE_PROVIDER must be either "google" or "nominatim".');
|
||||
}
|
||||
|
||||
return new Geocode({
|
||||
provider,
|
||||
nominatim: NominatimConfig.fromEnv(),
|
||||
google: GoogleConfig.fromEnv(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export default Geocode;
|
105
src/ext/location/GoogleLocationInfoProvider.ts
Normal file
105
src/ext/location/GoogleLocationInfoProvider.ts
Normal file
|
@ -0,0 +1,105 @@
|
|||
import { GPSPoint } from "../../models";
|
||||
|
||||
class GoogleLocationInfoProvider {
|
||||
private apiKey: string;
|
||||
private apiUrl: string;
|
||||
|
||||
constructor() {
|
||||
this.apiKey = $geocode.google.apiKey
|
||||
this.apiUrl = $geocode.google.url
|
||||
}
|
||||
|
||||
private parseAddressComponents(response: {
|
||||
results: {
|
||||
address_components: {
|
||||
long_name: string;
|
||||
short_name: string;
|
||||
types: string[];
|
||||
}[];
|
||||
}[]
|
||||
}): {
|
||||
address?: string;
|
||||
locality?: string;
|
||||
postalCode?: string;
|
||||
country?: string;
|
||||
description?: string;
|
||||
} {
|
||||
const result = {
|
||||
address: undefined,
|
||||
locality: undefined,
|
||||
postalCode: undefined,
|
||||
country: undefined,
|
||||
description: undefined,
|
||||
} as {
|
||||
address?: string;
|
||||
locality?: string;
|
||||
postalCode?: string;
|
||||
country?: string;
|
||||
description?: string;
|
||||
};
|
||||
|
||||
if (!response.results?.length) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const addressComponents = response.results[0].address_components.reduce(
|
||||
(acc: any, component: any) => {
|
||||
['street_number', 'route', 'locality', 'postal_code'].forEach((type) => {
|
||||
if (component.types.includes(type)) {
|
||||
acc[type] = component.long_name;
|
||||
}
|
||||
});
|
||||
|
||||
if (component.types.includes('country')) {
|
||||
acc.country = component.short_name.toLowerCase();
|
||||
}
|
||||
|
||||
return acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
|
||||
if (addressComponents.route) {
|
||||
result.address = (
|
||||
(addressComponents.route || '') +
|
||||
(addressComponents.street_number ? ' ' + addressComponents.street_number : '')
|
||||
).trim();
|
||||
|
||||
if (!result.address?.length) {
|
||||
result.address = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
['locality', 'postal_code', 'country'].forEach((key) => {
|
||||
if (addressComponents[key]) {
|
||||
// @ts-expect-error
|
||||
result[key] = addressComponents[key];
|
||||
}
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async getLocationInfo(location: GPSPoint): Promise<GPSPoint> {
|
||||
const response = await fetch(
|
||||
`${this.apiUrl}?latlng=${location.latitude},${location.longitude}&key=${this.apiKey}`,
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Error fetching location info: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const addressComponents = this.parseAddressComponents(data);
|
||||
|
||||
return new GPSPoint({
|
||||
...location,
|
||||
address: location.address || addressComponents.address,
|
||||
locality: location.locality || addressComponents.locality,
|
||||
postalCode: location.postalCode || addressComponents.postalCode,
|
||||
country: location.country || addressComponents.country,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default GoogleLocationInfoProvider;
|
24
src/ext/location/LocationInfoProvider.ts
Normal file
24
src/ext/location/LocationInfoProvider.ts
Normal file
|
@ -0,0 +1,24 @@
|
|||
import { GPSPoint } from "~/models";
|
||||
import { useGlobals } from '../../globals';
|
||||
import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider";
|
||||
import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider";
|
||||
|
||||
useGlobals();
|
||||
|
||||
abstract class LocationInfoProvider {
|
||||
// TODO Cache location info
|
||||
abstract getLocationInfo: (location: GPSPoint) => Promise<GPSPoint>;
|
||||
|
||||
static get(): LocationInfoProvider | undefined {
|
||||
switch ($geocode.provider) {
|
||||
case 'nominatim':
|
||||
return new NominatimLocationInfoProvider();
|
||||
case 'google':
|
||||
return new GoogleLocationInfoProvider();
|
||||
}
|
||||
|
||||
return undefined;
|
||||
};
|
||||
}
|
||||
|
||||
export default LocationInfoProvider;
|
54
src/ext/location/NominatimLocationInfoProvider.ts
Normal file
54
src/ext/location/NominatimLocationInfoProvider.ts
Normal file
|
@ -0,0 +1,54 @@
|
|||
import { GPSPoint } from "../../models";
|
||||
|
||||
class NominatimLocationInfoProvider {
|
||||
private apiUrl: string;
|
||||
private userAgent: string;
|
||||
|
||||
constructor() {
|
||||
this.apiUrl = $geocode.nominatim.url;
|
||||
this.userAgent = $geocode.nominatim.userAgent;
|
||||
}
|
||||
|
||||
async getLocationInfo(location: GPSPoint): Promise<GPSPoint> {
|
||||
const response = await fetch(
|
||||
`${this.apiUrl}/reverse?lat=${location.latitude}&lon=${location.longitude}&format=json`,
|
||||
{
|
||||
headers: {
|
||||
'User-Agent': this.userAgent,
|
||||
},
|
||||
}
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Error fetching location info: ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const address = data.address || {};
|
||||
|
||||
if (Object.keys(address).length > 0) {
|
||||
let addressString: string | undefined = (
|
||||
(address.road || '') + (
|
||||
address.house_number ? (' ' + address.house_number) : ''
|
||||
)
|
||||
).trim();
|
||||
|
||||
if (!addressString?.length) {
|
||||
addressString = undefined;
|
||||
}
|
||||
|
||||
return new GPSPoint({
|
||||
...location,
|
||||
description: location.description || address.amenity,
|
||||
address: addressString,
|
||||
locality: address.city || address.town || address.village,
|
||||
postalCode: address.postcode,
|
||||
country: address.country_code,
|
||||
})
|
||||
}
|
||||
|
||||
return location;
|
||||
}
|
||||
}
|
||||
|
||||
export default NominatimLocationInfoProvider;
|
9
src/ext/location/index.ts
Normal file
9
src/ext/location/index.ts
Normal file
|
@ -0,0 +1,9 @@
|
|||
import GoogleLocationInfoProvider from "./GoogleLocationInfoProvider";
|
||||
import LocationInfoProvider from "./LocationInfoProvider";
|
||||
import NominatimLocationInfoProvider from "./NominatimLocationInfoProvider";
|
||||
|
||||
export {
|
||||
GoogleLocationInfoProvider,
|
||||
LocationInfoProvider,
|
||||
NominatimLocationInfoProvider,
|
||||
};
|
|
@ -1,6 +1,7 @@
|
|||
import dotenv from 'dotenv';
|
||||
|
||||
import { Db } from './db';
|
||||
import Geocode from './config/Geocode';
|
||||
import Secrets from './Secrets';
|
||||
import Repositories from './repos';
|
||||
|
||||
|
@ -10,10 +11,12 @@ declare global {
|
|||
var $db: Db;
|
||||
var $repos: Repositories;
|
||||
var $secrets: Secrets;
|
||||
var $geocode: Geocode;
|
||||
}
|
||||
|
||||
export function useGlobals() {
|
||||
globalThis.$secrets = Secrets.fromEnv();
|
||||
globalThis.$db = Db.fromEnv();
|
||||
globalThis.$geocode = Geocode.fromEnv();
|
||||
globalThis.$repos = new Repositories();
|
||||
}
|
||||
|
|
|
@ -2,6 +2,7 @@ import { Request, Response } from 'express';
|
|||
|
||||
import { authenticate } from '../../../auth';
|
||||
import { AuthInfo } from '../../../auth';
|
||||
import { LocationInfoProvider } from '../../../ext/location';
|
||||
import { LocationRequest } from '../../../requests';
|
||||
import { Optional } from '../../../types';
|
||||
import { GPSPoint, RoleName } from '../../../models';
|
||||
|
@ -22,6 +23,30 @@ class GPSData extends ApiV1Route {
|
|||
}
|
||||
};
|
||||
|
||||
private enrichWithLocationInfo = async (gpsData: GPSPoint[]) => {
|
||||
const provider = LocationInfoProvider.get();
|
||||
if (!provider) {
|
||||
return gpsData;
|
||||
}
|
||||
|
||||
return await Promise.all(
|
||||
gpsData
|
||||
.map(async (point) => {
|
||||
// Only enrich points that have latitude and longitude, but no
|
||||
// location info
|
||||
if (
|
||||
!(point.latitude && point.longitude) ||
|
||||
(point.country && point.locality && point.address)
|
||||
) {
|
||||
return point;
|
||||
}
|
||||
|
||||
const locationInfo = await provider.getLocationInfo(point);
|
||||
return { ...point, ...locationInfo };
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
@authenticate()
|
||||
get = async (req: Request, res: Response, auth: Optional<AuthInfo>) => {
|
||||
let query: LocationRequest
|
||||
|
@ -44,7 +69,10 @@ class GPSData extends ApiV1Route {
|
|||
post = async (req: Request, res: Response, auth: Optional<AuthInfo>) => {
|
||||
const deviceIds = req.body.map((p: any) => p.deviceId).filter((d: any) => !!d);
|
||||
this.validateOwnership(deviceIds, auth!);
|
||||
await $repos.location.createPoints(req.body);
|
||||
|
||||
const points = await this.enrichWithLocationInfo(req.body as GPSPoint[]);
|
||||
console.log(`Storing ${points.length} location point${points.length > 1 ? 's' : ''}`);
|
||||
await $repos.location.createPoints(points);
|
||||
res.status(201).send();
|
||||
}
|
||||
|
||||
|
|
43
src/routes/api/v1/LocationInfo.ts
Normal file
43
src/routes/api/v1/LocationInfo.ts
Normal file
|
@ -0,0 +1,43 @@
|
|||
import { Request, Response } from 'express';
|
||||
|
||||
import { authenticate } from '../../../auth';
|
||||
import { GPSPoint } from '../../../models';
|
||||
import { LocationInfoProvider } from '../../../ext/location';
|
||||
import ApiV1Route from './Route';
|
||||
|
||||
class LocationInfo extends ApiV1Route {
|
||||
private provider: LocationInfoProvider | undefined;
|
||||
|
||||
constructor() {
|
||||
super('/location-info');
|
||||
this.provider = LocationInfoProvider.get();
|
||||
}
|
||||
|
||||
@authenticate()
|
||||
get = async (req: Request, res: Response) => {
|
||||
if (!this.provider) {
|
||||
res.status(500).send('Location info provider not configured');
|
||||
return;
|
||||
}
|
||||
|
||||
let location: GPSPoint;
|
||||
|
||||
try {
|
||||
location = new GPSPoint(req.query);
|
||||
if (!(location?.latitude && location?.longitude)) {
|
||||
res.status(400).send('Invalid GPS coordinates');
|
||||
return;
|
||||
}
|
||||
} catch (error) {
|
||||
const e = `Error parsing location request: ${error}`;
|
||||
console.warn(e);
|
||||
res.status(400).send(e);
|
||||
return;
|
||||
}
|
||||
|
||||
location = await this.provider.getLocationInfo(location);
|
||||
res.json(location);
|
||||
}
|
||||
}
|
||||
|
||||
export default LocationInfo;
|
|
@ -2,6 +2,7 @@ import Auth from "./Auth";
|
|||
import Devices from "./Devices";
|
||||
import DevicesById from "./DevicesById";
|
||||
import GPSData from "./GPSData";
|
||||
import LocationInfo from "./LocationInfo";
|
||||
import Routes from "../../Routes";
|
||||
import Stats from "./Stats";
|
||||
import Tokens from "./Tokens";
|
||||
|
@ -14,6 +15,7 @@ class ApiV1Routes extends Routes {
|
|||
new Devices(),
|
||||
new DevicesById(),
|
||||
new GPSData(),
|
||||
new LocationInfo(),
|
||||
new Stats(),
|
||||
new Tokens(),
|
||||
new TokensById(),
|
||||
|
|
Loading…
Add table
Reference in a new issue