Skip to content

Commit e71fd0b

Browse files
NdpntNdpnt
authored
Expose collection metadata through the collection API (#1123)
2 parents fc068e2 + 21348eb commit e71fd0b

34 files changed

+429
-119
lines changed

‎CHANGELOG.md

+12
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,18 @@
22

33
All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
44

5+
## Unreleased [major]
6+
7+
> Development of this release was supported by the [French Ministry for Foreign Affairs](https://www.diplomatie.gouv.fr/fr/politique-etrangere-de-la-france/diplomatie-numerique/) through its ministerial [State Startups incubator](https://beta.gouv.fr/startups/open-terms-archive.html) under the aegis of the Ambassador for Digital Affairs.
8+
9+
### Added
10+
11+
- Expose collection metadata through the collection API; requires a [metadata file](https://docs.opentermsarchive.org/collections/metadata/) at the root of your collection folder
12+
13+
### Changed
14+
15+
- **Breaking:** Replace `@opentermsarchive/engine.services.declarationsPath` with `@opentermsarchive/engine.collectionPath`; ensure your declarations are located in `./declarations` in your collection folder
16+
517
## 3.0.0 - 2024-12-03
618

719
_Full changeset and discussions: [#1122](https://github.com/OpenTermsArchive/engine/pull/1122)._

‎config/ci.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
{
22
"@opentermsarchive/engine": {
3-
"services": {
4-
"declarationsPath": "./demo-declarations/declarations"
5-
}
3+
"collectionPath": "./demo-declarations/"
64
}
75
}

‎config/default.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
{
22
"@opentermsarchive/engine": {
33
"trackingSchedule": "30 */12 * * *",
4-
"services": {
5-
"declarationsPath": "./declarations"
6-
},
4+
"collectionPath": "./",
75
"recorder": {
86
"versions": {
97
"storage": {

‎config/test.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
{
22
"@opentermsarchive/engine": {
3-
"services": {
4-
"declarationsPath": "./test/services"
5-
},
3+
"collectionPath": "./test/test-declarations",
64
"recorder": {
75
"versions": {
86
"storage": {

‎package-lock.json

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@
8282
"https-proxy-agent": "^5.0.0",
8383
"iconv-lite": "^0.6.3",
8484
"joplin-turndown-plugin-gfm": "^1.0.12",
85+
"js-yaml": "^4.1.0",
8586
"jsdom": "^18.1.0",
8687
"json-source-map": "^0.6.1",
8788
"lodash": "^4.17.21",

‎scripts/declarations/lint/index.mocha.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@ const ESLINT_CONFIG_PATH = path.join(ROOT_PATH, '.eslintrc.yaml');
1616
const eslint = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: false });
1717
const eslintWithFix = new ESLint({ overrideConfigFile: ESLINT_CONFIG_PATH, fix: true });
1818

19-
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
20-
const instancePath = path.resolve(declarationsPath, '../');
19+
const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
20+
const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
2121

2222
export default async options => {
2323
let servicesToValidate = options.services || [];

‎scripts/declarations/validate/index.mocha.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ const fs = fsApi.promises;
1919
const MIN_DOC_LENGTH = 100;
2020
const SLOW_DOCUMENT_THRESHOLD = 10 * 1000; // number of milliseconds after which a document fetch is considered slow
2121

22-
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
23-
const instancePath = path.resolve(declarationsPath, '../');
22+
const instancePath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
23+
const declarationsPath = path.resolve(instancePath, services.DECLARATIONS_PATH);
2424

2525
export default async options => {
2626
const schemaOnly = options.schemaOnly || false;

‎src/archivist/services/index.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import fsApi from 'fs';
1+
import fs from 'fs/promises';
22
import path from 'path';
33
import { pathToFileURL } from 'url';
44

@@ -8,8 +8,8 @@ import Service from './service.js';
88
import SourceDocument from './sourceDocument.js';
99
import Terms from './terms.js';
1010

11-
const fs = fsApi.promises;
12-
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.services.declarationsPath'));
11+
export const DECLARATIONS_PATH = './declarations';
12+
const declarationsPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'), DECLARATIONS_PATH);
1313

1414
export async function load(servicesIdsToLoad = []) {
1515
let servicesIds = await getDeclaredServicesIds();

‎src/archivist/services/service.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,6 @@ export default class Service {
7070
}
7171

7272
static getNumberOfTerms(services, servicesIds, termsTypes) {
73-
return servicesIds.reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
73+
return (servicesIds || Object.keys(services)).reduce((acc, serviceId) => acc + services[serviceId].getNumberOfTerms(termsTypes), 0);
7474
}
7575
}

‎src/collection-api/routes/index.js

+12-2
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,17 @@
1+
import path from 'path';
2+
3+
import config from 'config';
14
import express from 'express';
25
import helmet from 'helmet';
36

7+
import * as Services from '../../archivist/services/index.js';
8+
49
import docsRouter from './docs.js';
10+
import metadataRouter from './metadata.js';
511
import servicesRouter from './services.js';
612
import versionsRouter from './versions.js';
713

8-
export default function apiRouter(basePath) {
14+
export default async function apiRouter(basePath) {
915
const router = express.Router();
1016

1117
const defaultDirectives = helmet.contentSecurityPolicy.getDefaultDirectives();
@@ -27,7 +33,11 @@ export default function apiRouter(basePath) {
2733
res.json({ message: 'Welcome to an instance of the Open Terms Archive API. Documentation is available at /docs. Learn more on Open Terms Archive on https://opentermsarchive.org.' });
2834
});
2935

30-
router.use(servicesRouter);
36+
const collectionPath = path.resolve(process.cwd(), config.get('@opentermsarchive/engine.collectionPath'));
37+
const services = await Services.load();
38+
39+
router.use(await metadataRouter(collectionPath, services));
40+
router.use(servicesRouter(services));
3141
router.use(versionsRouter);
3242

3343
return router;

‎src/collection-api/routes/metadata.js

+165
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
import fs from 'fs/promises';
2+
import path from 'path';
3+
4+
import express from 'express';
5+
import yaml from 'js-yaml';
6+
7+
import Service from '../../archivist/services/service.js';
8+
9+
const METADATA_FILENAME = 'metadata.yml';
10+
const PACKAGE_JSON_PATH = '../../../package.json';
11+
12+
/**
13+
* @swagger
14+
* tags:
15+
* name: Metadata
16+
* description: Collection metadata API
17+
* components:
18+
* schemas:
19+
* Metadata:
20+
* type: object
21+
* description: Collection metadata
22+
* properties:
23+
* id:
24+
* type: string
25+
* description: Unique identifier of the collection
26+
* name:
27+
* type: string
28+
* description: Display name of the collection
29+
* tagline:
30+
* type: string
31+
* description: Short description of the collection
32+
* description:
33+
* type: string
34+
* nullable: true
35+
* description: Detailed description of the collection
36+
* totalTerms:
37+
* type: integer
38+
* description: Total number of terms tracked in the collection
39+
* totalServices:
40+
* type: integer
41+
* description: Total number of services tracked in the collection
42+
* engineVersion:
43+
* type: string
44+
* description: Version of the Open Terms Archive engine in SemVer format (MAJOR.MINOR.PATCH)
45+
* dataset:
46+
* type: string
47+
* format: uri
48+
* description: URL to the dataset releases
49+
* declarations:
50+
* type: string
51+
* format: uri
52+
* description: URL to the declarations repository
53+
* versions:
54+
* type: string
55+
* format: uri
56+
* description: URL to the versions repository
57+
* snapshots:
58+
* type: string
59+
* format: uri
60+
* description: URL to the snapshots repository
61+
* logo:
62+
* type: string
63+
* format: uri
64+
* nullable: true
65+
* description: URL to the collection logo
66+
* languages:
67+
* type: array
68+
* items:
69+
* type: string
70+
* description: List of ISO 639 language codes representing languages allowed by the collection
71+
* jurisdictions:
72+
* type: array
73+
* items:
74+
* type: string
75+
* description: List of ISO 3166-2 country codes representing jurisdictions covered by the collection
76+
* trackingPeriods:
77+
* type: array
78+
* items:
79+
* type: object
80+
* properties:
81+
* startDate:
82+
* type: string
83+
* format: date
84+
* description: The date when tracking started for this period
85+
* schedule:
86+
* type: string
87+
* description: A cron expression defining when terms are tracked (e.g. "0 0 * * *" for daily at midnight)
88+
* serverLocation:
89+
* type: string
90+
* description: The geographic location of the server used for tracking
91+
* endDate:
92+
* type: string
93+
* format: date
94+
* description: The date when tracking ended for this period
95+
* governance:
96+
* type: object
97+
* properties:
98+
* hosts:
99+
* type: array
100+
* items:
101+
* $ref: '#/components/schemas/Organization'
102+
* administrators:
103+
* type: array
104+
* items:
105+
* $ref: '#/components/schemas/Organization'
106+
* curators:
107+
* type: array
108+
* items:
109+
* $ref: '#/components/schemas/Organization'
110+
* maintainers:
111+
* type: array
112+
* items:
113+
* $ref: '#/components/schemas/Organization'
114+
* sponsors:
115+
* type: array
116+
* items:
117+
* $ref: '#/components/schemas/Organization'
118+
* Organization:
119+
* type: object
120+
* properties:
121+
* name:
122+
* type: string
123+
* description: Name of the organization
124+
* url:
125+
* type: string
126+
* format: uri
127+
* description: URL to the organization's website
128+
* logo:
129+
* type: string
130+
* format: uri
131+
* description: URL to the organization's logo
132+
*/
133+
export default async function metadataRouter(collectionPath, services) {
134+
const router = express.Router();
135+
136+
const STATIC_METADATA = yaml.load(await fs.readFile(path.join(collectionPath, METADATA_FILENAME), 'utf8'));
137+
const { version: engineVersion } = JSON.parse(await fs.readFile(new URL(PACKAGE_JSON_PATH, import.meta.url)));
138+
139+
/**
140+
* @swagger
141+
* /metadata:
142+
* get:
143+
* summary: Get collection metadata
144+
* tags: [Metadata]
145+
* produces:
146+
* - application/json
147+
* responses:
148+
* 200:
149+
* description: Collection metadata
150+
*/
151+
router.get('/metadata', (req, res) => {
152+
const dynamicMetadata = {
153+
totalServices: Object.keys(services).length,
154+
totalTerms: Service.getNumberOfTerms(services),
155+
engineVersion,
156+
};
157+
158+
res.json({
159+
...STATIC_METADATA,
160+
...dynamicMetadata,
161+
});
162+
});
163+
164+
return router;
165+
}

‎src/collection-api/routes/metadata.test.js

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import fs from 'fs/promises';
2+
3+
import { expect } from 'chai';
4+
import config from 'config';
5+
import request from 'supertest';
6+
7+
import app from '../server.js';
8+
9+
const basePath = config.get('@opentermsarchive/engine.collection-api.basePath');
10+
const { version: engineVersion } = JSON.parse(await fs.readFile(new URL('../../../package.json', import.meta.url)));
11+
12+
const EXPECTED_RESPONSE = {
13+
totalServices: 7,
14+
totalTerms: 8,
15+
id: 'test',
16+
name: 'test',
17+
tagline: 'Test collection',
18+
description: 'This is a test collection used for testing purposes.',
19+
dataset: 'https://github.com/OpenTermsArchive/test-versions/releases',
20+
declarations: 'https://github.com/OpenTermsArchive/test-declarations',
21+
versions: 'https://github.com/OpenTermsArchive/test-versions',
22+
snapshots: 'https://github.com/OpenTermsArchive/test-snapshots',
23+
donations: null,
24+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
25+
languages: [
26+
'en',
27+
],
28+
jurisdictions: [
29+
'EU',
30+
],
31+
governance: {
32+
hosts: [
33+
{ name: 'Localhost' },
34+
],
35+
administrators: [
36+
{
37+
name: 'Open Terms Archive',
38+
url: 'https://opentermsarchive.org/',
39+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
40+
},
41+
],
42+
curators: [
43+
{
44+
name: 'Open Terms Archive',
45+
url: 'https://opentermsarchive.org/',
46+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
47+
},
48+
],
49+
maintainers: [
50+
{
51+
name: 'Open Terms Archive',
52+
url: 'https://opentermsarchive.org/',
53+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
54+
},
55+
],
56+
sponsors: [
57+
{
58+
name: 'Open Terms Archive',
59+
url: 'https://opentermsarchive.org/',
60+
logo: 'https://opentermsarchive.org/images/logo/logo-open-terms-archive-black.png',
61+
},
62+
],
63+
},
64+
};
65+
66+
describe('Metadata API', () => {
67+
describe('GET /metadata', () => {
68+
let response;
69+
70+
before(async () => {
71+
response = await request(app).get(`${basePath}/v1/metadata`);
72+
});
73+
74+
it('responds with 200 status code', () => {
75+
expect(response.status).to.equal(200);
76+
});
77+
78+
it('responds with Content-Type application/json', () => {
79+
expect(response.type).to.equal('application/json');
80+
});
81+
82+
it('returns expected metadata object', () => {
83+
expect(response.body).to.deep.equal({
84+
...EXPECTED_RESPONSE,
85+
engineVersion,
86+
});
87+
});
88+
});
89+
});

0 commit comments

Comments
 (0)