diff --git a/fep/c551/fep-c551.md b/fep/c551/fep-c551.md new file mode 100644 index 0000000..21bd878 --- /dev/null +++ b/fep/c551/fep-c551.md @@ -0,0 +1,134 @@ +--- +slug: "c551" +authors: bengo +status: DRAFT +dateReceived: 2024-07-11 +trackingIssue: +discussionsTo: +--- + +# FEP-c551: Use ECMAScript Modules to Create Conformance Tests for Fediverse Enhancement Proposals + +## Summary + +This is a proposal to enhance the fediverse by creating test cases for FEPs as ECMAScript Modules. + + + +## Contents + +* [Context](#context) +* [Test Specifications](#test-specifications) +* [Test Modules](#test-modules) +* [Test Objects](#test-objects) +* [Test Functions](#test-functions) +* [Test Inputs](#test-inputs) +* [Test Results](#test-results) + + + +## Context + +[FEP-d9ad][] proposes to Create Conformance Tests for Fediverse Enhancement Proposals, and specifies components that all Conformance Tests may use and describe in their Test Specifications. It *does not* specify a format for implementing FEP-d9ad Conformance Tests in any programming language. + +This FEP-c551 proposes to supplement human-readable FEP-d9ad Conformance Tests with implementations of the test as [Test Objects](#test-objects) exported from [Test Modules](#test-modules). Each [Test Object][] has a `run` function parameterized by a [Test Input] and returning a Promise of a [Test Result][]. + +## Overview + +When a tester comes up with a new test for a FEP, they create a human-readable [Test Specification](#test-specifications) describing how to test whether some subject conforms to the FEP. + +ECMAScript developers implement Test Specifications as automatable code by using ECMAScript to create [Test Functions][] that execute the test logic and [Test Objects][] that group the Test Function with more info like the test's name, required input, and possible outcomes. Test Objects are distributed in ECMAScript Modules published on the web, e.g. in `.js` or `.mjs` files. + +Testers invoke the Test Function once for each Test Input, await any returned Promises, and receive a [Test Result][] describing the `outcome` of running the test. + +## Test Specifications + +Test Specifications are human-readable documents that specify the behavior of a test. + +Test Specifications SHOULD include Conformance Test Component specifications from [FEP-d9ad][]. + +An example of a test specification is [fep-521a-test-case.md](https://codeberg.org/fediverse/fep/src/branch/main/fep/521a/fep-521a-test-case.md). + +## Test Modules + +FEP testers MAY publish implementations of their test specifications as an [ECMAScript Module][] following the recommendations in this proposal. Such modules may be referred to as Test Modules. + +Test Modules SHOULD have no imports. This is to ensure portability of the test modules. + +Test Modules MAY export a default export object that is a Test Object + +Test Modules SHOULD be resilient to being parsed and evaluated in various ECMAScript runtimes (e.g. node.js or a web browser like Firefox). + +An example of a test module can be found [in activitypub-testing-fep-521a](https://codeberg.org/socialweb.coop/activitypub-testing-fep-521a/src/branch/main/fep/521a/actor-objects-must-express-signing-key-as-assertionMethod-multikey.js). + +### Example Test Module + +```javascript +export default { + name: 'invalid script module name', + run: (input) => ({ outcome: 'passed' }), + type: ['https://w3id.org/fep/c551#Test'], + '@context': ["https://www.w3.org/ns/activitystreams"], +} +``` + +## Test Objects + +Test Objects are ECMAScript Objects that represent a named, runnable test, e.g. a test specified by a [FEP-d9ad Conformance Test][]. + +Test Objects MUST have a property named `type` whose value is either the string `https://w3id.org/fep/c551#Test` or an Array containing that string. + +Test Objects MUST have a property named `name` whose value is a string. + +Test Objects SHOULD have a property named `@context` whose value is an Array containing `https://www.w3.org/ns/activitystreams`. + +An example of a test object is [exported in activitypub-testing-fep-521a](https://codeberg.org/socialweb.coop/activitypub-testing-fep-521a/src/commit/b6e49fd5f490b05f04a958f5f3c5c584e66f592b/fep/521a/actor-objects-must-express-signing-key-as-assertionMethod-multikey.js#L38). + +Test Objects SHOULD have a property named `run` whose value is a [Test Function][]]. + +## Test Functions + +Test Object `run` calls SHOULD return a `Promise` that resolves to a [Test Result][]. + +Test Object `run` functions SHOULD be resilient to being run in various ECMAScript runtimes (e.g. node.js or a web browser like Firefox). + +## Test Inputs + +A Test Input is the first parameter to a test's `run` function. + +Test Input MUST be an object. A test with several logically distinct inputs should give each input a name, and add each named input as a property within a top-level input object. + +Test Input values SHOULD conform to the specification of the called test's [Input](https://bengo.is/fep/d9ad/#input) spec. + +## Test Results + +[Test Results][] MUST have a property named `outcome` whose value is a string. + +Test Results SHOULD have a property named `info` whose value is a string. + +Test Results MAY have a property named `pointer` that contextualizes the `outcome`, e.g. an object with a property for each value that led to the `outcome`. For example, if a test outcome is `failed` because some number was too low, you can set the result `info` to "number too low" and `pointer` to `{ number: 100 }`. + + + +
+Conformance requirements are indicated by sentences containing MUST a la RFC2119. +
+ +## Copyright + +CC0 1.0 Universal (CC0 1.0) Public Domain Dedication + +To the extent possible under law, the authors of this Fediverse Enhancement Proposal have waived all copyright and related or neighboring rights to this work. + +[ECMAScript Module]: https://tc39.es/ecma262/#sec-modules +[Test Module]: #test-modules +[Test Modules]: #test-modules +[Test Inputs]: #test-inputs +[Test Result]: #test-results +[Test Results]: #test-results +[Test Object]: #test-objects +[Test Objects]: #test-objects +[Test Function]: #test-functions +[Test Functions]: #test-functions +[FEP-d9ad]: https://bengo.is/fep/d9ad/ +[FEP-d9ad Conformance Test]: https://bengo.is/fep/d9ad/ diff --git a/fep/c551/tests/.gitignore b/fep/c551/tests/.gitignore new file mode 100644 index 0000000..c925c21 --- /dev/null +++ b/fep/c551/tests/.gitignore @@ -0,0 +1,2 @@ +/dist +/node_modules diff --git a/fep/c551/tests/README.md b/fep/c551/tests/README.md new file mode 100644 index 0000000..b5a4827 --- /dev/null +++ b/fep/c551/tests/README.md @@ -0,0 +1,44 @@ +# fep-c551-tests + +A test suite that tests for conformance to [FEP-c551: Use ECMAScript Modules to Create Conformance Tests for Fediverse Enhancement Proposals][FEP-c551]. + +## Tests + +### fep-c551 module must export test object + +* slug: `fep-c551-module-must-export-test-object` +* [Test Module](./fep-c551-module-must-export-test-object.js) + +## Usage + +### Running local test files via Data URL + +```shell +activitypub-testing run test \ +--url="$(data-url ./fep-c551-module-must-export-test-object.js)" \ +--input.module="$(cat < { + return { outcome: 'inapplicable' } + } +} +EOF +)" +``` + +The `data-url` command is provided by the following shell function: + +```shell +data-url() { + if [ -z "$1" ]; then + echo "usage: data-url file" >&2 + exit 1 + fi + mimetype=$(file -bN --mime-type "$1") + content=$(base64 < "$1") + echo "data:$mimetype;base64,$content" +} +``` + +[FEP-c551]: https://codeberg.org/fediverse/fep/src/branch/main/fep/c551/fep-c551.md diff --git a/fep/c551/tests/fep-c551-module-must-export-test-object.js b/fep/c551/tests/fep-c551-module-must-export-test-object.js new file mode 100644 index 0000000..bfb2b7c --- /dev/null +++ b/fep/c551/tests/fep-c551-module-must-export-test-object.js @@ -0,0 +1,220 @@ +const name = 'fep-c551 module must export test object' +const slug = 'fep-c551-module-must-export-test-object' +const uuid = '14bab0ae-e682-4f4c-9474-ef65ca47d527' +const attributedTo = [ + 'https://bengo.is', +] + +/** ID URL for type of FEP-C551 Test */ +const fepC551TestType = 'https://w3id.org/fep/c551#Test' + +/** + * Expected input to the test rule. + * This will be checked for test applicability. + * If the test is applicable, the rule will derive a test target from the Input, + * then check expectations against the Target + * returning a result with Outcomes + * @typedef Input + * @property {unknown} module + */ + +/** + * The test will check expectations against test Target derived from Input + * @typedef Target + * @property {string} module + */ + +/** + * Outcome - every test Target has an outcome + * @typedef {"inapplicable"|"passed"|"failed"} Outcome + */ + +/** + * @typedef {any} TestResult + */ + +export default { + attributedTo, + testCases: [ + + { + name: 'valid actor - type value is array', + input: { + module: ` + export default { + type: ['https://w3id.org/fep/c551#Test'], + name: 'invalid script module name', + run: () => ({ outcome: 'passed' }), + '@context': ["https://www.w3.org/ns/activitystreams"], + }; + ` + }, + result: { + outcome: 'passed', + } + }, + + { + name: 'valid actor - type value is string', + input: { + module: ` + export default { + name: 'invalid script module name', + run: () => ({ outcome: 'passed' }), + type: 'https://w3id.org/fep/c551#Test', + '@context': ["https://www.w3.org/ns/activitystreams"], + }; + ` + }, + result: { + outcome: 'passed', + } + }, + + { + name: 'without run property', + input: { + module: ` + export default { + name: 'invalid script module name', + type: 'https://w3id.org/fep/c551#Test', + '@context': ["https://www.w3.org/ns/activitystreams"] + }; + ` + }, + result: { + outcome: 'failed', + } + }, + + { + name: 'with empty type array', + input: { + module: ` + export default { + name: 'invalid script module name', + run: () => ({ outcome: 'passed' }), + type: [], + '@context': ["https://www.w3.org/ns/activitystreams"], + }; + ` + }, + result: { + outcome: 'failed', + } + }, + ], + input: { + module: { + help: 'ECMAScript Module that exports a test object', + required: true, + } + }, + name, + run, + slug, + uuid, +} + +/** + * given test rule inputs, check for applicability. + * If the input does not pass test rule applicability requirements, + * return a result with outcome "inapplicable". + * (does some checks from 'Applicability' section of test rule) + * @param {Input} input + * @returns {{ outcome: "inapplicable", info: string } + * |{ module: string }} + */ +function checkApplicability(input) { + if (typeof input.module !== 'string') return { + outcome: "inapplicable", + info: 'applicability requires input.module MUST be a string' + } + return { + module: input.module, + } +} + +/** + * given test rule inputs, return test targets. + * (does some checks from 'Applicability' section of test rule) + * @param {Input & {console?:globalThis.console}} input + */ +function getTarget({ module, console = globalThis.console }) { + if (typeof module !== 'string') { + return { + result: { + outcome: 'inapplicable', + info: 'input.module MUST be a string', + } + } + } + + return { + targets: [{ module }] + } +} + +/** + * run expectations against target + * @param {Target} target + */ +async function expect({ module }) { + if (typeof module !== 'string') return { result: { outcome: 'failed', info: 'input.module MUST be a string' } } + const moduleUri = `data:text/javascript;charset=utf-8;base64,${btoa(module)}` + const test = await import(moduleUri).then(m => m.default) + if (typeof test?.run !== 'function') return { result: { outcome: 'failed', info: 'exported test.run MUST be a function', pointer: { run: test.run } } } + + // The default export MUST have a property named `@type` whose value is either the string `https://w3id.org/fep/c551#Test` or an Array containing that string. + const testTypeValues = Array.isArray(test.type) ? test.type : test.type || [] + if ( ! testTypeValues.includes(fepC551TestType)) { + return { + result: { + outcome: "failed", + info: "test must have type https://w3id.org/fep/c551#Test", + pointer: { + type: test.type + } + } + } + } + + return { result: { outcome: "passed" } } +} + +/** + * @param {Input} input + */ +async function run(input) { + // check input for whether this test applies + const applicability = await checkApplicability(input) + if ('outcome' in applicability && applicability.outcome === "inapplicable") { + return applicability + } + + // get test targets + const targeting = getTarget(input) + if ('result' in targeting) return targeting.result + /** @type {Array<{ target: Target, result: TestResult}>} */ + const results = [] + for (const target of targeting.targets) { + if (!target) throw new Error(`got undefined target. this should not happen`) + // check expectations against targets + const expectations = await expect(target) + if (expectations && 'result' in expectations) results.push({ + target, + result: expectations.result, + }) + } + if (results.length === 1) { + return results[0].result + } else if (results.length) { + return { + outcome: results.every(r => r.result.outcome === "passed") ? "passed" : "failed", + pointer: { + results + } + } + } + throw Object.assign(new Error('unexpected input'), { input }) +} diff --git a/fep/c551/tests/fep-c551-module-must-export-test-object.test.js b/fep/c551/tests/fep-c551-module-must-export-test-object.test.js new file mode 100644 index 0000000..d94a4c7 --- /dev/null +++ b/fep/c551/tests/fep-c551-module-must-export-test-object.test.js @@ -0,0 +1,54 @@ +import { describe, it } from 'node:test'; +import assert from "node:assert"; + +import testCase from "./fep-c551-module-must-export-test-object.js" + +await describe(`activitypub-testing test ${testCase.slug}`, async () => { + await describe('default export', async () => { + await it('has a uuid', () => { + assert.equal(typeof testCase.uuid, 'string') + }) + }) + + await it('has testCases', async () => { + await testHasTestCases(testCase, { minimum: 1 }) + }) + + await it('when inputs are {}, outcome is inapplicable', async () => { + // @ts-expect-error - testing even though typechecker should prevent + const result = await testCase.run({}) + assert.equal(result.outcome, 'inapplicable') + }) +}); + +/** + * @template Inputs + * @template Outcome + * @param {object} test + * @param {Array} [test.testCases] + * @param {(input: Inputs) => Promise} test.run + * @param {object} options + * @param {number} [options.minimum=0] - minimum required testCases + */ +export async function testHasTestCases({ testCases = [], run }, { minimum = 0 } = {}) { + let remainingForMinimum = minimum + for (const testCase of testCases) { + await it(`test case: "${testCase.name}"`, async () => { + const result = await run(testCase.input) + if (result.outcome !== testCase.result.outcome) { + throw Object.assign( + new Error(`expected result.outcome to be "${testCase.result.outcome}" but got '${result.outcome}'`), + { + name: 'UnexpectedOutcome', + testCase, + result + } + ) + } + remainingForMinimum-- + }) + } + if (remainingForMinimum > 0) { + throw new Error(`test had ${minimum - remainingForMinimum} testCases but failed to meet required minimum of ${minimum}`) + } +} diff --git a/fep/c551/tests/index.js b/fep/c551/tests/index.js new file mode 100644 index 0000000..e440b69 --- /dev/null +++ b/fep/c551/tests/index.js @@ -0,0 +1,5 @@ +import testfepC551ModuleMustExportTestObject from "./fep-c551-module-must-export-test-object"; + +export default [ + testfepC551ModuleMustExportTestObject, +] diff --git a/fep/c551/tests/package-lock.json b/fep/c551/tests/package-lock.json new file mode 100644 index 0000000..25bc86d --- /dev/null +++ b/fep/c551/tests/package-lock.json @@ -0,0 +1,44 @@ +{ + "name": "activitypub-testing-fep-c551", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "activitypub-testing-fep-c551", + "version": "1.0.0", + "devDependencies": { + "@types/node": "^20.14.10", + "typescript": "^5.5.3" + } + }, + "node_modules/@types/node": { + "version": "20.14.10", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.10.tgz", + "integrity": "sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/typescript": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.5.3.tgz", + "integrity": "sha512-/hreyEujaB0w76zKo6717l3L0o/qEUtRgdvUBvlkhoWeOVMjMuHNHk0BRBzikzuGDqNmPQbg5ifMEqsHLiIUcQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true + } + } +} diff --git a/fep/c551/tests/package.json b/fep/c551/tests/package.json new file mode 100644 index 0000000..7037b78 --- /dev/null +++ b/fep/c551/tests/package.json @@ -0,0 +1,14 @@ +{ + "name": "fep-c551-tests", + "type": "module", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "node --test", + "typecheck": "tsc --build" + }, + "devDependencies": { + "@types/node": "^20.14.10", + "typescript": "^5.5.3" + } +} diff --git a/fep/c551/tests/tsconfig.json b/fep/c551/tests/tsconfig.json new file mode 100644 index 0000000..2135fa2 --- /dev/null +++ b/fep/c551/tests/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "composite": true, + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "Bundler", + "esModuleInterop": true, + "allowJs": true, + "checkJs": true, + "strict": true, + "skipLibCheck": true, + "emitDeclarationOnly": true, + "outDir": "dist", + "lib": [ + "esnext", + "DOM" + ], + "jsx": "react-jsx", + "jsxImportSource": "hono/jsx" + }, + "include": ["*.js", "src", "test"], + "references": [ + ] +}