diff --git a/.editorconfig b/.editorconfig index c6c8b3621938..0f1786729b40 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,9 +1,9 @@ root = true [*] -indent_style = space -indent_size = 2 -end_of_line = lf charset = utf-8 -trim_trailing_whitespace = true +end_of_line = lf +indent_size = 2 +indent_style = space insert_final_newline = true +trim_trailing_whitespace = true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000000..6c3cf6d85b0c --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,32 @@ +jobs: + main: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-node@v4 + with: + node-version: lts/* + - run: npm install + - env: + GH_TOKEN: ${{secrets.GH_TOKEN}} + NPM_TOKEN: ${{secrets.NPM_TOKEN}} + OC_TOKEN: ${{secrets.OC_TOKEN}} + UNIFIED_OPTIMIZE_IMAGES: 1 + run: npm test + - uses: JamesIves/github-pages-deploy-action@releases/v4 + with: + branch: dist + commit-message: . + folder: build + git-config-email: tituswormer@gmail.com + git-config-name: Titus Wormer + single-commit: true +name: main +on: + push: + branches: + - main + schedule: + - cron: '45 7 * * *' diff --git a/.gitignore b/.gitignore index f8903057635d..d818400ae8c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,11 @@ -.DS_Store +*.d.ts *.log +*.map +*.tsbuildinfo +.DS_Store +.env build/ data/ node_modules/ -.env yarn.lock +!/generate/types.d.ts diff --git a/.npmrc b/.npmrc index 43c97e719a5a..766ef0dc4554 100644 --- a/.npmrc +++ b/.npmrc @@ -1 +1,3 @@ +ignore-scripts=true +legacy-peer-deps=true package-lock=false diff --git a/.prettierignore b/.prettierignore index 567609b1234a..08e1746b0bdb 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1 +1,3 @@ +*.json +*.md build/ diff --git a/.remarkrc.js b/.remarkrc.js index be386acb52bc..cc611eaee4d0 100644 --- a/.remarkrc.js +++ b/.remarkrc.js @@ -1,31 +1,56 @@ -var fs = require('fs') -var strip = require('strip-comments') -var unified = require('unified') +import fs from 'node:fs/promises' +import dictionaryEn from 'dictionary-en' +import remarkFrontmatter from 'remark-frontmatter' +import remarkLintFirstHeadingLevel from 'remark-lint-first-heading-level' +import remarkLintNoDeadUrls from 'remark-lint-no-dead-urls' +import remarkLintNoHtml from 'remark-lint-no-html' +import remarkPresetWooorm from 'remark-preset-wooorm' +import remarkRetext from 'remark-retext' +import remarkValidateLinks from 'remark-validate-links' +import retextEmoji from 'retext-emoji' +import retextEnglish from 'retext-english' +import retextEquality from 'retext-equality' +import retextPassive from 'retext-passive' +import retextPresetWooorm from 'retext-preset-wooorm' +import retextProfanities from 'retext-profanities' +import retextReadability from 'retext-readability' +import retextSimplify from 'retext-simplify' +import retextSpell from 'retext-spell' +import retextSyntaxMentions from 'retext-syntax-mentions' +import retextSyntaxUrls from 'retext-syntax-urls' +import stripComments from 'strip-comments' +import {unified} from 'unified' -var personal = strip(fs.readFileSync('dictionary.txt', 'utf8')) - -var naturalLanguage = unified().use([ - require('retext-english'), - require('retext-preset-wooorm'), - require('retext-equality'), - require('retext-passive'), - require('retext-profanities'), - [require('retext-readability'), {age: 18, minWords: 8}], - [require('retext-simplify'), {ignore: ['function', 'interface', 'maintain']}], - require('retext-syntax-mentions'), +const naturalLanguage = unified().use([ + retextEnglish, + retextPresetWooorm, + [retextEquality, {ignore: ['hosts', 'whitespace']}], + [retextPassive, {ignore: ['hidden']}], + [retextProfanities, {sureness: 1}], + [retextReadability, {age: 18, minWords: 8}], + [retextSimplify, {ignore: ['function', 'interface', 'maintain', 'type']}], + retextEmoji, + retextSyntaxMentions, + retextSyntaxUrls, [ - require('retext-spell'), - {dictionary: require('dictionary-en-gb'), personal: personal} + retextSpell, + { + dictionary: dictionaryEn, + personal: stripComments(await fs.readFile('dictionary.txt', 'utf8')) + } ] ]) -exports.plugins = [ - require('remark-preset-wooorm'), - require('remark-frontmatter'), - [require('remark-retext'), naturalLanguage], - [require('remark-validate-links'), false], - [require('remark-lint-no-dead-urls'), 'https://unifiedjs.com'], - [require('remark-lint-first-heading-level'), 2], - [require('remark-lint-no-html'), false], - [require('remark-toc'), {heading: 'contents', maxDepth: 3, tight: true}] -] +const config = { + plugins: [ + remarkPresetWooorm, + remarkFrontmatter, + [remarkRetext, naturalLanguage], + [remarkValidateLinks, false], + [remarkLintNoDeadUrls, 'https://unifiedjs.com'], + [remarkLintFirstHeadingLevel, 2], + [remarkLintNoHtml, false] + ] +} + +export default config diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0dc3198c6d27..000000000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: node_js -node_js: node -env: - - UNIFIED_OPTIMIZE_IMAGES=1 -deploy: - provider: pages - local_dir: build - target_branch: master - skip_cleanup: true - github_token: $GH_TOKEN - email: tituswormer@gmail.com - name: Titus Wormer - on: - branch: src diff --git a/asset/big.css b/asset/big.css index de909a066107..a64ac83fef2b 100644 --- a/asset/big.css +++ b/asset/big.css @@ -12,22 +12,22 @@ width: calc(12.333 * (1em + 1ex)); } -@media (min-width: 48.01em) { +@media (width > 48em) { .readme { padding-left: calc(1em + 1ex); padding-right: calc(1em + 1ex); } - .landing .article { - padding: calc(6 * (1em + 1ex) / 1.125) calc(3 * (1em + 1ex) / 1.125); - } - .article { font-size: 1.125em; padding-left: calc(3 * (1em + 1ex) / 1.125); padding-right: calc(3 * (1em + 1ex) / 1.125); } + .landing .article { + padding: calc(6 * (1em + 1ex) / 1.125) calc(3 * (1em + 1ex) / 1.125); + } + .x-hide-l { display: none; } @@ -102,7 +102,7 @@ } } -@media (min-width: 62.01em) { +@media (width > 62em) { .readme { padding-left: 0; padding-right: 0; diff --git a/asset/browser.js b/asset/browser.js new file mode 100644 index 000000000000..00bd34813f62 --- /dev/null +++ b/asset/browser.js @@ -0,0 +1,66 @@ +/// + +/* eslint-env browser */ + +import {computePosition, shift} from '@floating-ui/dom' + +if ('paintWorklet' in CSS) { + // @ts-expect-error: CSS is not yet fully typed. + CSS.paintWorklet.addModule('https://esm.sh/css-houdini-squircle@0.3?bundle') +} + +const popoverTargets = /** @type {Array} */ ( + Array.from(document.querySelectorAll('.rehype-twoslash-popover-target')) +) + +for (const popoverTarget of popoverTargets) { + /** @type {NodeJS.Timeout | number} */ + let timeout = 0 + + popoverTarget.addEventListener('click', function () { + popoverShow(popoverTarget) + }) + + popoverTarget.addEventListener('mouseenter', function () { + clearTimeout(timeout) + timeout = setTimeout(function () { + popoverShow(popoverTarget) + }, 300) + }) + + popoverTarget.addEventListener('mouseleave', function () { + clearTimeout(timeout) + }) + + if (popoverTarget.classList.contains('rehype-twoslash-autoshow')) { + popoverShow(popoverTarget) + } +} + +/** + * @param {HTMLElement} popoverTarget + * Popover target. + * @returns {undefined} + * Nothing. + */ +function popoverShow(popoverTarget) { + const id = popoverTarget.dataset.popoverTarget + if (!id) return + const popover = document.getElementById(id) + if (!popover) return + + popover.showPopover() + + computePosition(popoverTarget, popover, { + placement: 'bottom', + middleware: [shift({padding: 5})] + }).then( + /** + * @param {{x: number, y: number}} value + */ + function (value) { + popover.style.left = value.x + 'px' + popover.style.top = value.y + 'px' + } + ) +} diff --git a/asset/dark.css b/asset/dark.css index b3bec7146c77..cb51cb6a05d5 100644 --- a/asset/dark.css +++ b/asset/dark.css @@ -19,7 +19,7 @@ body { } .box { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); border-color: black; } @@ -28,26 +28,35 @@ body { } .card { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); box-shadow: - 0 0 0 0.2em rgba(46, 143, 255, 0), - 0 13px 27px -5px rgba(0, 0, 43, 0.25), - 0 8px 16px -8px rgba(0, 0, 0, 0.3), - 0 -6px 16px -6px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(46 143 255 / 0%), + 0 13px 27px -5px rgb(0 0 43 / 25%), + 0 8px 16px -8px rgb(0 0 0 / 30%), + 0 -6px 16px -6px rgb(0 0 0 / 2.5%); } .card:hover { box-shadow: - 0 0 0 0.2em rgba(46, 143, 255, 0), - 0 30px 60px -12px rgba(0, 0, 43, 0.25), - 0 18px 36px -18px rgba(0, 0, 0, 0.3), - 0 -12px 36px -8px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(46 143 255 / 0%), + 0 30px 60px -12px rgb(0 0 43 / 25%), + 0 18px 36px -18px rgb(0 0 0 / 30%), + 0 -12px 36px -8px rgb(0 0 0 / 2.5%); } .more { border-color: black; } +.releases { + background-color: rgb(0 0 0 / 20%); +} + +.release, +.releases .more { + background-color: var(--gray-9); +} + .tag { color: var(--blue-3); background-color: var(--blue-7); @@ -79,7 +88,7 @@ a.tag:focus .count { } .search button { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); } .search input:hover, @@ -98,10 +107,10 @@ a.tag:focus .count { .card:focus { /* --blue-3-83 */ box-shadow: - 0 0 0 0.2em rgb(46, 143, 255), - 0 30px 60px -12px rgba(0, 0, 43, 0.25), - 0 18px 36px -18px rgba(0, 0, 0, 0.3), - 0 -12px 36px -8px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(46 143 255), + 0 30px 60px -12px rgb(0 0 43 / 25%), + 0 18px 36px -18px rgb(0 0 0 / 30%), + 0 -12px 36px -8px rgb(0 0 0 / 2.5%); } .content h1, @@ -110,22 +119,22 @@ a.tag:focus .count { } .content kbd { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); border-color: var(--gray-8); box-shadow: inset 0 -1px 0 black; color: var(--gray-2); } .content pre { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); } .content code { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); } .content hr { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); } .content tr { @@ -147,16 +156,24 @@ a.tag:focus .count { } .content blockquote::before { - background-color: rgba(0, 0, 0, 0.2); + background-color: rgb(0 0 0 / 20%); } .content abbr.first { border-bottom-color: var(--gray-6); } -/* Not enough contrast */ -.hljs-comment { - color: var(--gray-4); +.highlight:is(:hover, :focus-within) .rehype-twoslash-popover-target { + background-color: var(--gray-5); +} + +.rehype-twoslash-popover { + background-color: var(--gray-9); + border-color: var(--gray-7); +} + +.rehype-twoslash-popover-description { + background-color: var(--gray-9); } .screen { diff --git a/asset/image/decap-cms-dark.png b/asset/image/decap-cms-dark.png new file mode 100644 index 000000000000..6df830a11ae8 Binary files /dev/null and b/asset/image/decap-cms-dark.png differ diff --git a/asset/image/decap-cms.png b/asset/image/decap-cms.png new file mode 100644 index 000000000000..6df830a11ae8 Binary files /dev/null and b/asset/image/decap-cms.png differ diff --git a/asset/image/netlify-cms-dark.png b/asset/image/netlify-cms-dark.png deleted file mode 100644 index c98190f5cab5..000000000000 Binary files a/asset/image/netlify-cms-dark.png and /dev/null differ diff --git a/asset/image/netlify-cms.png b/asset/image/netlify-cms.png deleted file mode 100644 index 397359ea36c6..000000000000 Binary files a/asset/image/netlify-cms.png and /dev/null differ diff --git a/asset/image/nextein-dark.png b/asset/image/nextein-dark.png deleted file mode 100644 index 3eb805190eb0..000000000000 Binary files a/asset/image/nextein-dark.png and /dev/null differ diff --git a/asset/image/nextein.png b/asset/image/nextein.png deleted file mode 100644 index 46716ff8bd3b..000000000000 Binary files a/asset/image/nextein.png and /dev/null differ diff --git a/asset/image/prettyhtml-dark.png b/asset/image/prettyhtml-dark.png deleted file mode 100644 index 6be21a179271..000000000000 Binary files a/asset/image/prettyhtml-dark.png and /dev/null differ diff --git a/asset/image/prettyhtml.png b/asset/image/prettyhtml.png deleted file mode 100644 index 895c6a0ba367..000000000000 Binary files a/asset/image/prettyhtml.png and /dev/null differ diff --git a/asset/image/unified-overview.png b/asset/image/unified-overview.png deleted file mode 100644 index 3666239dad3b..000000000000 Binary files a/asset/image/unified-overview.png and /dev/null differ diff --git a/asset/index.css b/asset/index.css index 8243b1676927..f613c8e4d154 100644 --- a/asset/index.css +++ b/asset/index.css @@ -30,8 +30,7 @@ html { color-scheme: light dark; font-family: system-ui; - -ms-text-size-adjust: 100%; - -webkit-text-size-adjust: 100%; + text-size-adjust: 100%; word-wrap: break-word; font-kerning: normal; font-feature-settings: 'kern', 'liga', 'clig', 'calt'; @@ -46,12 +45,7 @@ input { kbd, pre, code { - font-family: - SFMono-Regular, - Consolas, - Liberation Mono, - Menlo, - Courier, + font-family: SFMono-Regular, Consolas, 'Liberation Mono', Menlo, Courier, monospace; font-feature-settings: normal; } @@ -255,12 +249,15 @@ abbr { } .mdast .hl, +.micromark .hl, .remark .hl, .content .mdast:hover .hl, +.content .micromark:hover .hl, .content .remark:hover .hl, .content .mdast:focus .hl, +.content .micromark:focus .hl, .content .remark:focus .hl { - color: hsl(0, 97%, 43%); + color: hsl(0deg 97% 43%); } .mdx { @@ -279,7 +276,7 @@ abbr { .content .nlcst:hover .hl, .content .retext:focus .hl, .content .nlcst:focus .hl { - color: hsl(120, 97%, 43%); + color: hsl(120deg 97% 43%); } .rehype .hl, @@ -288,13 +285,22 @@ abbr { .content .hast:hover .hl, .content .rehype:focus .hl, .content .hast:focus .hl { - color: hsl(45, 97%, 43%); + color: hsl(45deg 97% 43%); +} + +.recma .hl, +.esast .hl, +.content .recma:hover .hl, +.content .esast:hover .hl, +.content .recma:focus .hl, +.content .esast:focus .hl { + color: hsl(180deg 97% 43%); } .xast .hl, .content .xast:hover .hl, .content .xast:focus .hl { - color: hsl(261, 51%, 51%); + color: hsl(261deg 51% 51%); } .redot .hl, @@ -303,7 +309,7 @@ abbr { .content .dotast:hover .hl, .content .redot:focus .hl, .content .dotast:focus .hl { - color: hsl(300, 97%, 43%); + color: hsl(300deg 97% 43%); } .lowlight, @@ -340,7 +346,7 @@ abbr { .search input { display: block; width: 100%; - -webkit-appearance: none; + appearance: none; background-color: var(--gray-0); } @@ -384,25 +390,52 @@ abbr { .box:hover, .box:focus { - border-color: currentColor; + border-color: currentcolor; } .card { transition: 200ms; transition-property: color, box-shadow; box-shadow: - 0 0 0 0.2em rgba(3, 102, 214, 0), - 0 13px 27px -5px rgba(50, 50, 93, 0.25), - 0 8px 16px -8px rgba(0, 0, 0, 0.3), - 0 -6px 16px -6px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(3 102 214 / 0%), + 0 13px 27px -5px rgb(50 50 93 / 25%), + 0 8px 16px -8px rgb(0 0 0 / 30%), + 0 -6px 16px -6px rgb(0 0 0 / 2.5%); } .card:hover { box-shadow: - 0 0 0 0.2em rgba(3, 102, 214, 0), - 0 30px 60px -12px rgba(50, 50, 93, 0.25), - 0 18px 36px -18px rgba(0, 0, 0, 0.3), - 0 -12px 36px -8px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(3 102 214 / 0%), + 0 30px 60px -12px rgb(50 50 93 / 25%), + 0 18px 36px -18px rgb(0 0 0 / 30%), + 0 -12px 36px -8px rgb(0 0 0 / 2.5%); +} + +.releases { + margin: calc(-1 * (0.25em + 0.25ex)); + background-color: rgb(0 0 0 / 4%); + border-radius: 8px; + overflow: hidden; + mask-image: paint(squircle); + + --squircle-radius: 8px; + --squircle-smooth: 1; +} + +.release { + padding: 0 calc(1.25 * (1em + 1ex)); + overflow: hidden; +} + +.release, +.releases .more { + border-radius: 3px; + background-color: var(--gray-0); + margin: calc(0.25 * (1em + 1ex)); +} + +.release:nth-child(odd) { + background-color: transparent; } .nl-root { @@ -456,8 +489,10 @@ abbr { .double-ellipsis { overflow: hidden; - display: -webkit-box; + /* stylelint-disable-next-line declaration-property-value-no-unknown */ + display: box; -webkit-line-clamp: 2; + line-clamp: 2; -webkit-box-orient: vertical; } @@ -478,7 +513,13 @@ abbr { padding-right: calc(0.125 * (1em + 1ex)); background-color: var(--hl); color: var(--blue-1); - transition: color 200ms, background-color 200ms; + transition: + color 200ms, + background-color 200ms; + mask-image: paint(squircle); + + --squircle-radius: 3px; + --squircle-smooth: 1; } a.tag:hover, @@ -525,20 +566,24 @@ a.tag:focus .count { .search input:focus, .search button:focus, .search button:active { - box-shadow: 0 0 0 0.2em rgba(3, 102, 214, 0.3); /* --blue-5 */ + box-shadow: 0 0 0 0.2em rgb(3 102 214 / 30%); /* --blue-5 */ } .card:focus { box-shadow: - 0 0 0 0.2em rgb(3, 102, 214), - 0 30px 60px -12px rgba(50, 50, 93, 0.25), - 0 18px 36px -18px rgba(0, 0, 0, 0.3), - 0 -12px 36px -8px rgba(0, 0, 0, 0.025); + 0 0 0 0.2em rgb(3 102 214), + 0 30px 60px -12px rgb(50 50 93 / 25%), + 0 18px 36px -18px rgb(0 0 0 / 30%), + 0 -12px 36px -8px rgb(0 0 0 / 2.5%); } .landing { color: var(--gray-0); background-image: linear-gradient(150deg, #0366d6 15%, #24292e 85%); + mask-image: paint(squircle); + + --squircle-radius: 16px; + --squircle-smooth: 1; } .landing .article { @@ -580,8 +625,8 @@ a.tag:focus .count { .content h3 { font-size: 1.5em; - margin-top: calc(0.6667 * (1em + 1.16667ex)); - margin-bottom: calc(0.6667 * (1em + 1.16667ex)); + margin-top: calc(0.6667 * (1em + 1.1667ex)); + margin-bottom: calc(0.6667 * (1em + 1.1667ex)); } .landing p, @@ -619,7 +664,7 @@ a.tag:focus .count { .content pre { word-wrap: normal; - background-color: rgba(0, 0, 0, 0.04); + background-color: rgb(0 0 0 / 4%); overflow: auto; padding: calc(1em + 1ex); margin-left: calc(-1 * (1em + 1ex)); @@ -635,7 +680,7 @@ a.tag:focus .count { } .content code { - background-color: rgba(0, 0, 0, 0.04); + background-color: rgb(0 0 0 / 4%); border-radius: 3px; padding: 0.2em 0.4em; } @@ -650,10 +695,14 @@ a.tag:focus .count { } .content hr { - background-color: rgba(0, 0, 0, 0.04); + background-color: rgb(0 0 0 / 4%); border: 0; border-radius: 3px; height: calc(0.25 * (1em + 1ex)); + mask-image: paint(squircle); + + --squircle-radius: 3px; + --squircle-smooth: 1; } .content table { @@ -695,6 +744,10 @@ a.tag:focus .count { border-radius: 3px; position: absolute; left: 0; + mask-image: paint(squircle); + + --squircle-radius: 3px; + --squircle-smooth: 1; } .content ul ul, @@ -730,7 +783,7 @@ a.tag:focus .count { } .content li { - word-wrap: break-all; + word-wrap: break-word; margin-top: calc(0.25 * (1em + 1ex)); margin-bottom: calc(0.25 * (1em + 1ex)); margin-left: calc(1 * (1em + 1ex)); @@ -778,7 +831,57 @@ a.tag:focus .count { border-bottom-color: var(--gray-3); } -@media (max-width: 48em) { +.rehype-twoslash-completion-deprecated { + opacity: 0.5; +} + +.rehype-twoslash-popover-target { + cursor: default; +} + +.highlight:is(:hover, :focus-within) .rehype-twoslash-popover-target { + background-color: var(--gray-2); +} + +/* Wavy underline for errors. */ +.rehype-twoslash-error-target { + background-repeat: repeat-x; + background-position: bottom left; + background-image: url('data:image/svg+xml,'); +} + +/* The content that will be shown in the tooltip. */ +.rehype-twoslash-popover { + position: absolute; + max-width: calc(45 * (1em + 1ex)); + padding: calc(0.5 * (1em + 1ex)); + margin: 0; + background-color: var(--gray-0); + border: 1px solid var(--gray-2); + border-radius: 3px; +} + +/* No padding if we have a padded code block (and perhaps more blocks) */ +.rehype-twoslash-popover:has(.rehype-twoslash-popover-code) { + padding: 0; +} + +.rehype-twoslash-popover-code { + /* Overwrite `.content pre` */ + margin: 0 !important; +} + +.rehype-twoslash-popover-code > code { + mask-image: none; + border-radius: 0; +} + +.rehype-twoslash-popover-description { + background-color: var(--gray-0); + padding: 0 calc(2 * (1em + 1ex)); +} + +@media (width <= 48em) { .x-show-l { display: none; } diff --git a/asset/search.js b/asset/search.js index 2007bf79e1fb..facdb00e6c07 100644 --- a/asset/search.js +++ b/asset/search.js @@ -1,242 +1,394 @@ +/// + +/* eslint-disable unicorn/prefer-global-this */ /* eslint-env browser */ -var FlexSearch = require('flexsearch') -var mean = require('compute-mean') -var toDom = require('hast-util-to-dom') -var data = require('../generate/data') -var searchForm = require('../generate/molecule/search') -var reduceScore = require('../generate/component/project/helper-reduce-score') -var keywordFilter = require('../generate/component/keyword/helper-filter') -var keywordPreview = require('../generate/component/keyword/search-preview') -var keywordEmpty = require('../generate/component/keyword/search-empty') -var keywordResults = require('../generate/component/keyword/search-results') -var topicFilter = require('../generate/component/topic/helper-filter') -var topicPreview = require('../generate/component/topic/search-preview') -var topicEmpty = require('../generate/component/topic/search-empty') -var topicResults = require('../generate/component/topic/search-results') -var packagePreview = require('../generate/component/package/search-preview') -var packageEmpty = require('../generate/component/package/search-empty') -var packageResults = require('../generate/component/package/search-results') -var projectPreview = require('../generate/component/project/search-preview') -var projectEmpty = require('../generate/component/project/search-empty') -var projectResults = require('../generate/component/project/search-results') -var unique = require('../generate/util/unique') -var {asc, desc} = require('../generate/util/sort') - -var loc = window.location -var home = '/explore/' -var param = 'q' -var id = 'search-root' +/** + * @import {ElementContent} from 'hast' + * @import {Index as FlexSearch} from 'flexsearch' + * @import {Data} from '../generate/data.js' + */ + +/** + * @callback Create + * @param {Search} search + * @returns {Promise} + * + * @callback Empty + * @param {Data} data + * @param {string} query + * @returns {ElementContent} + * + * @callback Filter + * @param {Data} data + * @param {ReadonlyArray} result + * @returns {Array} + * + * @callback Preview + * @param {Data} data + * @returns {ElementContent} + * + * @callback Results + * @param {Data} data + * @param {ReadonlyArray} result + * @returns {ElementContent} + * + * @typedef Search + * @property {HTMLElement} $scope + * @property {Create} create + * @property {Empty} empty + * @property {Filter | undefined} [filter] + * @property {FlexSearch} index + * @property {Preview} preview + * @property {Results} results + * @property {string} selector + * @property {Weight} weight + * + * @callback Weight + * @param {string} item + * @returns {number} + */ + +import {ok as assert} from 'devlop' +import flexsearch from 'flexsearch' +import {toDom} from 'hast-util-to-dom' +import {helperReduceScore} from '../generate/component/project/helper-reduce-score.js' +import {searchEmpty as projectEmpty} from '../generate/component/project/search-empty.js' +import {searchPreview as projectPreview} from '../generate/component/project/search-preview.js' +import {searchResults as projectResults} from '../generate/component/project/search-results.js' +import {helperFilter as keywordFilter} from '../generate/component/keyword/helper-filter.js' +import {searchEmpty as keywordEmpty} from '../generate/component/keyword/search-empty.js' +import {searchPreview as keywordPreview} from '../generate/component/keyword/search-preview.js' +import {searchResults as keywordResults} from '../generate/component/keyword/search-results.js' +import {helperFilter as topicFilter} from '../generate/component/topic/helper-filter.js' +import {searchEmpty as topicEmpty} from '../generate/component/topic/search-empty.js' +import {searchPreview as topicPreview} from '../generate/component/topic/search-preview.js' +import {searchResults as topicResults} from '../generate/component/topic/search-results.js' +import {searchEmpty as packageEmpty} from '../generate/component/package/search-empty.js' +import {searchPreview as packagePreview} from '../generate/component/package/search-preview.js' +import {searchResults as packageResults} from '../generate/component/package/search-results.js' +import {search as searchForm} from '../generate/molecule/search.js' +import {asc, desc} from '../generate/util/sort.js' +import {data} from '../generate/data.js' + +const Index = flexsearch.Index + +const loc = window.location +const home = '/explore/' +const parameter = 'q' +const id = 'search-root' // For some reason this can be fired multiple times. if (loc.pathname === home && !document.querySelector('#' + id + ' form')) { init() } -function init() { - var names = Object.keys(data.packageByName) - var repos = Object.keys(data.projectByRepo) - var keywords = Object.keys(data.packagesByKeyword) - var topics = Object.keys(data.projectsByTopic) - var $root = document.querySelector('#' + id) - var $form = toDom(searchForm(data, param)) - var $input = $form.querySelector('[name=' + param + ']') +async function init() { + const names = Object.keys(data.packageByName) + const repos = Object.keys(data.projectByRepo) + const keywords = Object.keys(data.packagesByKeyword) + const topics = Object.keys(data.projectsByTopic) + const $root = document.querySelector('#' + id) + assert($root) + const $form = toDom(searchForm(data, parameter)) + assert($form instanceof HTMLElement) + const $input = $form.querySelector('[name=' + parameter + ']') + assert($input) $root.prepend($form) - var promises = [ + /** @type {Array>} */ + const rawSearches = [ { - selector: '#root-keyword', - create: search => - new Promise(resolve => - window.requestAnimationFrame(() => { - keywords.forEach(d => search.index.add(d, d)) - resolve() + /** + * @param {Search} search + * @returns {Promise} + */ + create(search) { + return new Promise(function (resolve) { + window.requestAnimationFrame(function () { + for (const d of keywords) search.index.add(d, d) + resolve(undefined) }) - ), - weight: d => data.packagesByKeyword[d].length, + }) + }, + empty: keywordEmpty, filter: keywordFilter, preview: keywordPreview, - empty: keywordEmpty, - results: keywordResults + results: keywordResults, + selector: '#root-keyword', + /** + * @param {string} d + * @returns {number} + */ + weight(d) { + return data.packagesByKeyword[d].length + } }, { - selector: '#root-topic', - create: search => - new Promise(resolve => - window.requestAnimationFrame(() => { - topics.forEach(d => search.index.add(d, d)) - resolve() + /** + * @param {Search} search + * @returns {Promise} + */ + create(search) { + return new Promise(function (resolve) { + window.requestAnimationFrame(function () { + for (const d of topics) search.index.add(d, d) + resolve(undefined) }) - ), - weight: d => data.projectsByTopic[d].length, + }) + }, + empty: topicEmpty, filter: topicFilter, preview: topicPreview, - empty: topicEmpty, - results: topicResults + results: topicResults, + selector: '#root-topic', + /** + * @param {string} d + * @returns {number} + */ + weight(d) { + return data.projectsByTopic[d].length + } }, { - selector: '#root-package', - create: search => - new Promise(resolve => { - var size = 100 - - window.requestAnimationFrame(() => next(0)) + /** + * @param {Search} search + * @returns {Promise} + */ + create(search) { + return new Promise(function (resolve) { + const size = 100 + + window.requestAnimationFrame(function () { + next(0) + }) + /** + * @param {number} start + * @returns {undefined} + */ function next(start) { - var end = start + size - var slice = names.slice(start, end) + const end = start + size + const slice = names.slice(start, end) - slice.forEach(d => - search.index.add(d, d + ' ' + data.packageByName[d].description) - ) + for (const d of slice) + search.index.add( + d, + d.split('/').join(' ') + ' ' + data.packageByName[d].description + ) if (slice.length === 0) { - resolve() + resolve(undefined) } else { - window.requestAnimationFrame(() => next(end)) + window.requestAnimationFrame(function () { + next(end) + }) } } - }), - weight: d => data.packageByName[d].score, - preview: packagePreview, + }) + }, empty: packageEmpty, - results: packageResults + preview: packagePreview, + results: packageResults, + selector: '#root-package', + /** + * @param {string} d + * @returns {number} + */ + weight(d) { + return data.packageByName[d].score + } }, { - selector: '#root-project', - create: search => - new Promise(resolve => { - var size = 100 - - window.requestAnimationFrame(() => next(0)) + /** + * @param {Search} search + * @returns {Promise} + */ + create(search) { + return new Promise(function (resolve) { + const size = 100 + + window.requestAnimationFrame(function () { + next(0) + }) + /** + * @param {number} start + * @returns {undefined} + */ function next(start) { - var end = start + size - var slice = repos.slice(start, end) - - slice.forEach(d => - search.index.add(d, d + ' ' + data.projectByRepo[d].description) - ) + const end = start + size + const slice = repos.slice(start, end) + + for (const d of slice) { + search.index.add( + d, + d.split('/').join(' ') + ' ' + data.projectByRepo[d].description + ) + } if (slice.length === 0) { - resolve() + resolve(undefined) } else { - window.requestAnimationFrame(() => next(end)) + window.requestAnimationFrame(function () { + next(end) + }) } } - }), - weight: d => reduceScore(data, d), - preview: projectPreview, + }) + }, empty: projectEmpty, - results: projectResults + preview: projectPreview, + results: projectResults, + selector: '#root-project', + /** + * @param {string} d + * @returns {number} + */ + weight(d) { + return helperReduceScore(data, d) + } } - ].map(d => { - var $scope = document.querySelector(d.selector) - var index = new FlexSearch({ - profile: 'score', - encode: 'advanced', - tokenize: 'full' - }) - var res = {...d, index, $scope} + ] - return res.create(res).then(() => res) - }) + /** @type {Array} */ + const searches = [] - Promise.all(promises).then(searches => { - start() + for (const d of rawSearches) { + const $scope = document.querySelector(d.selector) + assert($scope instanceof HTMLElement) + const index = new Index({preset: 'score', tokenize: 'full'}) + /** @type {Search} */ + const view = {...d, $scope, index} - $form.addEventListener('submit', onsubmit) - window.addEventListener('popstate', onpopstate) + await view.create(view) - function start() { - var query = clean(new URL(loc).searchParams.get(param)) + searches.push(view) + } - if (query) { - onpopstate() - } - } + start() - function onpopstate() { - search(clean(new URL(loc).searchParams.get(param))) - } + $form.addEventListener('submit', onsubmit) + window.addEventListener('popstate', onpopstate) - function onsubmit(ev) { - var url = new URL(loc) - var current = clean(url.searchParams.get(param)) - var value = clean($input.value) + /** + * @returns {undefined} + */ + function start() { + const query = clean( + new URL(loc.href).searchParams.get(parameter) || undefined + ) - ev.preventDefault() + if (query) { + onpopstate() + } + } - if (current === value) { - return - } + /** + * @returns {undefined} + */ + function onpopstate() { + search(clean(new URL(loc.href).searchParams.get(parameter) || undefined)) + } - if (value) { - url.searchParams.set(param, value) - } else { - url.searchParams.delete(param) - } + /** + * @param {HTMLElementEventMap['submit']} event + * @returns {undefined} + */ + function onsubmit(event) { + const url = new URL(loc.href) + const current = clean(url.searchParams.get(parameter) || undefined) + assert($input instanceof HTMLInputElement) + const value = clean($input.value) + + event.preventDefault() - history.pushState( - {}, - null, - url.pathname + url.search.replace(/%20/g, '+') - ) + if (current === value) { + return + } - search(value) + if (value) { + url.searchParams.set(parameter, value) + } else { + url.searchParams.delete(parameter) } - function search(query) { - $input.value = query + history.pushState({}, '', url.pathname + url.search.replaceAll('%20', '+')) - if (!query) { - searches.forEach(search => replace(search, [], query)) - return - } + search(value) + } - searches.forEach(search => { - search.index.search(query, {suggest: true}, function(res) { - var clean = res.filter(unique) - var weighted = desc(clean, weight) + /** + * @param {string} query + * @returns {undefined} + */ + function search(query) { + const $release = document.querySelector('#root-release') + assert($release instanceof HTMLElement) + assert($input instanceof HTMLInputElement) + $input.value = query + + if (!query) { + $release.style.removeProperty('display') + for (const search of searches) replace(search, [], query) + return + } - replace(search, asc(clean, combined), query) + $release.style.display = 'block' - function combined(d) { - return mean([clean.indexOf(d), weighted.indexOf(d)]) - } + for (const search of searches) { + search.index.searchAsync(query, {suggest: true}, function (result) { + const clean = [...new Set(/** @type {Array} */ (result))] + const weighted = desc(clean, function (d) { + return search.weight(d) }) - function weight(d) { - return search.weight(d) + replace(search, asc(clean, combined), query) + + /** + * @param {string} d + * @returns {number} + */ + function combined(d) { + return (clean.indexOf(d) + weighted.indexOf(d)) / 2 } }) } - }) + } } -function replace(search, res, query) { - var {$scope, filter, preview, empty, results} = search +/** + * @param {Search} search + * @param {ReadonlyArray} result + * @param {string} query + */ +function replace(search, result, query) { + const {$scope, filter, preview, empty, results} = search if (filter) { - res = filter(data, res) + result = filter(data, result) } - var $next = toDom( - res.length === 0 + const $next = toDom( + result.length === 0 ? query ? empty(data, query) : preview(data) - : results(data, res) + : results(data, result) ) while ($scope.firstChild) { - $scope.removeChild($scope.firstChild) + $scope.firstChild.remove() } $scope.append($next) } +/** + * @param {string | undefined} value + * @returns {string} + */ function clean(value) { return (value || '').trim().toLowerCase() } diff --git a/crawl/ecosystem.js b/crawl/ecosystem.js index 7b5af30a9853..e8c24a8313f8 100644 --- a/crawl/ecosystem.js +++ b/crawl/ecosystem.js @@ -1,210 +1,418 @@ -var fs = require('fs').promises -var path = require('path') -var {promisify} = require('util') -var hostedGitInfo = require('hosted-git-info') -var trough = require('trough') -var chalk = require('chalk') -var fetch = require('node-fetch') -var pAll = require('p-all') - -require('dotenv').config() - -var ghToken = process.env.GH_TOKEN -var npmToken = process.env.NPM_TOKEN - -if (!ghToken || !npmToken) { - console.log('Cannot crawl ecosystem without GH or npm tokens') - /* eslint-disable-next-line unicorn/no-process-exit */ - process.exit() +/** + * @import {PackageJson} from 'type-fest' + */ + +/** + * @typedef GHError + * @property {string} documentation_url + * @property {string} message + * + * @typedef NpmsPackageCollectedBucket + * @property {number} count + * @property {string} from + * @property {string} to + * + * @typedef NpmsPackageCollectedContributor + * @property {number} commitsCount + * @property {string} username + * + * @typedef NpmsPackageCollectedGithub + * @property {Array} commits + * @property {Array} contributors + * @property {number} forksCount + * @property {string} homepage + * @property {{count: number, distribution: Record, isDisabled: boolean, openCount: number}} issues + * @property {number} starsCount + * @property {number} subscribersCount + * + * @typedef NpmsPackageCollectedLinks + * @property {string | undefined} [bugs] + * @property {string | undefined} [homepage] + * @property {string} npm + * @property {string | undefined} [repository] + * + * @typedef NpmsPackageCollectedMetadata + * @property {NpmsPackageCollectedPerson} author + * @property {Array} contributors + * @property {string} date + * @property {Record} dependencies + * @property {string | undefined} [deprecated] + * @property {string} description + * @property {Record} devDependencies + * @property {boolean | undefined} [hasSelectiveFiles] + * @property {boolean} hasTestScript + * @property {Array} keywords + * @property {string} license + * @property {NpmsPackageCollectedLinks} links + * @property {Array} maintainers + * @property {string} name + * @property {NpmsPackageCollectedPerson} publisher + * @property {string} readme + * @property {Array} releases + * @property {NpmsPackageCollectedRepository} repository + * @property {string} scope + * @property {string} version + * + * @typedef NpmsPackageCollectedNpm + * @property {Array} downloads + * @property {number} starsCount + * + * @typedef NpmsPackageCollectedPerson + * @property {string} email + * @property {string | undefined} [name] + * @property {string | undefined} [url] + * @property {string} username + * + * @typedef NpmsPackageCollectedRepository + * @property {'git'} type + * @property {string} url + * + * @typedef NpmsPackageCollectedSource + * @property {Array<{info: unknown, urls: unknown}>} badges + * @property {number} coverage + * @property {{hasChangelog: boolean, readmeSize: number, testsSize: number}} files + * @property {Array} linters + * + * @typedef NpmsPackageCollected + * @property {NpmsPackageCollectedGithub} github + * @property {NpmsPackageCollectedMetadata} metadata + * @property {NpmsPackageCollectedNpm} npm + * @property {NpmsPackageCollectedSource} source + * + * @typedef NpmsPackageEvaluation + * @property {{commitsFrequency: number, issuesDistribution: number, openIssues: number, releasesFrequency: number}} maintenance + * @property {{communityInterest: number, dependentsCount: number, downloadsAcceleration: number, downloadsCount: number}} popularity + * @property {{branding: number, carefulness: number, health: number, tests: number}} quality + * + * @typedef NpmsPackageResult + * @property {string} analyzedAt + * @property {NpmsPackageCollected | undefined} [collected] + * @property {NpmsPackageEvaluation} evaluation + * @property {NpmsPackageScore | undefined} [score] + * + * @typedef NpmsPackageScoreDetail + * @property {number} maintenance + * @property {number} popularity + * @property {number} quality + * + * @typedef NpmsPackageScore + * @property {number} final + * @property {NpmsPackageScoreDetail} detail + * + * + * @typedef PackageInfo + * @property {string | undefined} description + * @property {Array} keywords + * @property {string | undefined} latest + * @property {string | undefined} license + * @property {string} name + * @property {number} score + * + * @typedef RawPackage + * @property {string | undefined} description + * @property {number} downloads + * @property {number | undefined} [gzip] + * @property {Array} keywords + * @property {string | undefined} [latest] + * @property {string | undefined} [license] + * @property {string | undefined} [manifestBase] + * @property {string} name + * @property {string} readmeName + * @property {string} repo + * @property {number} score + * + * @typedef RawProject + * @property {string | undefined} default + * @property {string} description + * @property {boolean} hasPackages + * @property {number} issueClosed + * @property {number} issueOpen + * @property {Array} manifests + * @property {number} prClosed + * @property {number} prOpen + * @property {string} repo + * @property {number} size + * @property {number} stars + * @property {Array} topics + * @property {string | undefined} url + * + * @typedef RawRelease + * @property {string} description + * @property {string} published + * @property {string} repo + * @property {string} tag + * + * @typedef SimpleProject + * @property {string} description + * @property {string} repo + * @property {number} stars + * @property {ReadonlyArray} topics + * @property {string | undefined} url + */ + +import assert from 'node:assert/strict' +import fs from 'node:fs/promises' +import process from 'node:process' +import bytes from 'bytes' +import dotenv from 'dotenv' +import hostedGitInfo from 'hosted-git-info' +import randomUseragent from 'random-useragent' +import {constantCollective} from '../generate/util/constant-collective.js' +import {constantTopic} from '../generate/util/constant-topic.js' + +dotenv.config() + +const ghToken = process.env.GH_TOKEN +assert(ghToken) +const npmToken = process.env.NPM_TOKEN +assert(npmToken) + +const hawkgirl = 'application/vnd.github.hawkgirl-preview+json' +const ghEndpoint = 'https://api.github.com/graphql' +const npmsEndpoint = 'https://api.npms.io/v2/package' +const npmDownloadsEndpoint = 'https://api.npmjs.org/downloads' + +/** @type {Set} */ +const reposSet = new Set() + +for (const d of constantCollective) { + const results = await searchOrg(d) + for (const d of results) reposSet.add(d) } -var topics = require('../generate/util/constant-topic') -var orgs = require('../generate/util/constant-collective') - -var outpath = path.join('data') -var readmePath = path.join(outpath, 'readme') -var projectsPath = path.join(outpath, 'projects.json') -var packagesPath = path.join(outpath, 'packages.json') - -var concurrency = {concurrency: 1} - -var hawkgirl = 'application/vnd.github.hawkgirl-preview+json' -var ghEndpoint = 'https://api.github.com/graphql' -var npmsEndpoint = 'https://api.npms.io/v2/package' -var bundlePhobiaEndpoint = 'https://bundlephobia.com/api/size' -var npmDownloadsEndpoint = 'https://api.npmjs.org/downloads' - -var topicPipeline = promisify(trough().use(searchTopic).run) -var orgPipeline = promisify(trough().use(searchOrg).run) -var repoPipeline = promisify(trough().use(crawlRepo).run) -var pkgPipeline = promisify( - trough() - .use(getManifest) - .use(getPackage) - .use(getDownloads) - .use(getSize).run -) - -var main = promisify( - trough() - .use(findProjectsByTopic) - .use(findProjectsInOrganizations) - .use(findRepositories) - .use(findPackages) - .use(writeResults) - .use(writeReadmes).run -) - -main({ - ghToken, - npmToken, - repos: [], - topics: topics, - orgs: orgs -}).then( - res => { - console.log( - chalk.green('✓') + ' done (%d packages, %d projects, %d readmes)', - res.packages.length, - res.projects.length, - res.readmes.length - ) - }, - err => { - console.log(chalk.red('✖') + ' error') - console.error(err) - } -) - -async function findProjectsByTopic(ctx) { - var {topics, repos} = ctx - - var results = await pAll( - topics.map(topic => () => topicPipeline({...ctx, topic})), - concurrency - ) - - return {...ctx, repos: repos.concat(results.flatMap(d => d.matches))} +for (const d of constantTopic) { + const results = await searchTopic(d) + for (const d of results) reposSet.add(d) } -async function findProjectsInOrganizations(ctx) { - var {orgs, repos} = ctx +const repos = [...reposSet] - var results = await pAll( - orgs.map(org => () => orgPipeline({...ctx, org})), - concurrency - ) +/** @type {Array<{project: RawProject, releases: Array}>} */ +const results = [] - return {...ctx, repos: repos.concat(results.flatMap(d => d.matches))} +for (const d of repos) { + results.push(await crawlRepo(d)) } -async function findRepositories(ctx) { - var {repos} = ctx - - repos = repos.filter((d, i, data) => data.indexOf(d) === i) +/** @type {Array} */ +const projects = [] +/** @type {Array} */ +const releases = [] - var results = await pAll( - repos.map(repo => () => repoPipeline({...ctx, repo})), - concurrency - ) - - return {...ctx, projects: results.map(d => d.project)} +for (const d of results) { + projects.push(d.project) + releases.push(...d.releases) } -async function findPackages(ctx) { - var {projects} = ctx +await fs.writeFile( + new URL('../data/releases.js', import.meta.url), + [ + '/**', + " * @import {Root} from 'hast'", + ' */', + '', + '/**', + ' * @typedef Release', + ' * @property {string} description', + ' * @property {Root} [descriptionRich]', + ' * @property {string} published', + ' * @property {string} repo', + ' * @property {string} tag', + ' */', + '', + '/** @type {ReadonlyArray} */', + 'export const releases = ' + JSON.stringify(releases, undefined, 2), + '' + ].join('\n') +) - var packages = projects.flatMap(d => - d.manifests.map(m => ({manifest: m, project: d})) - ) +const packages = await findPackages(projects) - var results = await pAll( - packages.map(({manifest, project}) => () => - pkgPipeline({...ctx, manifest, project}) - ), - {concurrency: 1} - ) +/** @type {Array} */ +const projectsWithPackages = [] - var readmes = [] - var projectsWithPackages = {} +for (const project of projects) { + if (project.hasPackages) { + projectsWithPackages.push(project) + } +} - packages = results - .filter(d => d.proper) - .map(d => { - var {packageDist, readme, project} = d - var {name} = packageDist - var {repo} = project - var readmeName = name.replace(/^@/g, '').replace(/\//g, '-') + '.md' +/** @type {Array} */ +const simpleProjects = [] + +for (const d of projectsWithPackages) { + simpleProjects.push({ + description: d.description, + repo: d.repo, + stars: d.stars, + topics: d.topics, + url: d.url + }) +} - projectsWithPackages[repo] = project +await fs.writeFile( + new URL('../data/projects.js', import.meta.url), + [ + '/**', + " * @import {Root} from 'hast'", + ' */', + '', + '/**', + ' * @typedef Project', + ' * @property {string} description', + ' * @property {Root} [descriptionRich]', + ' * @property {string} repo', + ' * @property {number} stars', + ' * @property {ReadonlyArray} topics', + ' * @property {string} [url]', + ' */', + '', + '/** @type {ReadonlyArray} */', + 'export const projects = ' + JSON.stringify(simpleProjects, undefined, 2), + '' + ].join('\n') +) - readmes.push({name: readmeName, value: readme}) +console.info('✓ done (%d projects)', projectsWithPackages.length) - return {...packageDist, repo, readmeName} - }) +const meta = {issueClosed: 0, issueOpen: 0, prClosed: 0, prOpen: 0, size: 0} +const metaKeys = /** @type {Array} */ (Object.keys(meta)) - projects = Object.values(projectsWithPackages).map(p => ({ - ...p, - default: undefined, - manifests: undefined - })) +for (const d of projectsWithPackages) { + const [owner] = d.repo.split('/') - return {...ctx, readmes, projects, packages} + if (constantCollective.includes(owner)) { + for (const key of metaKeys) { + meta[key] += d[key] + } + } } -async function writeResults(ctx) { - var {projects, packages} = ctx +await fs.writeFile( + new URL('../data/meta.js', import.meta.url), + // Types are inferred correctly. + 'export const meta = ' + JSON.stringify(meta, undefined, 2) + '\n' +) - await fs.writeFile(projectsPath, JSON.stringify(projects, null, 2) + '\n') - await fs.writeFile(packagesPath, JSON.stringify(packages, null, 2) + '\n') -} +await fs.writeFile( + new URL('../data/packages.js', import.meta.url), + [ + '/**', + " * @import {Root} from 'hast'", + ' */', + '', + '/**', + ' * @typedef Package', + ' * @property {string} [description]', + ' * @property {Root} [descriptionRich]', + ' * @property {number} downloads', + ' * @property {number} [gzip]', + ' * @property {Array} keywords', + ' * @property {string} [latest]', + ' * @property {string} [license]', + ' * @property {string} [manifestBase]', + ' * @property {string} name', + ' * @property {string} readmeName', + ' * @property {string} repo', + ' * @property {number} score', + ' */', + '', + '/** @type {ReadonlyArray} */', + 'export const packages = ' + JSON.stringify(packages, undefined, 2), + '' + ].join('\n') +) -async function writeReadmes(ctx) { - var {readmes} = ctx +console.info('✓ done (%d packages)', packages.length) + +/** + * @param {ReadonlyArray} projects + * @returns {Promise>} + */ +async function findPackages(projects) { + /** @type {Array} */ + const packages = [] + + for (const project of projects) { + for (const manifest of project.manifests) { + const {manifestBase, packageJson} = await getManifest(project, manifest) + if (!packageJson) continue + const packageInfo = await getPackage(project, manifest, packageJson) + if (!packageInfo) continue + const [readme, downloads, size] = await Promise.all([ + getReadme(project, manifestBase), + getDownloads(packageInfo.name), + getSize(packageInfo.name) + ]) + if (!readme) continue + project.hasPackages = true + + const readmeName = + packageInfo.name.replaceAll(/^@/g, '').replaceAll('/', '-') + '.md' + + await fs.writeFile( + new URL('../data/readme/' + readmeName, import.meta.url), + readme + ) - await pAll( - readmes.map(({name, value}) => () => - fs.writeFile(path.join(readmePath, name), value) - ), - {concurrency: 10} - ) + packages.push({ + description: packageInfo.description, + downloads, + gzip: size, + keywords: packageInfo.keywords, + latest: packageInfo.latest, + license: packageInfo.license, + manifestBase, + name: packageInfo.name, + readmeName, + repo: project.repo, + score: packageInfo.score + }) + } + } + + return packages } -async function searchTopic(ctx) { - var {topic, ghToken} = ctx - var matches = [] - var done = false - var after - var res - var data - - var query = ` - query($query: String!, $after: String) { - search(query: $query, type: REPOSITORY, first: 100, after: $after) { - repositoryCount +/** + * @param {string} topic + * @returns {Promise>} + */ +async function searchTopic(topic) { + /** @type {Array} */ + const matches = [] + let done = false + /** @type {string | undefined} */ + let after + + const query = ` + query($after: String, $query: String!) { + search(after: $after, first: 100, query: $query, type: REPOSITORY) { + nodes { ... on Repository { nameWithOwner } } pageInfo { hasNextPage endCursor } - nodes { - ... on Repository { nameWithOwner } - } + repositoryCount } } ` while (!done) { - // eslint-disable-next-line no-await-in-loop - res = await fetch(ghEndpoint, { - method: 'POST', + const response = await fetch(ghEndpoint, { body: JSON.stringify({ query, - variables: {query: 'sort:stars-desc topic:' + topic, after} + variables: {after, query: 'sort:stars-desc topic:' + topic} }), headers: { - 'Content-Type': 'application/json', - Authorization: 'bearer ' + ghToken - } - }).then(x => x.json()) + Authorization: 'bearer ' + ghToken, + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + const json = + /** @type {{data: {search: {nodes: Array<{nameWithOwner: string}>, pageInfo: {hasNextPage: boolean, endCursor: string}, repositoryCount: number}}}} */ ( + await response.json() + ) - data = res.data.search + const data = json.data.search if (data.pageInfo.hasNextPage) { after = data.pageInfo.endCursor @@ -212,26 +420,31 @@ async function searchTopic(ctx) { done = true } - matches = matches.concat(data.nodes.map(d => d.nameWithOwner)) + for (const d of data.nodes) { + matches.push(d.nameWithOwner) + } } - return {matches, ...ctx} + return matches } -async function searchOrg(ctx) { - var {org, ghToken} = ctx - var matches = [] - var done = false - var after - var res - var data - - var query = ` +/** + * @param {string} org + * @returns {Promise>} + */ +async function searchOrg(org) { + /** @type {Array} */ + const matches = [] + let done = false + /** @type {string | undefined} */ + let after + + const query = ` query($org: String!, $after: String) { organization(login: $org) { repositories( - first: 100 after: $after + first: 100 isFork: false isLocked: false privacy: PUBLIC @@ -250,17 +463,20 @@ async function searchOrg(ctx) { ` while (!done) { - // eslint-disable-next-line no-await-in-loop - res = await fetch(ghEndpoint, { - method: 'POST', + const response = await fetch(ghEndpoint, { body: JSON.stringify({query, variables: {org, after}}), headers: { - 'Content-Type': 'application/json', - Authorization: 'bearer ' + ghToken - } - }).then(x => x.json()) + Authorization: 'bearer ' + ghToken, + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + const json = + /** @type {{data: {organization: {repositories: {nodes: Array<{hasIssuesEnabled: boolean, isArchived: boolean, isDisabled: boolean, isTemplate: boolean, nameWithOwner: string}>, pageInfo: {hasNextPage: boolean, endCursor: string}}}}}} */ ( + await response.json() + ) - data = res.data.organization.repositories + const data = json.data.organization.repositories if (data.pageInfo.hasNextPage) { after = data.pageInfo.endCursor @@ -268,298 +484,507 @@ async function searchOrg(ctx) { done = true } - matches = matches.concat( - data.nodes - .filter( - d => - d.hasIssuesEnabled && - !d.isArchived && - !d.isDisabled && - !d.isTemplate - ) - .map(d => d.nameWithOwner) - ) + for (const d of data.nodes) { + if ( + d.hasIssuesEnabled && + !d.isArchived && + !d.isDisabled && + !d.isTemplate + ) { + matches.push(d.nameWithOwner) + } + } } - return {matches, ...ctx} + return matches } -async function crawlRepo(ctx) { - var {repo, ghToken} = ctx - var [owner, name] = repo.split('/') +/** + * @param {string} repo + * @returns {Promise<{project: RawProject, releases: Array}>} + */ +async function crawlRepo(repo) { + const [owner, name] = repo.split('/') - var res = await fetch(ghEndpoint, { - method: 'POST', + const response = await fetch(ghEndpoint, { body: JSON.stringify({ query: ` query($owner: String!, $name: String!) { repository(owner: $owner, name: $name) { + defaultBranchRef { name } + dependencyGraphManifests(withDependencies: true, first: 100) { + nodes { exceedsMaxSize filename parseable } + } + diskUsage description + issueClosed: issues(states: CLOSED) { totalCount } + issueOpen: issues(states: OPEN) { totalCount } + latestRelease { publishedAt } homepageUrl - stargazers { totalCount } - defaultBranchRef { name } + prClosed: pullRequests(states: CLOSED) { totalCount } + prOpen: pullRequests(states: OPEN) { totalCount } repositoryTopics(first: 100) { nodes { topic { name } } } - dependencyGraphManifests(withDependencies: true, first: 100) { - nodes { filename exceedsMaxSize parseable } - } + stargazers { totalCount } } } `, - variables: {owner, name} + variables: {name, owner} }), headers: { - 'Content-Type': 'application/json', + Accept: hawkgirl, Authorization: 'bearer ' + ghToken, - Accept: hawkgirl - } - }).then(x => x.json()) + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + const json = /** + * @type {({ + * errors?: Array, + * data: { + * repository?: { + * defaultBranchRef?: {name: string}, + * dependencyGraphManifests?: {nodes?: Array<{exceedsMaxSize: boolean, filename: string, parseable: boolean}>}, + * diskUsage: number + * description?: string, + * issueClosed?: {totalCount: number}, + * issueOpen?: {totalCount: number}, + * latestRelease?: {publishedAt: string}, + * homepageUrl: string, + * prClosed?: {totalCount: number}, + * prOpen?: {totalCount: number}, + * repositoryTopics: {nodes: Array<{topic: {name: string}}>} + * stargazers: {totalCount: number} + * } + * } + * })} + */ (await response.json()) // Manifests aren’t always loaded, giving errors here: print them. - if (res.errors) { + if (json.errors) { console.warn( '%s: non-exceptional errors (probably loading manifests):', repo, - res.errors + json.errors ) } - var data = (res.data || {}).repository || {} - var defaultBranch = (data.defaultBranchRef || {}).name || null - - var project = { - repo, - description: data.description || '', - stars: (data.stargazers || {}).totalCount || 0, - default: defaultBranch, - url: data.homepageUrl || null, - topics: ((data.repositoryTopics || {}).nodes || []) - .map(d => d.topic.name) - .filter(validTag) - .filter(unique), - manifests: ((data.dependencyGraphManifests || {}).nodes || []) - .filter( - d => - defaultBranch && - path.posix.basename(d.filename) === 'package.json' && - d.parseable && - !d.exceedsMaxSize + const repository = json?.data?.repository + const defaultBranch = repository?.defaultBranchRef?.name || undefined + + const lastReleaseAt = repository?.latestRelease + ? new Date(repository.latestRelease.publishedAt) + : undefined + + /** @type {Array} */ + const releases = [] + + if (lastReleaseAt && recentRelease(lastReleaseAt)) { + const releaseResponse = await fetch(ghEndpoint, { + body: JSON.stringify({ + query: ` + query($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + releases(first: 20, orderBy: {direction: DESC, field: CREATED_AT}) { + nodes { description publishedAt tagName url } + } + } + } + `, + variables: {name, owner} + }), + headers: { + Authorization: 'bearer ' + ghToken, + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + + const releaseJson = + /** @type {{data?: {repository?: {releases?: {nodes?: Array<{description: string, publishedAt: string, tagName: string, url: string }>}}}}} */ ( + await releaseResponse.json() ) - .map(d => d.filename) + + const data = releaseJson?.data?.repository + const nodes = data?.releases?.nodes || [] + + for (const d of nodes) { + if (!recentRelease(new Date(d.publishedAt))) continue + + releases.push({ + description: d.description, + published: d.publishedAt, + repo, + tag: d.tagName + }) + } } - return {...ctx, project} -} + /** @type {Array} */ + const manifests = [] + + if (defaultBranch && repository?.dependencyGraphManifests?.nodes) { + for (const d of repository.dependencyGraphManifests.nodes) { + if ( + d.filename.endsWith('package.json') && + d.parseable && + !d.exceedsMaxSize + ) { + manifests.push(d.filename) + } + } + } -async function getManifest(ctx) { - var {project, manifest, ghToken} = ctx - var {repo} = project - var [owner, name] = repo.split('/') - var target = [project.default || 'master', manifest].join(':') - var manifestBase = path.dirname(manifest) - var res + /** @type {Array} */ + const topics = [] - if (manifestBase === '.') { - manifestBase = undefined + if (repository?.repositoryTopics?.nodes) { + for (const d of repository.repositoryTopics.nodes) { + const name = d.topic.name + if (validTag(name)) { + topics.push(name) + } + } } + return { + project: { + default: defaultBranch, + description: repository?.description || '', + hasPackages: false, + issueClosed: repository?.issueClosed?.totalCount || 0, + issueOpen: repository?.issueOpen?.totalCount || 0, + manifests, + prClosed: repository?.prClosed?.totalCount || 0, + prOpen: repository?.prOpen?.totalCount || 0, + repo, + // Size of repo in bytes. + size: (repository?.diskUsage || 0) * 1024, + stars: repository?.stargazers?.totalCount || 0, + topics, + url: repository?.homepageUrl || undefined + }, + releases + } +} + +/** + * @param {RawProject} project + * @param {string} manifest + * @returns {Promise<{manifestBase: string | undefined, packageJson: PackageJson | undefined}>} + */ +async function getManifest(project, manifest) { + const {repo} = project + const [owner, name] = repo.split('/') + const target = [project.default || 'master', manifest].join(':') + const parts = manifest.split('/') + const tail = parts.pop() + assert(tail === 'package.json') + const manifestBase = parts.join('/') || undefined + /** @type {string | undefined} */ + let packageJsonText + /** @type {PackageJson | undefined} */ + let packageJson + try { - res = await fetch(ghEndpoint, { - method: 'POST', + const response = await fetch(ghEndpoint, { body: JSON.stringify({ query: ` - query($owner: String!, $name: String!, $target: String!) { - repository(owner: $owner, name: $name) { + query($name: String!, $owner: String!, $target: String!) { + repository(name: $name, owner: $owner) { object(expression: $target) { ... on Blob { text } } } } `, - variables: {owner, name, target} + variables: {name, owner, target} }), headers: { - 'Content-Type': 'application/json', - Authorization: 'bearer ' + ghToken - } - }).then(x => x.json()) + Authorization: 'bearer ' + ghToken, + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + const result = + /** @type {{data: {repository: {object?: {text: string}}}} | GHError} */ ( + await response.json() + ) + + if ('data' in result && result.data) { + packageJsonText = result.data.repository.object?.text + } else { + console.warn('%s#%s: no data:', repo, manifest, result) + } } catch (error) { - console.warn('Could not fetch manifest:', error) + console.warn('%s#%s: could not fetch manifest:', repo, manifest, error) } - var proper = true - var pkg - try { - pkg = JSON.parse((res.data.repository.object || {}).text) - } catch (_) { + packageJson = packageJsonText ? JSON.parse(packageJsonText) : undefined + } catch { console.warn('%s#%s: could not parse manifest', repo, manifest) - proper = false } - if (pkg && !pkg.name) { - console.warn('%s#%s: ignoring manifest without name', repo, manifest) - proper = false - } else if (pkg && pkg.private) { - console.warn('%s#%s: ignoring private manifest', repo, manifest) - proper = false - } else if (pkg && /gatsby/i.test(pkg.name)) { - console.warn('%s#%s: ignoring gatsby-related package', repo, manifest) - proper = false + if (packageJson) { + if (!packageJson.name) { + console.warn('%s#%s: ignoring manifest without name', repo, manifest) + packageJson = undefined + } else if (packageJson.private) { + console.warn('%s#%s: ignoring private manifest', repo, manifest) + packageJson = undefined + } else if (/gatsby/i.test(packageJson.name)) { + console.warn('%s#%s: ignoring gatsby-related package', repo, manifest) + packageJson = undefined + } } - return {...ctx, proper, manifestBase, packageSource: pkg} + return {manifestBase, packageJson} } -async function getPackage(ctx) { - var {proper, manifest, manifestBase, project, packageSource} = ctx - var {repo} = project - var res - - if (!proper) { - return - } +/** + * @param {RawProject} project + * @param {string} manifest + * @param {PackageJson} packageJson + * @returns {Promise} + */ +async function getPackage(project, manifest, packageJson) { + assert(packageJson.name) // Checked earlier. + // const {manifestBase, project} = ctx + const {repo} = project + /** @type {NpmsPackageResult | undefined} */ + let body - res = await fetch( - [npmsEndpoint, encodeURIComponent(packageSource.name)].join('/') - ).then(x => x.json()) + try { + const response = await fetch( + [npmsEndpoint, encodeURIComponent(packageJson.name)].join('/') + ) + body = /** @type {NpmsPackageResult} */ (await response.json()) + } catch {} - if (res.code === 'NOT_FOUND') { - console.warn('%s#%s: could not find package', repo, manifest) - ctx.proper = false + if (!body || !body.collected || !body.score) { + console.warn('%s#%s: could not connect to npms', repo, manifest) return } - var name = res.collected.metadata.name || '' - var description = res.collected.metadata.description || '' - var keywords = res.collected.metadata.keywords || [] - var license = res.collected.metadata.license || null - var deprecated = res.collected.metadata.deprecated - var readme = res.collected.metadata.readme || '' - var latest = res.collected.metadata.version || null - var repos = res.collected.metadata.repository - var url = (repos && repos.url) || '' - var dependents = res.collected.npm.dependentsCount || 0 - var score = res.score.final || 0 - - if (deprecated) { + // To do: this would probably go higher? If there’s an error returned? + // if (body.code === 'NOT_FOUND') { + // console.warn('%s#%s: could not find package (on npms)', repo, manifest) + // return + // } + + const name = body.collected.metadata.name || undefined + const description = body.collected.metadata.description || undefined + const keywords = body.collected.metadata.keywords || [] + const license = body.collected.metadata.license || undefined + const latest = body.collected.metadata.version || undefined + const repository = body.collected.metadata.repository + const url = (repository && repository.url) || undefined + const score = body.score.final || 0 + // Note: we used to look at `dependents`, but that’s always on `0` apparently now? + + if (!name) return + + if (body.collected.metadata.deprecated) { console.warn( '%s#%s: ignoring deprecated package: %s', repo, manifest, - deprecated + body.collected.metadata.deprecated ) - ctx.proper = false - return - } - - if (!readme || readme.length < 20) { - console.warn('%s#%s: ignoring package without readme', repo, manifest) - ctx.proper = false return } if (!url) { console.warn('%s#%s: ignoring unknown repo', repo, manifest) - ctx.proper = false return } - var info = hostedGitInfo.fromUrl(url) + const info = hostedGitInfo.fromUrl(url) if (!info) { console.warn('%s#%s: ignoring non-parsable repo: %s', repo, manifest, url) - ctx.proper = false return } if (info.type !== 'github') { console.warn('%s#%s: ignoring non-github repo: %s', repo, manifest, url) - ctx.proper = false return } - var slug = [info.user, info.project].join('/') + const slug = [info.user, info.project].join('/') if (slug !== repo) { console.warn('%s#%s: ignoring mismatched repos: %s', repo, manifest, url) - ctx.proper = false return } - keywords = keywords.filter(validTag).filter(unique) + /** @type {Array} */ + const validKeywords = [] - return { - ...ctx, - readme, - packageDist: { - name, - manifestBase, - latest, - description, - keywords, - license, - dependents, - score + for (const d of keywords) { + if (validTag(d) && !validKeywords.includes(d)) { + validKeywords.push(d) } } + + return { + description, + keywords: validKeywords, + latest, + license, + name, + score + } } -async function getDownloads(ctx) { - var {proper, packageDist, npmToken} = ctx +/** + * @param {RawProject} project + * @param {string | undefined} manifestBase + * @returns {Promise} + */ +async function getReadme(project, manifestBase) { + const {repo} = project + const [owner, name] = repo.split('/') + let base = (project.default || 'master') + ':' + + if (manifestBase) base += manifestBase + '/' + + // Instead of going through the folder and looking for the first that matches + // `/^readme(?=\.|$)/i`, we throw the frequently used ones at GH. + const response = await fetch(ghEndpoint, { + body: JSON.stringify({ + query: ` + query($cmd: String!, $c: String!, $lmd: String!, $l: String!, $name: String!, $owner: String!, $umd: String!, $u: String!) { + repository(owner: $owner, name: $name) { + cmd: object(expression: $cmd) { ... on Blob { text } } + c: object(expression: $c) { ... on Blob { text } } + lmd: object(expression: $lmd) { ... on Blob { text } } + l: object(expression: $l) { ... on Blob { text } } + umd: object(expression: $umd) { ... on Blob { text } } + u: object(expression: $u) { ... on Blob { text } } + } + } + `, + variables: { + cmd: base + 'Readme.md', + c: base + 'Readme', + lmd: base + 'readme.md', + l: base + 'readme', + name, + owner, + umd: base + 'README.md', + u: base + 'README' + } + }), + headers: { + Authorization: 'bearer ' + ghToken, + 'Content-Type': 'application/json' + }, + method: 'POST' + }) + const result = + /** @type {{data: {repository: {umd?: {text: string}, u?: {text: string}, cmd?: {text: string}, c?: {text: string}, lmd?: {text: string}, l?: {text: string}}}}} */ ( + await response.json() + ) - if (!proper) { + const repository = result?.data?.repository + const object = + repository?.umd || + repository?.u || + repository?.cmd || + repository?.c || + repository?.lmd || + repository?.l + + if (!object) { + console.warn('%s#%s: could not find readme', repo, base) return } - var endpoint = [ + const readme = object?.text + + if (!readme || readme.length < 20) { + console.warn('%s#%s: ignoring package without readme', repo, base) + return + } + + return readme +} + +/** + * @param {string} name + * @returns {Promise} + */ +async function getDownloads(name) { + const endpoint = [ npmDownloadsEndpoint, 'point', 'last-month', - packageDist.name + encodeURIComponent(name) ].join('/') - var res = await fetch(endpoint, { - headers: {Authorization: 'Bearer ' + npmToken} - }).then(x => x.json()) + const response = await fetch(endpoint, { + // Passing an npm token recently seems to crash npm. + // headers: {Authorization: 'Bearer ' + npmToken} + }) + const result = + /** @type {{downloads: number, end: string, start: string}} */ ( + await response.json() + ) - ctx.packageDist = {...ctx.packageDist, downloads: res.downloads} + return result.downloads } -async function getSize(ctx) { - var {proper, manifest, project, packageDist} = ctx - var {repo} = project - var res - - if (!proper) { - return - } - - var endpoint = - bundlePhobiaEndpoint + - '?package=' + - encodeURIComponent(packageDist.name + '@' + packageDist.latest) +/** + * @param {string} name + * @returns {Promise} + */ +async function getSize(name) { + /** @type {string | undefined} */ + let value try { - res = await fetch(endpoint).then(x => x.json()) + const response = await fetch( + 'https://img.shields.io/bundlephobia/minzip/' + + encodeURIComponent(name) + + '.json', + { + headers: {'User-Agent': randomUseragent.getRandom()} + } + ) + const result = + /** @type {{color: string, label: string, link: unknown, message: string, name: string, value: string}} */ ( + await response.json() + ) + value = result.value } catch (error) { - console.warn('%s#%s: could not contact bundlephobia', repo, manifest, error) - // Still “proper”. + console.warn('%s: could not contact `shields.io`:', name, error) return } - var gzip = res.gzip - var esm = Boolean(res.hasJSModule) - var dependencies = res.dependencyCount - - ctx.packageDist = {...ctx.packageDist, gzip, esm, dependencies} + // I’m not 100% why exactly but this is how bundlephobia’s JSON converts to + // what it displays on the site: + // * https://bundlephobia.com/api/size?package=micromark@3.0.0 = 14273 + // * https://bundlephobia.com/package/micromark@3.0.0 = 13.9kb + return (((bytes.parse(value) / 1024) * 1000) / 1024) * 1000 } -function unique(d, i, data) { - return data.indexOf(d) === i +/** + * @param {string} d + * @returns {boolean} + */ +function validTag(d) { + return /^[a-zA-Z\d-]+$/.test(d) } -function validTag(d) { - return /^[a-zA-Z0-9-]+$/.test(d) +/** + * Whether this release was in the last 60 days. + * + * @param {Date} date + * @returns {boolean} + */ +function recentRelease(date) { + return date.valueOf() > Date.now() - 60 * 24 * 60 * 60 * 1000 } diff --git a/crawl/github-sponsors.js b/crawl/github-sponsors.js new file mode 100644 index 000000000000..c383f32cffa9 --- /dev/null +++ b/crawl/github-sponsors.js @@ -0,0 +1,130 @@ +/** + * @typedef GithubOrganizationData + * Github organization data. + * @property {{nodes: ReadonlyArray>}} lifetimeReceivedSponsorshipValues + * Sponsorships. + * + * @typedef GithubSponsor + * GitHub sponsor. + * @property {string} avatarUrl + * Avatar URL. + * @property {string | null | undefined} [bio] + * Bio. + * @property {string | null | undefined} [description] + * Description. + * @property {string} login + * Username. + * @property {string | null} name + * Name. + * @property {string | null} websiteUrl + * URL. + * + * @typedef GithubSponsorNode + * GitHub sponsor node. + * @property {number} amountInCents + * Total price. + * @property {Readonly} sponsor + * Sponsor. + * + * @typedef GithubSponsorsResponse + * GitHub sponsors response. + * @property {{organization: Readonly}} data + * Data. + * + * @typedef SponsorRaw + * Sponsor (raw). + * @property {string | undefined} description + * Description. + * @property {string} github + * GitHub username. + * @property {string} image + * Image. + * @property {number} total + * Total amount. + * @property {string | undefined} name + * Name. + * @property {string | undefined} url + * URL. + */ + +import fs from 'node:fs/promises' +import process from 'node:process' +import dotenv from 'dotenv' + +dotenv.config() + +const key = process.env.GH_TOKEN + +if (!key) throw new Error('Missing `GH_TOKEN`') + +const outUrl = new URL('../data/github-sponsors.json', import.meta.url) + +const endpoint = 'https://api.github.com/graphql' + +// To do: paginate. +const query = `query($org: String!) { + organization(login: $org) { + lifetimeReceivedSponsorshipValues(first: 100, orderBy: {field: LIFETIME_VALUE, direction: DESC}) { + nodes { + amountInCents + sponsor { + ... on Organization { avatarUrl description login name websiteUrl } + ... on User { avatarUrl bio login name websiteUrl } + } + } + } + } +} +` + +const response = await fetch(endpoint, { + body: JSON.stringify({query, variables: {org: 'unifiedjs'}}), + headers: {Authorization: 'bearer ' + key, 'Content-Type': 'application/json'}, + method: 'POST' +}) +const body = /** @type {Readonly} */ ( + await response.json() +) + +const collective = + body.data.organization.lifetimeReceivedSponsorshipValues.nodes + .map(function (d) { + return clean(d) + }) + .sort(sort) + // `10` dollar minimum. + .filter(function (d) { + return d.total >= 10 + }) + +await fs.mkdir(new URL('./', outUrl), {recursive: true}) +await fs.writeFile(outUrl, JSON.stringify(collective, undefined, 2) + '\n') + +/** + * @param {Readonly} d + * Sponsor node. + * @returns {SponsorRaw} + * Sponsor. + */ +function clean(d) { + return { + description: d.sponsor.bio || d.sponsor.description || undefined, + github: d.sponsor.login, + image: d.sponsor.avatarUrl, + total: Math.floor(d.amountInCents / 100), + name: d.sponsor.name || undefined, + url: d.sponsor.websiteUrl || undefined + } +} + +/** + * @param {Readonly} a + * Left. + * @param {Readonly} b + * Right. + * @returns {number} + * Sort order. + */ +function sort(a, b) { + return b.total - a.total +} diff --git a/crawl/opencollective.js b/crawl/opencollective.js new file mode 100644 index 000000000000..15438b0f1669 --- /dev/null +++ b/crawl/opencollective.js @@ -0,0 +1,203 @@ +/** + * @typedef OcAccount + * Open Collective account. + * @property {string | undefined} description + * Description. + * @property {string | undefined} githubHandle + * GitHub username. + * @property {string} id + * ID. + * @property {string} imageUrl + * Image URL. + * @property {string} name + * Name. + * @property {string} slug + * Slug. + * @property {string | undefined} website + * Website. + * + * @typedef OcCollective + * Open Collective collective. + * @property {{nodes: ReadonlyArray>}} members + * Members. + * + * @typedef OcData + * Open Collective data. + * @property {Readonly} collective + * Collective. + * + * @typedef OcMember + * Open Collective member. + * @property {Readonly} account + * Account. + * @property {Readonly<{value: number}>} totalDonations + * Total donations. + * + * @typedef OcResponse + * Open Collective response. + * @property {Readonly} data + * Data. + * + * @typedef {Omit} Sponsor + * Sponsor. + * + * @typedef SponsorRaw + * Sponsor (raw). + * @property {string | undefined} [description] + * Description. + * @property {string | undefined} [github] + * GitHub username. + * @property {string} image + * Image. + * @property {string} name + * Name. + * @property {string} oc + * Open Collective slug. + * @property {boolean} spam + * Whether it’s spam. + * @property {number} total + * Total donations. + * @property {string | undefined} [url] + * URL. + */ + +import fs from 'node:fs/promises' +import process from 'node:process' +import dotenv from 'dotenv' + +dotenv.config() + +const key = process.env.OC_TOKEN + +if (!key) throw new Error('Missing `OC_TOKEN`') + +const min = 10 + +const endpoint = 'https://api.opencollective.com/graphql/v2' + +const variables = {slug: 'unified'} + +const ghBase = 'https://github.com/' + +// To do: paginate. +const query = `query($slug: String) { + collective(slug: $slug) { + members(limit: 100, role: BACKER) { + nodes { + account { + description + githubHandle + id + imageUrl + name + slug + website + } + totalDonations { value } + } + } + } +} +` + +const sponsorsTxt = await fs.readFile( + new URL('sponsors.txt', import.meta.url), + 'utf8' +) + +const collectiveResponse = await fetch(endpoint, { + body: JSON.stringify({query, variables}), + headers: {'Api-Key': key, 'Content-Type': 'application/json'}, + method: 'POST' +}) +const collectiveBody = /** @type {Readonly} */ ( + await collectiveResponse.json() +) + +/** @type {Array<{oc: string, spam: boolean}>} */ +const control = [] + +for (const d of sponsorsTxt.split('\n')) { + const spam = d.charAt(0) === '-' + + control.push({oc: spam ? d.slice(1) : d, spam}) +} + +/** @type {Set} */ +const seen = new Set() +/** @type {Array} */ +const members = [] + +for (const d of collectiveBody.data.collective.members.nodes) { + const oc = d.account.slug + const github = d.account.githubHandle || undefined + let url = d.account.website || undefined + const info = control.find(function (d) { + return d.oc === oc + }) + + if (url === ghBase + github) { + url = undefined + } + + if (!info) { + console.error( + '✖ @%s is an unknown sponsor, please define whether it’s spam or not in `sponsors.txt`', + oc + ) + } + + /** @type {Readonly} */ + const person = { + description: d.account.description || undefined, + github, + image: d.account.imageUrl, + name: d.account.name, + oc, + spam: !info || info.spam, + total: d.totalDonations.value, + url + } + + const ignore = person.spam || seen.has(person.oc) // Ignore dupes in data. + seen.add(person.oc) + + if (person.total > min && !ignore) { + members.push(person) + } +} + +members.sort(sort) + +/** @type {Array} */ +const stripped = [] + +for (const d of members) { + const {spam, ...rest} = d + stripped.push(rest) +} + +await fs.writeFile( + new URL('../data/opencollective.js', import.meta.url), + [ + '/**', + ' * @import {Sponsor} from "../crawl/opencollective.js"', + ' */', + '', + '/** @type {Array} */', + 'export const sponsors = ' + JSON.stringify(stripped, undefined, 2), + '' + ].join('\n') +) + +/** + * @param {Readonly} a + * Left. + * @param {Readonly} b + * Right. + * @returns {number} + * Sort value. + */ +function sort(a, b) { + return b.total - a.total +} diff --git a/crawl/sponsors.js b/crawl/sponsors.js deleted file mode 100644 index fb8a620e98c4..000000000000 --- a/crawl/sponsors.js +++ /dev/null @@ -1,95 +0,0 @@ -var fs = require('fs').promises -var path = require('path') -var fetch = require('node-fetch') - -require('dotenv').config() - -var token = process.env.OC_TOKEN - -if (!token) { - console.log('Cannot crawl sponsors without OC token') - /* eslint-disable-next-line unicorn/no-process-exit */ - process.exit() -} - -var outpath = path.join('data', 'sponsors.json') -var min = 5 - -var endpoint = 'https://api.opencollective.com/graphql/v2' - -var variables = {slug: 'unified'} - -var ghBase = 'https://github.com/' -var twBase = 'https://twitter.com/' - -var query = `query($slug: String) { - collective(slug: $slug) { - members(limit: 100, role: BACKER) { - nodes { - totalDonations { value } - tier { name } - account { - id - slug - name - description - website - twitterHandle - githubHandle - imageUrl - } - } - } - } -} -` - -fetch(endpoint, { - method: 'POST', - body: JSON.stringify({query: query, variables: variables}), - headers: { - 'Content-Type': 'application/json', - 'Api-Key': token - } -}) - .then(res => res.json()) - .then(function(res) { - var seen = [] - var members = res.data.collective.members.nodes - .map(d => { - var github = d.account.githubHandle || undefined - var twitter = d.account.twitterHandle || undefined - var url = d.account.website || undefined - - if (url === ghBase + github || url === twBase + twitter) { - url = undefined - } - - return { - name: d.account.name, - description: d.account.description || undefined, - image: d.account.imageUrl, - oc: d.account.slug, - github, - twitter, - url, - gold: - (d.tier && d.tier.name && /gold/i.test(d.tier.name)) || undefined, - amount: d.totalDonations.value - } - }) - .filter(d => { - var ignore = seen.includes(d.oc) // Ignore dupes in data. - seen.push(d.oc) - return d.amount > min && !ignore - }) - .sort(sort) - .map(d => Object.assign(d, {amount: undefined})) - - return fs.writeFile(outpath, JSON.stringify(members, null, 2) + '\n') - }) - .catch(console.error) - -function sort(a, b) { - return b.amount - a.amount -} diff --git a/crawl/sponsors.txt b/crawl/sponsors.txt new file mode 100644 index 000000000000..9acee26efd3b --- /dev/null +++ b/crawl/sponsors.txt @@ -0,0 +1,70 @@ +-baocasino +-guest-c78d32a1 +-guest-408805a9 +-github-1 +-github-sponsors +-greenpromocode-com +-incognito-08f32cbb +-kasinosivut +-masonslots +-user-134c19fe +1337lawyers +alex2 +antfu +asset-web +balsa +bnb +boosthub +cocopon +codiumai +compositor +daanvanderzwaag +danburzo +danoc +dopplerteam +emmatown +expo1 +fred-edwards +frontendmasters +gatsbyjs +gitbook +hashicorp +holloway +indeed +jin-zhao +jonathan-haines +jpoehnelt +jrfnl +knksmith57 +kvnsmth +lars-trieloff +markdown-space +matt-vague +mgan +mitchellhamilton +motif +netlify +odiak +opencollective +radwebhosting +ricky +see-yishu +shawnbot +sindresorhus-unicorn +stackaid +themeisle +theodorechu +thinkmill +tomasz-czajecki +torutek +travisarnold +triplebyte +tripwire +valtown +vercel +vhf +vincent-kempers +voxpelli +xdamman +yangshun +znarf diff --git a/crawl/team.js b/crawl/team.js index 1922dfc9b23f..b4140927a748 100644 --- a/crawl/team.js +++ b/crawl/team.js @@ -1,29 +1,64 @@ -var fs = require('fs').promises -var {join, basename, extname} = require('path') -var yaml = require('js-yaml') -var fetch = require('node-fetch') +import fs from 'node:fs/promises' +import process from 'node:process' +import dotenv from 'dotenv' +import yaml from 'yaml' -require('dotenv').config() +dotenv.config() -var ghToken = process.env.GH_TOKEN +const ghToken = process.env.GH_TOKEN if (!ghToken) { - console.log('Cannot crawl team without GH token') - /* eslint-disable-next-line unicorn/no-process-exit */ + console.error('Cannot crawl team without GH token') process.exit() } -var headers = {Authorization: 'bearer ' + ghToken} +const base = 'https://raw.githubusercontent.com/unifiedjs/collective/HEAD/data/' +const humans = 'humans.yml' +const files = [humans, 'teams.yml'] -var base = 'https://raw.githubusercontent.com/unifiedjs/collective/master/data/' -var files = ['humans.yml', 'teams.yml'] +const humansTypes = [ + '/**', + ' * @typedef Human', + ' * @property {string} name', + ' * @property {string} [email]', + ' * @property {string} [url]', + ' * @property {string} github', + ' * @property {string} npm', + ' */', + '', + '/** @type {ReadonlyArray} */', + '' +].join('\n') -files.forEach(filename => - fetch(base + filename, {headers}) - .then(d => d.text()) - .then(d => { - var stem = basename(filename, extname(filename)) - var data = JSON.stringify(yaml.safeLoad(d), null, 2) + '\n' - return fs.writeFile(join('data', stem + '.json'), data) - }) -) +const teamsTypes = [ + '/**', + " * @typedef {'contributor' | 'maintainer' | 'merger' | 'releaser'} Role", + ' *', + ' * @typedef Team', + ' * @property {true} [collective]', + ' * @property {Record} humans', + ' * @property {string} [lead]', + ' * @property {string} name', + ' */', + '', + '/** @type {ReadonlyArray} */', + '' +].join('\n') + +for (const filename of files) { + const response = await fetch(base + filename, { + headers: {Authorization: 'bearer ' + ghToken} + }) + const d = await response.text() + const stem = filename.replace(/\.[a-z]+$/i, '') + await fs.writeFile( + new URL('../data/' + stem + '.js', import.meta.url), + [ + filename === humans ? humansTypes : teamsTypes, + 'export const ' + + stem + + ' = ' + + JSON.stringify(yaml.parse(d), undefined, 2) + ].join('\n') + ) +} diff --git a/data/meta.json b/data/meta.json new file mode 100644 index 000000000000..6a5176c985c0 --- /dev/null +++ b/data/meta.json @@ -0,0 +1,7 @@ +{ + "size": 123123, + "issueOpen": 12, + "issueClosed": 34, + "prOpen": 56, + "prClosed": 78 +} diff --git a/data/releases.json b/data/releases.json new file mode 100644 index 000000000000..98fabddb19d6 --- /dev/null +++ b/data/releases.json @@ -0,0 +1,8 @@ +[ + { + "repo": "example/example", + "published": "2021-02-22T00:00:00Z", + "tag": "1.9.0", + "description": ":100:" + } +] diff --git a/dictionary.txt b/dictionary.txt index 2b33ccda24bb..c603fa0da515 100644 --- a/dictionary.txt +++ b/dictionary.txt @@ -1,45 +1,53 @@ // Jargon -API +APIs +ASTs +AST +CLI +XSS attacher +bundlers bundler -CLI -debounced -hacky -linter -math -middleware +esbuild minified minify minifying performant -pluggable +plugable programmatically readme -stdin -stdout -stringifier +shortcodes +shortcode stringify syntaxes +whitespace // Names, products, etc. +ATX BundlePhobia -browserify +CDN +CommonMark DOM +GFM HSL -hast +ISC +JSDoc +JSON JSX -MacBook +MDXs MDX +MacBook +Otander +PostCSS +Preact +Setext +gemoji mdast -Node.js nlcst npm -Otander rehype -remark retext unist -unified vfile +webpack xast xo diff --git a/doc/learn/build-a-syntax-tree.md b/doc/learn/build-a-syntax-tree.md new file mode 100644 index 000000000000..d51f0665a1b9 --- /dev/null +++ b/doc/learn/build-a-syntax-tree.md @@ -0,0 +1,158 @@ +--- +authorGithub: ChristianMurphy +author: Christian Murphy +description: How to build content with syntax trees +group: recipe +index: 8 +modified: 2024-08-02 +published: 2020-06-09 +tags: + - esast + - hast + - mdast + - nlcst + - unist + - xast +title: Build a syntax tree +--- + +## How to build a syntax tree + +It’s often useful to build new (fragments of) syntax trees when adding or +replacing content. +It’s possible to create trees with plain object and array literals (JSON) or +programmatically with a small utility. +Finally it’s even possible to use JSX to build trees. + +### JSON + +The most basic way to create a tree is with plain object and arrays. +To prevent type errors, this can be checked with the types for the given syntax +tree language, in this case mdast: + +```ts twoslash +import type {Root} from 'mdast' + +// Note the `: Root` is a TypeScript annotation. +// For plain JavaScript, remove it (and the import). +const mdast: Root = { + type: 'root', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'text', + value: 'example' + } + ] + } + ] +} +``` + +#### `unist-builder` + +It’s also possible to build trees with [`unist-builder`][u]. +It allows a more concise, “hyperscript” like syntax (which is also like +`React.createElement`): + +```js twoslash +/** + * @import {Root} from 'mdast' + */ + +import {u} from 'unist-builder' + +/** @type {Root} */ +const mdast = u('root', [ + u('paragraph', [ + u('text', 'example') + ]) +]) +``` + +#### `hastscript` + +When working with hast (HTML), [`hastscript`][h] can be used. + +```js twoslash +/// +// ---cut--- +import {h, s} from 'hastscript' + +console.log( + h('div#some-id.foo', [ + h('span', 'some text'), + h('input', {type: 'text', value: 'foo'}), + h('a.alpha.bravo.charlie', {download: true}, 'delta') + ]) +) + +// SVG: +console.log( + s('svg', {viewBox: '0 0 500 500', xmlns: 'http://www.w3.org/2000/svg'}, [ + s('title', 'SVG `` element'), + s('circle', {cx: 120, cy: 120, r: 100}) + ]) +) +``` + +`hastscript` can also be used as a JSX configuration comment: + +```tsx twoslash +/// +// ---cut--- +/** @jsxImportSource hastscript */ + +console.log( +
+ + + +
+) +``` + +#### `xastscript` + +When working with xast (XML), [`xastscript`][x] +can be used. + +```js twoslash +/// +// ---cut--- +import {x} from 'xastscript' + +console.log( + x('album', {id: 123}, [ + x('name', 'Exile in Guyville'), + x('artist', 'Liz Phair'), + x('releasedate', '1993-06-22') + ]) +) +``` + +`xastscript` can also be used as a JSX configuration comment: + +```tsx twoslash +/// +// ---cut--- +/** @jsxImportSource xastscript */ + +console.log( + + Born in the U.S.A. + Bruce Springsteen + 1984-04-06 + +) +``` + + + +[u]: https://github.com/syntax-tree/unist-builder + +[h]: https://github.com/syntax-tree/hastscript + +[x]: https://github.com/syntax-tree/xastscript diff --git a/doc/learn/create-a-plugin.md b/doc/learn/create-a-plugin.md index 4e6cfab14a9d..bb35b06ac632 100644 --- a/doc/learn/create-a-plugin.md +++ b/doc/learn/create-a-plugin.md @@ -1,289 +1,19 @@ --- -group: guide -title: Create a plugin -description: Guide that shows how to create a (retext) plugin +archive: true +authorGithub: wooorm author: Titus Wormer -authorTwitter: wooorm +description: Guide that shows how to create a plugin +group: guide +modified: 2024-08-13 +published: 2017-05-03 tags: - plugin - - retext -published: 2017-05-03 -modified: 2019-12-12 +title: Create a plugin --- -## Creating a plugin with unified - -This guide shows how to create a plugin for retext that checks the amount of -spaces between sentences. -The concepts here apply to the other syntaxes of unified as well. - -> Stuck? -> A good place to get help fast is [Spectrum][]. -> Have an idea for another guide? -> Share it on spectrum! - -### Contents - -* [Plugin basics](#plugin-basics) -* [Case](#case) -* [Setting up](#setting-up) -* [Plugin](#plugin) -* [Further exercises](#further-exercises) - -### Plugin basics - -A unified plugin changes the way the applied-on processor works, in several -ways. -In this guide we’ll review how to inspect syntax trees. - -Plugins can contain two parts: an **attacher**, which is a function that is -invoked when someone calls `.use`, and a **transformer**, which is an optional -function invoked each time a file is processed with a syntax tree and a virtual -file. - -In this case, we want to check the syntax tree of each processed file, so we do -specify a transformer. - -Now you know the basics of plugins in unified. -On to our case! - -### Case - -Before we start, let’s first outline what we want to make. -Say we have the following text file: - -```markdown -One sentence. Two sentences. - -One sentence. Two sentences. -``` - -We want to get a warning for the second paragraph, saying that one space instead -of two spaces should be used. - -In the next step we’ll write the code to use our plugin. - -### Setting up - -Now, let’s create an `example.js` file that will process our text file and -report any found problems. - -```javascript -var fs = require('fs') -var retext = require('retext') -var report = require('vfile-reporter') -var spacing = require('.') - -var doc = fs.readFileSync('example.md') - -retext() - .use(spacing) - .process(doc, function(err, file) { - console.error(report(err || file)) - }) -``` - -> Don’t forget to `npm install` dependencies! - -If you read the guide on [using unified][use], you’ll see some familiar -statements. -First, we load dependencies, then we read the file in. -We process that file with the plugin we’ll create in a second, and finally we -report either a fatal error or any found linting messages. - -Note that we directly depend on retext. -This is a package that exposes a unified processor, and comes with the parser -and compiler attached. - -When running our example (it doesn’t work yet though) we want to see a message -for the second paragraph, saying that one space instead of two spaces should be -used. - -Now we’ve got everything set up except for the plugin itself. -We’ll do that in the next section. - -### Plugin - -As we read in Plugin Basics, we’ll need an attacher, and for our case also a -transformer. -Let’s create them in our plugin file `index.js`: - -```javascript -module.exports = attacher - -function attacher() { - return transformer - - function transformer(tree, file) { - } -} -``` - -First things first, we need to check `tree` for a pattern. -We can use a utility to help us to recursively walk our tree, namely -[`unist-util-visit`][visit]. -Let’s add that. - -```diff -+var visit = require('unist-util-visit') -+ -module.exports = attacher - -function attacher() { - return transformer - - function transformer(tree, file) { -+ visit(tree, 'ParagraphNode', visitor) -+ -+ function visitor(node) { -+ console.log(node) -+ } - } -} -``` - -> Don’t forget to `npm install` the utility! - -If we now run our example with Node.js, as follows, we’ll see that visitor is -invoked with both paragraphs in our example: - -```sh -$ node example.js -{ type: 'ParagraphNode', - children: - [ { type: 'SentenceNode', children: [Object] }, - { type: 'WhiteSpaceNode', value: ' ' }, - { type: 'SentenceNode', children: [Object] } ] } -{ type: 'ParagraphNode', - children: - [ { type: 'SentenceNode', children: [Object] }, - { type: 'WhiteSpaceNode', value: ' ' }, - { type: 'SentenceNode', children: [Object] } ] } -no issues found -``` - -This output already shows that paragraphs contain two types of nodes: -`SentenceNode` and `WhiteSpaceNode`. -The latter is what we want to check, but the former is important because we only -warn about white-space between sentences in this plugin (that could be another -plugin though). - -Let’s now loop through the children of each paragraph. -Only checking white-space between sentences. -We use a small utility for checking node types: [`unist-util-is`][is]. - -```diff -var visit = require('unist-util-visit') -+var is = require('unist-util-is') - -module.exports = attacher - -function attacher() { - return transformer - - function transformer(tree, file) { - visit(tree, 'ParagraphNode', visitor) - - function visitor(node) { -- console.log(node) -+ var children = node.children -+ -+ children.forEach(function(child, index) { -+ if ( -+ is(children[index - 1], 'SentenceNode') && -+ is(child, 'WhiteSpaceNode') && -+ is(children[index + 1], 'SentenceNode') -+ ) { -+ console.log(child) -+ } -+ }) - } - } -} -``` - -> Don’t forget to `npm install` the utility! - -If we now run our example with Node, as follows, we’ll see that only white-space -between sentences is logged. - -```sh -$ node example.js -{ type: 'WhiteSpaceNode', value: ' ' } -{ type: 'WhiteSpaceNode', value: ' ' } -no issues found -``` - -Finally, let’s add a warning for the second white-space, as it has more -characters than needed. -We can use [`file.message()`][message] to associate a message with the file. - -```diff -var visit = require('unist-util-visit') -var is = require('unist-util-is') - -module.exports = attacher - -function attacher() { - return transformer - - function transformer(tree, file) { - visit(tree, 'ParagraphNode', visitor) - - function visitor(node) { - var children = node.children - - children.forEach(function(child, index) { - if ( - is('SentenceNode', children[index - 1]) && - is('WhiteSpaceNode', child) && - is('SentenceNode', children[index + 1]) - ) { -- console.log(child) -+ if (child.value.length !== 1) { -+ file.message( -+ 'Expected 1 space between sentences, not ' + child.value.length, -+ child -+ ) -+ } - } - }) - } - } -} -``` - -If we now run our example one final time, we’ll see a message for our problem! - -```sh -$ node example.js -3:14-3:16 warning Expected 1 space between sentences, not 2 - -⚠ 1 warning -``` - -### Further exercises - -One space between sentences isn’t for everyone. -This plugin could receive the preferred amount of spaces instead of a hard-coded -`1`. - -If you want to warn for tabs or newlines between sentences, maybe create a -plugin for that too? - -If you haven’t already, check out the other articles in the -[learn section][learn]! - - - -[spectrum]: https://spectrum.chat/unified - -[visit]: https://github.com/syntax-tree/unist-util-visit - -[is]: https://github.com/syntax-tree/unist-util-is - -[message]: https://github.com/vfile/vfile#vfilemessagereason-position-origin - -[learn]: /learn/ - -[use]: /learn/guide/using-unified/ +This guide is archived. +See [“Create a retext plugin”](/learn/guide/create-a-retext-plugin/) for the +current version. +See also [“Create a rehype plugin”](/learn/guide/create-a-rehype-plugin/) and +[“Create a remark plugin”](/learn/guide/create-a-remark-plugin/) +for the other similar guides. diff --git a/doc/learn/create-a-rehype-plugin.md b/doc/learn/create-a-rehype-plugin.md new file mode 100644 index 000000000000..76f5e98587ce --- /dev/null +++ b/doc/learn/create-a-rehype-plugin.md @@ -0,0 +1,432 @@ +--- +authorGithub: wooorm +author: Titus Wormer +description: Guide that shows how to create a rehype plugin +group: guide +modified: 2024-08-13 +published: 2024-08-13 +tags: + - hast + - plugin + - rehype +title: Create a rehype plugin +--- + +## Create a rehype plugin + +This guide shows how to create a plugin for rehype that adds `id` attributes to +headings. + +> Stuck? +> Have an idea for another guide? +> See [`support.md`][support]. + +### Contents + +* [Case](#case) +* [Setting up](#setting-up) +* [Plugin](#plugin) + +### Case + +Before we start, let’s first outline what we want to make. +Say we have the following file: + +```html +

Solar System

+

Formation and evolution

+

Structure and composition

+

Orbits

+

Composition

+

Distances and scales

+

Interplanetary environment

+

+``` + +And we’d like to turn that into: + +```html +

Solar System

+

Formation and evolution

+

Structure and composition

+

Orbits

+

Composition

+

Distances and scales

+

Interplanetary environment

+

+``` + +In the next step we’ll write the code to use our plugin. + +### Setting up + +Let’s set up a project. +Create a folder, `example`, enter it, and initialize a new project: + +```sh +mkdir example +cd example +npm init -y +``` + +Then make sure the project is a module, so that `import` and `export` work, +by changing `package.json`: + +```diff +--- a/package.json ++++ b/package.json +@@ -1,6 +1,7 @@ + { + "name": "example", + "version": "1.0.0", ++ "type": "module", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" +``` + +Make sure `input.html` exists with: + +```html +

Solar System

+

Formation and evolution

+

Structure and composition

+

Orbits

+

Composition

+

Distances and scales

+

Interplanetary environment

+

+``` + +Now, let’s create an `example.js` file that will process our file and report +any found problems. + +```js twoslash +// @filename: plugin.d.ts +import type {Root} from 'hast' +export default function rehypeSlug(): (tree: Root) => undefined; +// @filename: example.js +/// +// ---cut--- +import fs from 'node:fs/promises' +import {rehype} from 'rehype' +import rehypeSlug from './plugin.js' + +const document = await fs.readFile('input.html', 'utf8') + +const file = await rehype() + .data('settings', {fragment: true}) + .use(rehypeSlug) + .process(document) + +await fs.writeFile('output.html', String(file)) +``` + +> Don’t forget to `npm install rehype`! + +If you read the guide on [using unified][use], +you’ll see some familiar statements. +First, +we load dependencies, +then we read the file in. +We process that file with the plugin we’ll create next and finally we write +it out again. + +Note that we directly depend on `rehype`. +This is a package that exposes a `unified` processor, +and comes with the HTML parser and HTML compiler attached. + +Now we’ve got everything set up except for the plugin itself. +We’ll do that in the next section. + +### Plugin + +We’ll need a plugin and for our case also a transform. +Let’s create them in our plugin file `plugin.js`: + +```js twoslash +/** + * @import {Root} from 'hast' + */ + +/** + * Add `id`s to headings. + * + * @returns + * Transform. + */ +export default function rehypeSlug() { + /** + * @param {Root} tree + * @return {undefined} + */ + return function (tree) { + } +} +``` + +That’s how most plugins start. +A function that returns another function. + +Next, +for this use case, +we can walk the tree and change nodes with +[`unist-util-visit`][visit]. +That’s how many plugins work. + +Let’s start there, +to use `unist-util-visit` to look for headings: + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -2,6 +2,8 @@ + * @import {Root} from 'hast' + */ + ++import {visit} from 'unist-util-visit' ++ + /** + * Add `id`s to headings. + * +@@ -14,5 +16,17 @@ export default function rehypeSlug() { + * @return {undefined} + */ + return function (tree) { ++ visit(tree, 'element', function (node) { ++ if ( ++ node.tagName === 'h1' || ++ node.tagName === 'h2' || ++ node.tagName === 'h3' || ++ node.tagName === 'h4' || ++ node.tagName === 'h5' || ++ node.tagName === 'h6' ++ ) { ++ console.log(node) ++ } ++ }) + } + } +``` + +> Don’t forget to `npm install unist-util-visit`! + +If we now run our example with Node.js, +we’ll see that `console.log` is called: + +```sh +node example.js +``` + +```txt +{ + type: 'element', + tagName: 'h1', + properties: {}, + children: [ { type: 'text', value: 'Solar System', position: [Object] } ], + position: … +} +{ + type: 'element', + tagName: 'h2', + properties: {}, + children: [ + { + type: 'text', + value: 'Formation and evolution', + position: [Object] + } + ], + position: … +} +… +``` + +This output shows that we find our heading element. +That’s what we want. + +Next we want to get a string representation of what is inside the headings. +There’s another utility for that: +[`hast-util-to-string`][hast-util-to-string]. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -2,6 +2,7 @@ + * @import {Root} from 'hast' + */ + ++import {toString} from 'hast-util-to-string' + import {visit} from 'unist-util-visit' + + /** +@@ -25,7 +26,8 @@ export default function rehypeSlug() { + node.tagName === 'h5' || + node.tagName === 'h6' + ) { +- console.log(node) ++ const value = toString(node) ++ console.log(value) + } + }) + } +``` + +> Don’t forget to `npm install hast-util-to-string`! + +If we now run our example with Node.js, +we’ll see the text printed: + +```sh +node example.js +``` + +```txt +Solar System +Formation and evolution +Structure and composition +Orbits +Composition +Distances and scales +Interplanetary environment +``` + +Then we want to turn that text into slugs. +You have many options here. +For this case, +we’ll use [`github-slugger`][github-slugger]. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -3,6 +3,7 @@ + */ + + import {toString} from 'hast-util-to-string' ++import Slugger from 'github-slugger' + import {visit} from 'unist-util-visit' + + /** +@@ -17,6 +18,8 @@ export default function rehypeSlug() { + * @return {undefined} + */ + return function (tree) { ++ const slugger = new Slugger() ++ + visit(tree, 'element', function (node) { + if ( + node.tagName === 'h1' || +@@ -27,7 +30,8 @@ export default function rehypeSlug() { + node.tagName === 'h6' + ) { + const value = toString(node) +- console.log(value) ++ const id = slugger.slug(value) ++ console.log(id) + } + }) + } +``` + +> Don’t forget to `npm install github-slugger`! + +The reason `const slugger = new Slugger()` is there, +is because we want to create a new slugger for each document. +If we’d create it outside of the function, +we’d reuse the same slugger for each document, +which would lead to slugs from different documents being mixed. +That becomes a problem for documents with the same headings. + +If we now run our example with Node.js, +we’ll see the slugs printed: + +```sh +node example.js +``` + +```txt +solar-system +formation-and-evolution +structure-and-composition +orbits +composition +distances-and-scales +interplanetary-environment +``` + +Finally, +we want to add the `id` to the heading elements. +This is also a good time to make sure we don’t overwrite existing `id`s. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -22,16 +22,17 @@ export default function rehypeSlug() { + + visit(tree, 'element', function (node) { + if ( +- node.tagName === 'h1' || +- node.tagName === 'h2' || +- node.tagName === 'h3' || +- node.tagName === 'h4' || +- node.tagName === 'h5' || +- node.tagName === 'h6' ++ !node.properties.id && ++ (node.tagName === 'h1' || ++ node.tagName === 'h2' || ++ node.tagName === 'h3' || ++ node.tagName === 'h4' || ++ node.tagName === 'h5' || ++ node.tagName === 'h6') + ) { + const value = toString(node) + const id = slugger.slug(value) +- console.log(id) ++ node.properties.id = id + } + }) + } +``` + +If we now run our example again with Node… + +```sh +node example.js +``` + +…and open `output.html`, +we’ll see that the IDs are there! + +```html +

Solar System

+

Formation and evolution

+

Structure and composition

+

Orbits

+

Composition

+

Distances and scales

+

Interplanetary environment

+

+``` + +That’s it! +For a complete version of this plugin, +see [`rehype-slug`][rehype-slug]. + +If you haven’t already, check out the other articles in the +[learn section][learn]! + + + +[support]: https://github.com/unifiedjs/.github/blob/main/support.md + +[hast-util-to-string]: https://github.com/rehypejs/rehype-minify/tree/main/packages/hast-util-to-string + +[github-slugger]: https://github.com/Flet/github-slugger + +[visit]: https://github.com/syntax-tree/unist-util-visit + +[rehype-slug]: https://github.com/rehypejs/rehype-slug + +[learn]: /learn/ + +[use]: /learn/guide/using-unified/ diff --git a/doc/learn/create-a-remark-plugin.md b/doc/learn/create-a-remark-plugin.md new file mode 100644 index 000000000000..6c8c29c89780 --- /dev/null +++ b/doc/learn/create-a-remark-plugin.md @@ -0,0 +1,280 @@ +--- +authorGithub: wooorm +author: Titus Wormer +description: Guide that shows how to create a remark plugin +group: guide +modified: 2024-08-13 +published: 2024-08-13 +tags: + - mdast + - plugin + - remark +title: Create a remark plugin +--- + +## Create a remark plugin + +This guide shows how to create a plugin for remark that turns emoji +shortcodes ([gemoji][], such as `:+1:`) into Unicode emoji (`👍`). +It looks for a regex in the text and replaces it. + +> Stuck? +> Have an idea for another guide? +> See [`support.md`][support]. + +### Contents + +* [Case](#case) +* [Setting up](#setting-up) +* [Plugin](#plugin) + +### Case + +Before we start, let’s first outline what we want to make. +Say we have the following file: + +```markdown +Look, the moon :new_moon_with_face: +``` + +And we’d like to turn that into: + +```markdown +Look, the moon 🌚 +``` + +In the next step we’ll write the code to use our plugin. + +### Setting up + +Let’s set up a project. +Create a folder, `example`, enter it, and initialize a new project: + +```sh +mkdir example +cd example +npm init -y +``` + +Then make sure the project is a module, so that `import` and `export` work, +by changing `package.json`: + +```diff +--- a/package.json ++++ b/package.json +@@ -1,6 +1,7 @@ + { + "name": "example", + "version": "1.0.0", ++ "type": "module", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" +``` + +Make sure `input.md` exists with: + +```markdown +Look, the moon :new_moon_with_face: +``` + +Now, let’s create an `example.js` file that will process our file and report +any found problems. + +```js twoslash +// @filename: plugin.d.ts +import type {Root} from 'mdast' +export default function remarkGemoji(): (tree: Root) => undefined; +// @filename: example.js +/// +// ---cut--- +import fs from 'node:fs/promises' +import {remark} from 'remark' +import remarkGemoji from './plugin.js' + +const document = await fs.readFile('input.md', 'utf8') + +const file = await remark().use(remarkGemoji).process(document) + +await fs.writeFile('output.md', String(file)) +``` + +> Don’t forget to `npm install remark`! + +If you read the guide on [using unified][use], +you’ll see some familiar statements. +First, +we load dependencies, +then we read the file in. +We process that file with the plugin we’ll create next and finally we write +it out again. + +Note that we directly depend on `remark`. +This is a package that exposes a `unified` processor, +and comes with the markdown parser and markdown compiler attached. + +Now we’ve got everything set up except for the plugin itself. +We’ll do that in the next section. + +### Plugin + +We’ll need a plugin, and for our case also a transform. +Let’s create them in our plugin file `plugin.js`: + +```js twoslash +/** + * @import {Root} from 'mdast' + */ + +/** + * Turn gemoji shortcodes (`:+1:`) into emoji (`👍`). + * + * @returns + * Transform. + */ +export default function remarkGemoji() { + /** + * @param {Root} tree + * @return {undefined} + */ + return function (tree) { + } +} +``` + +That’s how most plugins start. +A function that returns another function. + +For this use case, +we could walk the tree and replace nodes with +[`unist-util-visit`][visit], +which is how many plugins work. +But a different utility is even simpler: +[`mdast-util-find-and-replace`][find-and-replace]. +It looks for a regex and lets you then replace that match. + +Let’s add that. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -2,6 +2,8 @@ + * @import {Root} from 'mdast' + */ + ++import {findAndReplace} from 'mdast-util-find-and-replace' ++ + /** + * Turn gemoji shortcodes (`:+1:`) into emoji (`👍`). + * +@@ -14,5 +16,16 @@ export default function remarkGemoji() { + * @return {undefined} + */ + return function (tree) { ++ findAndReplace(tree, [ ++ /:(\+1|[-\w]+):/g, ++ /** ++ * @param {string} _ ++ * @param {string} $1 ++ * @return {undefined} ++ */ ++ function (_, $1) { ++ console.log(arguments) ++ } ++ ]) + } + } +``` + +> Don’t forget to `npm install mdast-util-find-and-replace`! + +If we now run our example with Node.js, +we’ll see that `console.log` is called: + +```sh +node example.js +``` + +```txt +[Arguments] { + '0': ':new_moon_with_face:', + '1': 'new_moon_with_face', + '2': { + index: 15, + input: 'Look, the moon :new_moon_with_face:', + stack: [ [Object], [Object], [Object] ] + } +} +``` + +This output shows that the regular expression matches the emoji shortcode. +The second argument is the name of the emoji. +That’s what we want. + +We can look that name up to find the corresponding Unicode emoji. +We can use the [`gemoji`][gemoji] package for that. +It exposes a `nameToEmoji` record. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -2,6 +2,7 @@ + * @import {Root} from 'mdast' + */ + ++import {nameToEmoji} from 'gemoji' + import {findAndReplace} from 'mdast-util-find-and-replace' + + /** +@@ -21,10 +22,10 @@ export default function remarkGemoji() { + /** + * @param {string} _ + * @param {string} $1 +- * @return {undefined} ++ * @return {string | false} + */ + function (_, $1) { +- console.log(arguments) ++ return Object.hasOwn(nameToEmoji, $1) ? nameToEmoji[$1] : false + } + ]) + } +``` + +> Don’t forget to `npm install gemoji`! + +If we now run our example again with Node… + +```sh +node example.js +``` + +…and open `output.md`, +we’ll see that the shortcode is replaced with the emoji! + +```markdown +Look, the moon 🌚 +``` + +That’s it! +For a complete version of this plugin, +see [`remark-gemoji`][remark-gemoji]. + +If you haven’t already, check out the other articles in the +[learn section][learn]! + + + +[support]: https://github.com/unifiedjs/.github/blob/main/support.md + +[find-and-replace]: https://github.com/syntax-tree/mdast-util-find-and-replace + +[gemoji]: https://github.com/wooorm/gemoji/blob/main/support.md + +[visit]: https://github.com/syntax-tree/unist-util-visit + +[remark-gemoji]: https://github.com/remarkjs/remark-gemoji + +[learn]: /learn/ + +[use]: /learn/guide/using-unified/ diff --git a/doc/learn/create-a-retext-plugin.md b/doc/learn/create-a-retext-plugin.md new file mode 100644 index 000000000000..06cf969c9abb --- /dev/null +++ b/doc/learn/create-a-retext-plugin.md @@ -0,0 +1,342 @@ +--- +authorGithub: wooorm +author: Titus Wormer +description: Guide that shows how to create a retext plugin +group: guide +modified: 2024-08-13 +published: 2024-08-13 +tags: + - nlcst + - plugin + - retext +title: Create a retext plugin +--- + +## Create a retext plugin + +This guide shows how to create a plugin for retext that checks the amount of +spaces between sentences. + +> Stuck? +> Have an idea for another guide? +> See [`support.md`][support]. + +### Contents + +* [Case](#case) +* [Setting up](#setting-up) +* [Plugin](#plugin) +* [Further exercises](#further-exercises) + +### Case + +Before we start, let’s first outline what we want to make. +Say we have the following text file: + +```markdown +One sentence. Two sentences. + +One sentence. Two sentences. +``` + +We want to get a warning for the second paragraph, saying that one space instead +of two spaces should be used. + +In the next step we’ll write the code to use our plugin. + +### Setting up + +Let’s set up a project. +Create a folder, `example`, enter it, and initialize a new project: + +```sh +mkdir example +cd example +npm init -y +``` + +Then make sure the project is a module, so that `import` and `export` work, +by changing `package.json`: + +```diff +--- a/package.json ++++ b/package.json +@@ -1,6 +1,7 @@ + { + "name": "example", + "version": "1.0.0", ++ "type": "module", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" +``` + +Make sure `example.md` exists with: + +```markdown +One sentence. Two sentences. + +One sentence. Two sentences. +``` + +Now, let’s create an `example.js` file that will process our text file and +report any found problems. + +```js twoslash +// @filename: plugin.d.ts +import type {Root} from 'nlcst' +import type {VFile} from 'vfile' +export default function retextSentenceSpacing(): (tree: Root, file: VFile) => undefined; +// @filename: example.js +/// +// ---cut--- +import fs from 'node:fs/promises' +import {retext} from 'retext' +import {reporter} from 'vfile-reporter' +import retextSentenceSpacing from './plugin.js' + +const document = await fs.readFile('example.md', 'utf8') + +const file = await retext() + .use(retextSentenceSpacing) + .process(document) + +console.error(reporter(file)) +``` + +> Don’t forget to `npm install retext vfile-reporter`! + +If you read the guide on [using unified][use], +you’ll see some familiar statements. +First, +we load dependencies, +then we read the file in. +We process that file with the plugin we’ll create next and finally we report +either a fatal error or any found linting messages. + +Note that we directly depend on `retext`. +This is a package that exposes a `unified` processor, +and comes with the parser and compiler attached. + +When running our example (it doesn’t work yet though) we want to see a message +for the second paragraph, saying that one space instead of two spaces should be +used. + +Now we’ve got everything set up except for the plugin itself. +We’ll do that in the next section. + +### Plugin + +We’ll need a plugin, and for our case also a transform which will inspect. +Let’s create them in our plugin file `plugin.js`: + +```js twoslash +/** + * @import {Root} from 'nlcst' + * @import {VFile} from 'vfile' + */ + +export default function retextSentenceSpacing() { + /** + * @param {Root} tree + * @param {VFile} file + * @return {undefined} + */ + return function (tree, file) { + } +} +``` + +First things first, we need to check `tree` for a pattern. +We can use a utility to help us to recursively walk our tree, namely +[`unist-util-visit`][visit]. +Let’s add that. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -3,6 +3,8 @@ + * @import {VFile} from 'vfile' + */ + ++import {visit} from 'unist-util-visit' ++ + export default function retextSentenceSpacing() { + /** + * @param {Root} tree +@@ -10,5 +12,8 @@ export default function retextSentenceSpacing() { + * @return {undefined} + */ + return function (tree, file) { ++ visit(tree, 'ParagraphNode', function (node) { ++ console.log(node) ++ }) + } + } +``` + +> Don’t forget to `npm install unist-util-visit`. + +If we now run our example with Node.js, as follows, we’ll see that visitor is +called with both paragraphs in our example: + +```sh +node example.js +``` + +```txt +{ + type: 'ParagraphNode', + children: [ + { type: 'SentenceNode', children: [Array], position: [Object] }, + { type: 'WhiteSpaceNode', value: ' ', position: [Object] }, + { type: 'SentenceNode', children: [Array], position: [Object] } + ], + position: { + start: { line: 1, column: 1, offset: 0 }, + end: { line: 1, column: 29, offset: 28 } + } +} +{ + type: 'ParagraphNode', + children: [ + { type: 'SentenceNode', children: [Array], position: [Object] }, + { type: 'WhiteSpaceNode', value: ' ', position: [Object] }, + { type: 'SentenceNode', children: [Array], position: [Object] } + ], + position: { + start: { line: 3, column: 1, offset: 30 }, + end: { line: 3, column: 30, offset: 59 } + } +} +no issues found +``` + +This output already shows that paragraphs contain two types of nodes: +`SentenceNode` and `WhiteSpaceNode`. +The latter is what we want to check, but the former is important because we only +warn about whitespace between sentences in this plugin (that could be another +plugin though). + +Let’s now loop through the children of each paragraph. +Only checking whitespace between sentences. +We use a small utility for checking node types: [`unist-util-is`][is]. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -13,7 +13,23 @@ export default function retextSentenceSpacing() { + */ + return function (tree, file) { + visit(tree, 'ParagraphNode', function (node) { +- console.log(node) ++ let index = -1 ++ ++ while (++index < node.children.length) { ++ const previous = node.children[index - 1] ++ const child = node.children[index] ++ const next = node.children[index + 1] ++ ++ if ( ++ previous && ++ next && ++ previous.type === 'SentenceNode' && ++ child.type === 'WhiteSpaceNode' && ++ next.type === 'SentenceNode' ++ ) { ++ console.log(child) ++ } ++ } + }) + } + } +``` + +If we now run our example with Node, as follows, we’ll see that only whitespace +between sentences is logged. + +```sh +node example.js +``` + +```txt +{ + type: 'WhiteSpaceNode', + value: ' ', + position: { + start: { line: 1, column: 14, offset: 13 }, + end: { line: 1, column: 15, offset: 14 } + } +} +{ + type: 'WhiteSpaceNode', + value: ' ', + position: { + start: { line: 3, column: 14, offset: 43 }, + end: { line: 3, column: 16, offset: 45 } + } +} +no issues found +``` + +Finally, let’s add a warning for the second whitespace, as it has more +characters than needed. +We can use [`file.message()`][message] to associate a message with the file. + +```diff +--- a/plugin.js ++++ b/plugin.js +@@ -25,9 +25,15 @@ export default function retextSentenceSpacing() { + next && + previous.type === 'SentenceNode' && + child.type === 'WhiteSpaceNode' && +- next.type === 'SentenceNode' ++ next.type === 'SentenceNode' && ++ child.value.length !== 1 + ) { +- console.log(child) ++ file.message( ++ 'Unexpected `' + ++ child.value.length + ++ '` spaces between sentences, expected `1` space', ++ child ++ ) + } + } + }) +``` + +If we now run our example one final time, we’ll see a message for our problem! + +```sh +$ node example.js +3:14-3:16 warning Unexpected `2` spaces between sentences, expected `1` space + +⚠ 1 warning +``` + +### Further exercises + +One space between sentences isn’t for everyone. +This plugin could receive the preferred amount of spaces instead of a hard-coded +`1`. + +If you want to warn for tabs or newlines between sentences, maybe create a +plugin for that too? + +If you haven’t already, check out the other articles in the +[learn section][learn]! + + + +[support]: https://github.com/unifiedjs/.github/blob/main/support.md + +[visit]: https://github.com/syntax-tree/unist-util-visit + +[is]: https://github.com/syntax-tree/unist-util-is + +[message]: https://github.com/vfile/vfile#vfilemessagereason-options + +[learn]: /learn/ + +[use]: /learn/guide/using-unified/ diff --git a/doc/learn/create-an-editor.md b/doc/learn/create-an-editor.md index 557082488f43..9828015889a0 100644 --- a/doc/learn/create-an-editor.md +++ b/doc/learn/create-an-editor.md @@ -1,61 +1,60 @@ --- -group: guide -title: Create an editor -description: Guide that shows how to create a fancy app ✨ +authorGithub: wooorm author: Titus Wormer -authorTwitter: wooorm +description: Guide that shows how to create a fancy app ✨ +group: guide +modified: 2024-08-06 +published: 2017-05-03 tags: - - editor - dingus -published: 2017-05-03 -modified: 2019-12-12 + - editor + - playground +title: Create an editor --- ## Creating an editor -> I’m not entirely sure how to call this thing. -> “Demo” is too generic, “dingus” too vague, and I think editor is pretty apt. -> Anyway, drop a line on [Spectrum][] if you know a better name. - This guide shows how to create an interactive online editor with unified. -In it we’ll visualise syntactic properties of text by “syntax highlighting” +Something sometimes called a “playground”. +Or a “dingus”. +In it we’ll visualize syntactic properties of text by “syntax highlighting” them. -The editor will run in a browser. -It’ll be fast as we’re using [`virtual-dom`][vdom] (but you could use [React][] -and the like too). +It’s made with [React][] and runs in a browser. -For this example we’ll create an app that visualises sentence length. -It’s based on a tip by [Gary Provost][gary], and the visualisation is based on -[a tweet by @gregoryciotti][tweet]. +For this example we’ll create an app that visualizes sentence length. +Based on a tip by [Gary Provost][gary-provost]. +The visualization is based on +[a tweet by `@gregoryciotti`][gregoryciotti-tweet]. -You can also [view this project][write-music] with some more features online. +You can also [view this project][wooorm-write-music] with some more features +online. > Stuck? -> A good place to get help fast is [Spectrum][]. > Have an idea for another guide? -> Share it on spectrum! +> See [`support.md`][unified-support]. ### Contents -* [Case](#case) -* [Project structure](#project-structure) -* [Setting up JavaScript](#setting-up-javascript) -* [Natural language syntax tree](#natural-language-syntax-tree) -* [Virtual DOM](#virtual-dom) -* [Highlight](#highlight) -* [Colour](#colour) -* [Squashing bugs](#squashing-bugs) -* [Further exercises](#further-exercises) +* [Case](#case) +* [Project structure](#project-structure) +* [Setting up JavaScript](#setting-up-javascript) +* [Natural language syntax tree](#natural-language-syntax-tree) +* [Virtual DOM](#virtual-dom) +* [Highlight](#highlight) +* [Further exercises](#further-exercises) ### Case Before we start, let’s first outline what we want to make. We want to highlight sentences in text based on how many words they have. -The user should be able to change text, and it should highlight live. +The user should be able to change text. +And it should highlight live. -We’ll use [xo][] as a linter, and [browserify][] as a bundler to compile our -JavaScript with `require` calls to JavaScript that works in the browser (you -can swap those out for your favourite linter and bundler). +We’ll use [esbuild][] as a bundler to compile our JavaScript to code that +works in the browser in production. +We’ll use [xo][github-sindresorhus-xo] and [prettier][] to lint and format +our code. +You can swap those out for your favorite tools. ### Project structure @@ -63,68 +62,83 @@ Let’s first outline our project structure: ```txt demo/ -├─ index.js -├─ build.js -├─ index.html +├─ bundle.mjs ├─ index.css +├─ index.html +├─ index.jsx └─ package.json ``` -…where `demo/` is our directory, and `build.js` is the JavaScript generated by -compiling `index.js`. +…where `demo/` is our folder and `bundle.mjs` is the JavaScript generated by +compiling `index.jsx`. -Keep `index.js`, `index.html`, and `index.css` empty for now, and fill -`package.json` with the following. +Keep `index.jsx`, `index.html`, and `index.css` empty for now, and fill +`package.json` with the following: ```json { - "name": "demo", - "private": true, - "dependencies": {}, "devDependencies": { - "browserify": "^16.0.0", - "xo": "^0.23.0" + "esbuild": "^0.23.0", + "prettier": "^3.0.0", + "xo": "^0.59.0" }, + "name": "demo", + "prettier": { + "bracketSpacing": false, + "semi": false, + "singleQuote": true, + "tabWidth": 2, + "trailingComma": "none", + "useTabs": false + }, + "private": true, "scripts": { - "build": "browserify index.js > build.js", - "lint": "xo", - "test": "npm run build && npm run lint" + "build": "esbuild index.jsx --bundle --format=esm --jsx=automatic --minify --outfile=bundle.mjs --target=es2020", + "format": "prettier . --log-level warn --write && xo --fix", + "test": "npm run build && npm run format" }, + "type": "module", "xo": { - "space": true, - "esnext": false, "envs": [ "browser" ], "ignore": [ - "build.js" - ] + "bundle.mjs" + ], + "prettier": true } } ``` -> [`private: true`][private] means you can’t accidentally publish your package +> `private: true` means you can’t accidentally publish your package > to npm. -Now, after running `npm install` and `npm test` you’ll see `build.js` appear +The above package sets up [xo][github-sindresorhus-xo], [prettier][], and +[esbuild][]. +Now, after running `npm install` and `npm test` you’ll see `bundle.mjs` appear too. -The above package sets up [xo][] as the linter and [browserify][] as the -bundler. -Now, fill `index.html` with the following: +Also add `.prettierignore` file to not format our build: + +```ignore +bundle.mjs +``` + +Fill `index.html` with the following: ```html - + demo - +
- + ``` -This links `index.css` and `build.js`, and adds an element (`#root`) which we’ll -add our editor to later. -Oh, did you know that ``, ``, and `` are optional? +This links `index.css` and `bundle.mjs`, and adds an element (`#root`) which +we’ll add our editor to later. + +Did you know that ``, ``, and `` are optional? For this example we’ll keep the HTML minimal, but feel free to add them if you prefer them. @@ -132,69 +146,60 @@ prefer them. Alright! Now, let’s set up our JavaScript. -Start by adding the following to `index.js`: - -```js -var h = require('virtual-dom/h') -var createElement = require('virtual-dom/create-element') -var diff = require('virtual-dom/diff') -var patch = require('virtual-dom/patch') - -var root = document.getElementById('root') -var tree = render('The initial text.') -var dom = root.appendChild(createElement(tree)) - -function onchange(ev) { - var next = render(ev.target.value) - dom = patch(dom, diff(tree, next)) - tree = next +Start by adding the following to `index.jsx`: + +```jsx twoslash +/// +/* eslint-env browser */ +import ReactDom from 'react-dom/client' +import React from 'react' + +const main = document.querySelector('#root') +if (!main) throw new Error('No root element found') +const root = ReactDom.createRoot(main) + +const sample = 'The initial text.' + +root.render(React.createElement(Playground)) + +function Playground() { + const [text, setText] = React.useState(sample) + + return ( +
+
+ {/* Trailing whitespace in a `textarea` is shown, but not in a `div` + with `white-space: pre-wrap`. + Add a `br` to make the last newline explicit. */} + {/\n[ \t]*$/.test(text) ?
: undefined} +
+