Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More readme fixes #430

Merged
merged 8 commits into from
Nov 1, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions e2e/cypress/integration/readme.spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -157,4 +157,63 @@ describe('Readme style', () => {
cy.request($a[0].href).its('status').should('equal', 200)
})
})

it('renders :emoji: in readme and project description', () => {
const username = getFakeUsername()
const email = faker.unique(faker.internet.email)
const password = '123456'

const repoName = 'ogx360'
const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360'

cy.createUser(username, email, password)
cy.visit('/')
cy.get('[data-cy=user-menu]')

cy.forceVisit('/projects/new')

// Migrate the repo
cy.get('[data-cy=sync-field]').type(syncedRepoUrl)
cy.get('button').contains('Sync').click()

// Wait for redirection for project page
cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`)
// Wait for the repo to finish processing, by checking the visibility of info-bar.
cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible')

// the project description isAdd modern xinput USB support to your Original 📺 🎮
cy.get('[data-cy=project-description]').should('contain.text', '📺 🎮')
cy.get('[data-cy=readme]').should('contain.text', '🤓')
})

it('preserves URLs for GitHub Actions badges', () => {
const username = getFakeUsername()
const email = faker.unique(faker.internet.email)
const password = '123456'

const repoName = 'ogx360'
const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360'

cy.createUser(username, email, password)
cy.visit('/')
cy.get('[data-cy=user-menu]')

cy.forceVisit('/projects/new')

// Migrate the repo
cy.get('[data-cy=sync-field]').type(syncedRepoUrl)
cy.get('button').contains('Sync').click()

// Wait for redirection for project page
cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`)
// Wait for the repo to finish processing, by checking the visibility of info-bar.
cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible')

// The first image in the readme is the GitHub Actions badge.
cy.get('[data-cy=readme] img')
.first()
.each($img => {
cy.request($img[0].src).its('status').should('equal', 200)
})
})
})
25 changes: 25 additions & 0 deletions frontend/src/components/Board/Readme.jsx
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,31 @@ const Readme = ({ renderedReadme }) => (
#readme input[type='checkbox'] {
margin-right: 5px;
}

#readme table {
border-collapse: collapse;
border-spacing: 0;
display: block;
max-width: 100%;
overflow: auto;
width: 100%;
width: max-content;
}

#readme table tr {
background-color: #ffffff;
border-top: 1px solid hsla(210, 18%, 91%, 1);
}

#readme table th,
#readme table td {
padding: 6px 13px;
border: 1px solid #d0d7de;
}

#readme table tr:nth-child(2n) {
background-color: #f6f8fa;
}
`}</style>
</div>
)
Expand Down
3 changes: 1 addition & 2 deletions processor/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,12 @@
"bullmq": "^1.91.1",
"cheerio": "^1.0.0-rc.10",
"chokidar": "^3.4.3",
"escape-html": "^1.0.3",
"express": "^4.17.2",
"express-fileupload": "^1.2.1",
"globule": "^1.0.0",
"hast-util-has-property": "^2.0.0",
"js-yaml": "^3.3.1",
"jszip": "^3.7.0",
"linkify-it": "^4.0.1",
"lodash.debounce": "^4.0.8",
"loglevel": "^1.7.1",
"meilisearch": "^0.25.1",
Expand All @@ -46,6 +44,7 @@
"rehype-shift-heading": "^1.0.2",
"rehype-slug": "^5.0.1",
"rehype-stringify": "^9.0.3",
"remark-emoji": "^3.0.2",
"remark-gfm": "^3.0.1",
"remark-parse": "^10.0.1",
"remark-rehype": "^10.1.0",
Expand Down
3 changes: 2 additions & 1 deletion processor/src/giteatDB.ts
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ export const giteaDB: GiteaDB = {
row =>
row.repo_id === repoId &&
row.type === TaskType.Migration &&
row.status === MigrationStatus.Done,
row.status === MigrationStatus.Done &&
row.default_branch !== ''
)
},

Expand Down
5 changes: 4 additions & 1 deletion processor/src/tasks/processReadme/index.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
import path from 'node:path'

import { unified } from 'unified'
import globule from 'globule'
import path from 'node:path'
import rehypeAutolinkHeadings from 'rehype-autolink-headings'
import rehypeHighlight from 'rehype-highlight'
import rehypeRaw from 'rehype-raw'
import rehypeSanitize from 'rehype-sanitize'
import rehypeShiftHeading from 'rehype-shift-heading'
import rehypeSlug from 'rehype-slug'
import rehypeStringify from 'rehype-stringify'
import remarkEmoji from 'remark-emoji'
import remarkGfm from 'remark-gfm'
import remarkParse from 'remark-parse'
import remarkRehype from 'remark-rehype'
Expand Down Expand Up @@ -101,6 +103,7 @@ async function renderMarkdown(
) {
const Remarker = unified()
.use(remarkParse)
.use(remarkEmoji)
.use(remarkGfm)
.use(remarkRehype, { allowDangerousHtml: true })
.use(rehypeRaw)
Expand Down
68 changes: 33 additions & 35 deletions processor/src/tasks/writeKitspaceYaml.ts
Original file line number Diff line number Diff line change
@@ -1,59 +1,57 @@
import escape from 'escape-html'
import LinkifyIt from 'linkify-it'
import path from 'node:path'

import { unified } from 'unified'
import rehypeSanitize from 'rehype-sanitize'
import rehypeStringify from 'rehype-stringify'
import remarkEmoji from 'remark-emoji'
import remarkGfm from 'remark-gfm'
kasbah marked this conversation as resolved.
Show resolved Hide resolved
import remarkParse from 'remark-parse'
import remarkRehype from 'remark-rehype'

import { JobData } from '../jobData.js'
import * as utils from '../utils.js'

const linkify = new LinkifyIt()

export default function writeKitspaceYaml(
export default async function writeKitspaceYaml(
job,
{ kitspaceYaml, outputDir }: Partial<JobData>,
) {
const kitspaceYamlJson = path.join(outputDir, 'kitspace-yaml.json')
job.updateProgress({ status: 'in_progress', file: kitspaceYamlJson })
const KitspaceYamlJsonLinkified = linkifyKitspaceYaml(kitspaceYaml)

const rendered = await renderKitspaceYamlSummaries(kitspaceYaml)

return utils
.writeFile(kitspaceYamlJson, JSON.stringify(KitspaceYamlJsonLinkified, null, 2))
.writeFile(kitspaceYamlJson, JSON.stringify(rendered, null, 2))
.then(() => job.updateProgress({ status: 'done', file: kitspaceYamlJson }))
.catch(error =>
job.updateProgress({ status: 'failed', file: kitspaceYamlJson, error }),
)
}

function linkifyKitspaceYaml(kitspaceYaml) {
async function renderKitspaceYamlSummaries(kitspaceYaml) {
if (kitspaceYaml.multi) {
const linkifiedKitspaceYaml = kitspaceYaml
Object.keys(kitspaceYaml.multi).forEach(subProject => {
linkifiedKitspaceYaml.multi[subProject] = linkifyProjectSummary(kitspaceYaml.multi[subProject])
})
return linkifiedKitspaceYaml
}

return linkifyProjectSummary(kitspaceYaml)
}

function linkifyProjectSummary(kitspaceYaml) {
let escapedSummary = escape(kitspaceYaml.summary || '')
const matches = linkify.match(escapedSummary)

if (matches) {
for (const match of matches) {
// Use https by default
const url = new URL(match.url)
if (!match.schema) {
url.protocol = 'https:'
const rendered = { multi: {} }
for (const key of Object.keys(kitspaceYaml.multi)) {
const subProject = kitspaceYaml.multi[key]
rendered.multi[key] = {
...subProject,
summary: await renderSummary(subProject.summary),
}

escapedSummary = escapedSummary.replace(
match.raw,
`<a href="${url.toString()}" rel="noopener noreferrer" target="_blank">${match.text}</a>`
)
}
return rendered
}
return { ...kitspaceYaml, summary: await renderSummary(kitspaceYaml.summary) }
}

kitspaceYaml.summary = escapedSummary
return kitspaceYaml
const Remarker = unified()
.use(remarkParse)
.use(remarkEmoji)
.use(remarkGfm)
.use(remarkRehype)
.use(rehypeSanitize)
.use(rehypeStringify)

async function renderSummary(summary = ''): Promise<string> {
const rendered = await Remarker.process(summary)
return String(rendered)
}
11 changes: 8 additions & 3 deletions processor/src/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,14 @@ export function toGitHubRawUrl(url: string) {
if (parsedUrl.hostname === 'github.com') {
parsedUrl.hostname = 'raw.githubusercontent.com'
const urlPath = parsedUrl.pathname.split('/')
// Remove `/blob/` or '/raw/' from the path.
parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/')
url = parsedUrl.toString()
// Avoid modifying github actions status badges.
const isWorkflowPath = ['workflows', 'actions'].includes(urlPath?.[3])

if (!isWorkflowPath) {
// Remove `/blob/` or '/raw/' from the path.
parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/')
url = parsedUrl.toString()
}
}
return url
}
17 changes: 11 additions & 6 deletions processor/src/watcher.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,8 @@ export function watch(repoDir, { giteaDB }: WatchOptions) {
dirWatchers[gitDir].queuing = false
return
}
defaultBranch = giteaRepo.default_branch
originalUrl = giteaRepo.original_url
repoDescription = giteaRepo.description

giteaId = giteaRepo.id
// use case-correct names from the DB
ownerName = giteaRepo.owner_name
repoName = giteaRepo.name

if (giteaRepo.is_empty) {
await giteaDB.waitForNonEmpty(giteaId)
Expand All @@ -56,6 +51,16 @@ export function watch(repoDir, { giteaDB }: WatchOptions) {
if (giteaRepo.is_mirror) {
await giteaDB.waitForRepoMigration(giteaId)
}

// Get the repo info again after the migration is done.
// Some fields, (e.g., default_branch) only gets populated after migration
const finalGiteaRepoData = await giteaDB.getRepoInfo(ownerName, repoName)
originalUrl = finalGiteaRepoData.original_url
repoDescription = finalGiteaRepoData.description
defaultBranch = finalGiteaRepoData.default_branch
// use case-correct names from the DB
ownerName = finalGiteaRepoData.owner_name
repoName = finalGiteaRepoData.name
}

await addProjectToQueues({
Expand Down
35 changes: 22 additions & 13 deletions processor/yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -1125,6 +1125,11 @@ emoji-regex@^8.0.0:
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==

emoticon@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/emoticon/-/emoticon-4.0.1.tgz#2d2bbbf231ce3a5909e185bbb64a9da703a1e749"
integrity sha512-dqx7eA9YaqyvYtUhJwT4rC1HIp82j5ybS1/vQ42ur+jBe17dJMwZE4+gvL1XadSFfxaPFFGt3Xsw+Y8akThDlw==

encodeurl@~1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
Expand Down Expand Up @@ -2314,13 +2319,6 @@ lie@~3.3.0:
dependencies:
immediate "~3.0.5"

linkify-it@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-4.0.1.tgz#01f1d5e508190d06669982ba31a7d9f56a5751ec"
integrity sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==
dependencies:
uc.micro "^1.0.1"

locate-path@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"
Expand Down Expand Up @@ -3036,6 +3034,13 @@ node-cleanup@^2.1.2:
resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c"
integrity sha1-esGavSl+Caf3KnFUXZUbUX5N3iw=

node-emoji@^1.11.0:
version "1.11.0"
resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
integrity sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==
dependencies:
lodash "^4.17.21"

[email protected]:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
Expand Down Expand Up @@ -3521,6 +3526,15 @@ rehype-stringify@^9.0.3:
hast-util-to-html "^8.0.0"
unified "^10.0.0"

remark-emoji@^3.0.2:
version "3.0.2"
resolved "https://registry.yarnpkg.com/remark-emoji/-/remark-emoji-3.0.2.tgz#786e88af1ecae682d74d7e1219989f34708205da"
integrity sha512-hEgxEv2sBtvhT3tNG/tQeeFY3EbslftaOoG14dDZndLo25fWJ6Fbg4ukFbIotOWWrfXyASjXjyHT+6n366k3mg==
dependencies:
emoticon "^4.0.0"
node-emoji "^1.11.0"
unist-util-visit "^4.1.0"

remark-gfm@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/remark-gfm/-/remark-gfm-3.0.1.tgz#0b180f095e3036545e9dddac0e8df3fa5cfee54f"
Expand Down Expand Up @@ -4046,11 +4060,6 @@ typescript@^4.5.5:
resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==

uc.micro@^1.0.1:
version "1.0.6"
resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==

unbox-primitive@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"
Expand Down Expand Up @@ -4138,7 +4147,7 @@ unist-util-visit@^3.0.0:
unist-util-is "^5.0.0"
unist-util-visit-parents "^4.0.0"

unist-util-visit@^4.0.0, unist-util-visit@^4.1.1:
unist-util-visit@^4.0.0, unist-util-visit@^4.1.0, unist-util-visit@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-4.1.1.tgz#1c4842d70bd3df6cc545276f5164f933390a9aad"
integrity sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==
Expand Down