From 05056b8aa2cc6cdb9eff12badb03d6143b6ae5e5 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Thu, 16 Jan 2025 13:37:06 +0000 Subject: [PATCH 001/125] [v10.0/forgejo] Refactor e2e tests to simplify authentication setup (#6585) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6400 Replaced manual login and context loading across tests with Playwright's `test.use` configuration for user authentication. This simplifies test setup, improves readability, and reduces repetition. #6362 first part ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [x] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [ ] I did not document these changes and I do not expect someone else to do it. ### Release notes - [x] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Co-authored-by: Julian Schlarb Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6585 Reviewed-by: Otto Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- .gitignore | 1 + tests/e2e/README.md | 14 ++- tests/e2e/actions.test.e2e.ts | 89 ++++++-------- tests/e2e/dashboard-ci-status.test.e2e.ts | 21 ++-- tests/e2e/git-notes.test.e2e.ts | 12 +- tests/e2e/issue-comment.test.e2e.ts | 21 ++-- tests/e2e/issue-sidebar.test.e2e.ts | 36 ++---- tests/e2e/markdown-editor.test.e2e.ts | 26 +--- tests/e2e/org-settings.test.e2e.ts | 9 +- tests/e2e/profile_actions.test.e2e.ts | 8 +- tests/e2e/reaction-selectors.test.e2e.ts | 12 +- tests/e2e/release.test.e2e.ts | 12 +- tests/e2e/repo-code.test.e2e.ts | 41 +++---- tests/e2e/repo-migrate.test.e2e.ts | 16 +-- tests/e2e/repo-new.test.e2e.ts | 21 ++-- tests/e2e/repo-settings.test.e2e.ts | 17 +-- tests/e2e/right-settings-button.test.e2e.ts | 68 +++++------ tests/e2e/utils_e2e.ts | 30 ++++- tests/e2e/utils_e2e_test.go | 127 ++++++++++++++++++++ 19 files changed, 327 insertions(+), 254 deletions(-) diff --git a/.gitignore b/.gitignore index 744577248d..f040fdaf37 100644 --- a/.gitignore +++ b/.gitignore @@ -72,6 +72,7 @@ cpu.out /tests/e2e/reports /tests/e2e/test-artifacts /tests/e2e/test-snapshots +/tests/e2e/.auth /tests/*.ini /tests/**/*.git/**/*.sample /node_modules diff --git a/tests/e2e/README.md b/tests/e2e/README.md index 8d8858bfd5..35fc5e7d1d 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -250,16 +250,18 @@ test('For anyone', async ({page}) => { If you need a user account, you can use something like: ~~~js -import {test, login_user, login} from './utils_e2e.ts'; +import {test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); // or another user -}); +// reuse user2 token from scope `shared` +test.use({user: 'user2', authScope: 'shared'}) -test('For signed users only', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('For signed users only', async ({page}) => { + +}) ~~~ +users are created in [utils_e2e_test.go](utils_e2e_test.go) + ### Run tests very selectively Browser testing can take some time. diff --git a/tests/e2e/actions.test.e2e.ts b/tests/e2e/actions.test.e2e.ts index a66b608080..6236fe70d3 100644 --- a/tests/e2e/actions.test.e2e.ts +++ b/tests/e2e/actions.test.e2e.ts @@ -10,72 +10,61 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; - -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +import {save_visual, test} from './utils_e2e.ts'; const workflow_trigger_notification_text = 'This workflow has a workflow_dispatch event trigger.'; +test.describe('Workflow Authenticated user2', () => { + test.use({user: 'user2'}); -test('workflow dispatch present', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); + test('workflow dispatch present', async ({page}) => { + await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); - await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); + await expect(page.getByText(workflow_trigger_notification_text)).toBeVisible(); - await expect(page.getByText(workflow_trigger_notification_text)).toBeVisible(); + const run_workflow_btn = page.locator('#workflow_dispatch_dropdown>button'); + await expect(run_workflow_btn).toBeVisible(); - const run_workflow_btn = page.locator('#workflow_dispatch_dropdown>button'); - await expect(run_workflow_btn).toBeVisible(); - - const menu = page.locator('#workflow_dispatch_dropdown>.menu'); - await expect(menu).toBeHidden(); - await run_workflow_btn.click(); - await expect(menu).toBeVisible(); - await save_visual(page); -}); - -test('workflow dispatch error: missing inputs', async ({browser}, workerInfo) => { - test.skip(workerInfo.project.name === 'Mobile Safari', 'Flaky behaviour on mobile safari; see https://codeberg.org/forgejo/forgejo/pulls/3334#issuecomment-2033383'); - - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); - - await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); - - await page.locator('#workflow_dispatch_dropdown>button').click(); - - // Remove the required attribute so we can trigger the error message! - await page.evaluate(() => { - const elem = document.querySelector('input[name="inputs[string2]"]'); - elem?.removeAttribute('required'); + const menu = page.locator('#workflow_dispatch_dropdown>.menu'); + await expect(menu).toBeHidden(); + await run_workflow_btn.click(); + await expect(menu).toBeVisible(); + await save_visual(page); }); - await page.locator('#workflow-dispatch-submit').click(); + test('dispatch error: missing inputs', async ({page}, testInfo) => { + test.skip(testInfo.project.name === 'Mobile Safari', 'Flaky behaviour on mobile safari; see https://codeberg.org/forgejo/forgejo/pulls/3334#issuecomment-2033383'); - await expect(page.getByText('Require value for input "String w/o. default".')).toBeVisible(); - await save_visual(page); -}); + await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); -test('workflow dispatch success', async ({browser}, workerInfo) => { - test.skip(workerInfo.project.name === 'Mobile Safari', 'Flaky behaviour on mobile safari; see https://codeberg.org/forgejo/forgejo/pulls/3334#issuecomment-2033383'); + await page.locator('#workflow_dispatch_dropdown>button').click(); - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); + // Remove the required attribute so we can trigger the error message! + await page.evaluate(() => { + const elem = document.querySelector('input[name="inputs[string2]"]'); + elem?.removeAttribute('required'); + }); - await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); + await page.locator('#workflow-dispatch-submit').click(); - await page.locator('#workflow_dispatch_dropdown>button').click(); + await expect(page.getByText('Require value for input "String w/o. default".')).toBeVisible(); + await save_visual(page); + }); - await page.fill('input[name="inputs[string2]"]', 'abc'); - await save_visual(page); - await page.locator('#workflow-dispatch-submit').click(); + test('dispatch success', async ({page}, testInfo) => { + test.skip(testInfo.project.name === 'Mobile Safari', 'Flaky behaviour on mobile safari; see https://codeberg.org/forgejo/forgejo/pulls/3334#issuecomment-2033383'); + await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); - await expect(page.getByText('Workflow run was successfully requested.')).toBeVisible(); + await page.locator('#workflow_dispatch_dropdown>button').click(); - await expect(page.locator('.run-list>:first-child .run-list-meta', {hasText: 'now'})).toBeVisible(); - await save_visual(page); + await page.fill('input[name="inputs[string2]"]', 'abc'); + await save_visual(page); + await page.locator('#workflow-dispatch-submit').click(); + + await expect(page.getByText('Workflow run was successfully requested.')).toBeVisible(); + + await expect(page.locator('.run-list>:first-child .run-list-meta', {hasText: 'now'})).toBeVisible(); + await save_visual(page); + }); }); test('workflow dispatch box not available for unauthenticated users', async ({page}) => { diff --git a/tests/e2e/dashboard-ci-status.test.e2e.ts b/tests/e2e/dashboard-ci-status.test.e2e.ts index 1d23122b44..800fc951e6 100644 --- a/tests/e2e/dashboard-ci-status.test.e2e.ts +++ b/tests/e2e/dashboard-ci-status.test.e2e.ts @@ -3,21 +3,24 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('Correct link and tooltip', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); +test.describe.configure({retries: 2}); + +test('Correct link and tooltip', async ({page}, testInfo) => { + if (testInfo.retry) { + await page.goto('/user2/test_workflows/actions'); + } + + const searchResponse = page.waitForResponse((resp) => resp.url().includes('/repo/search?') && resp.status() === 200); const response = await page.goto('/?repo-search-query=test_workflows'); expect(response?.status()).toBe(200); + await searchResponse; + const repoStatus = page.locator('.dashboard-repos .repo-owner-name-list > li:nth-child(1) > a:nth-child(2)'); - // wait for network activity to cease (so status was loaded in frontend) - await page.waitForLoadState('networkidle'); // eslint-disable-line playwright/no-networkidle await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000}); await expect(repoStatus).toHaveAttribute('data-tooltip-content', /^(Error|Failure)$/); await save_visual(page); diff --git a/tests/e2e/git-notes.test.e2e.ts b/tests/e2e/git-notes.test.e2e.ts index 8b80a3aa77..4245853b24 100644 --- a/tests/e2e/git-notes.test.e2e.ts +++ b/tests/e2e/git-notes.test.e2e.ts @@ -1,14 +1,10 @@ // @ts-check -import {test, expect} from '@playwright/test'; -import {login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; +import {expect} from '@playwright/test'; +import {save_visual, test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('Change git note', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); +test('Change git note', async ({page}) => { let response = await page.goto('/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/issue-comment.test.e2e.ts b/tests/e2e/issue-comment.test.e2e.ts index 4fce16764b..933e65fa32 100644 --- a/tests/e2e/issue-comment.test.e2e.ts +++ b/tests/e2e/issue-comment.test.e2e.ts @@ -5,14 +5,11 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, login_user, login} from './utils_e2e.ts'; +import {test, save_visual} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('Menu accessibility', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('Menu accessibility', async ({page}) => { await page.goto('/user2/repo1/issues/1'); await expect(page.getByLabel('user2 reacted eyes. Remove eyes')).toBeVisible(); await expect(page.getByLabel('reacted laugh. Remove laugh')).toBeVisible(); @@ -24,9 +21,8 @@ test('Menu accessibility', async ({browser}, workerInfo) => { await expect(page.getByLabel('user1, user2 reacted laugh. Remove laugh')).toBeVisible(); }); -test('Hyperlink paste behaviour', async ({browser}, workerInfo) => { +test('Hyperlink paste behaviour', async ({page}, workerInfo) => { test.skip(['Mobile Safari', 'Mobile Chrome', 'webkit'].includes(workerInfo.project.name), 'Mobile clients seem to have very weird behaviour with this test, which I cannot confirm with real usage'); - const page = await login({browser}, workerInfo); await page.goto('/user2/repo1/issues/new'); await page.locator('textarea').click(); // same URL @@ -58,8 +54,7 @@ test('Hyperlink paste behaviour', async ({browser}, workerInfo) => { await page.locator('textarea').fill(''); }); -test('Always focus edit tab first on edit', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('Always focus edit tab first on edit', async ({page}) => { const response = await page.goto('/user2/repo1/issues/1'); expect(response?.status()).toBe(200); @@ -82,9 +77,8 @@ test('Always focus edit tab first on edit', async ({browser}, workerInfo) => { await save_visual(page); }); -test('Quote reply', async ({browser}, workerInfo) => { +test('Quote reply', async ({page}, workerInfo) => { test.skip(workerInfo.project.name !== 'firefox', 'Uses Firefox specific selection quirks'); - const page = await login({browser}, workerInfo); const response = await page.goto('/user2/repo1/issues/1'); expect(response?.status()).toBe(200); @@ -157,9 +151,8 @@ test('Quote reply', async ({browser}, workerInfo) => { await editorTextarea.fill(''); }); -test('Pull quote reply', async ({browser}, workerInfo) => { +test('Pull quote reply', async ({page}, workerInfo) => { test.skip(workerInfo.project.name !== 'firefox', 'Uses Firefox specific selection quirks'); - const page = await login({browser}, workerInfo); const response = await page.goto('/user2/commitsonpr/pulls/1/files'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/issue-sidebar.test.e2e.ts b/tests/e2e/issue-sidebar.test.e2e.ts index f4d50a13ba..fe2a6cec87 100644 --- a/tests/e2e/issue-sidebar.test.e2e.ts +++ b/tests/e2e/issue-sidebar.test.e2e.ts @@ -7,14 +7,13 @@ /* eslint playwright/expect-expect: ["error", { "assertFunctionNames": ["check_wip"] }] */ import {expect, type Page} from '@playwright/test'; -import {test, save_visual, login_user, login} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); test.describe('Pull: Toggle WIP', () => { const prTitle = 'pull5'; + async function toggle_wip_to({page}, should: boolean) { await page.waitForLoadState('domcontentloaded'); if (should) { @@ -39,8 +38,7 @@ test.describe('Pull: Toggle WIP', () => { } } - test.beforeEach(async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); + test.beforeEach(async ({page}) => { const response = await page.goto('/user2/repo1/pulls/5'); expect(response?.status()).toBe(200); // Status OK // ensure original title @@ -50,9 +48,8 @@ test.describe('Pull: Toggle WIP', () => { await check_wip({page}, false); }); - test('simple toggle', async ({browser}, workerInfo) => { + test('simple toggle', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); await page.goto('/user2/repo1/pulls/5'); // toggle to WIP await toggle_wip_to({page}, true); @@ -62,9 +59,8 @@ test.describe('Pull: Toggle WIP', () => { await check_wip({page}, false); }); - test('manual edit', async ({browser}, workerInfo) => { + test('manual edit', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); await page.goto('/user2/repo1/pulls/5'); // manually edit title to another prefix await page.locator('#issue-title-edit-show').click(); @@ -76,9 +72,8 @@ test.describe('Pull: Toggle WIP', () => { await check_wip({page}, false); }); - test('maximum title length', async ({browser}, workerInfo) => { + test('maximum title length', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); await page.goto('/user2/repo1/pulls/5'); // check maximum title length is handled gracefully const maxLenStr = prTitle + 'a'.repeat(240); @@ -96,17 +91,16 @@ test.describe('Pull: Toggle WIP', () => { }); }); -test('Issue: Labels', async ({browser}, workerInfo) => { +test('Issue: Labels', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - async function submitLabels({page}: {page: Page}) { + async function submitLabels({page}: { page: Page }) { const submitted = page.waitForResponse('/user2/repo1/issues/labels'); await page.locator('textarea').first().click(); // close via unrelated element await submitted; await page.waitForLoadState(); } - const page = await login({browser}, workerInfo); // select label list in sidebar only const labelList = page.locator('.issue-content-right .labels-list a'); const response = await page.goto('/user2/repo1/issues/1'); @@ -144,9 +138,8 @@ test('Issue: Labels', async ({browser}, workerInfo) => { await expect(labelList.filter({hasText: 'label1'})).toBeVisible(); }); -test('Issue: Assignees', async ({browser}, workerInfo) => { +test('Issue: Assignees', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); // select label list in sidebar only const assigneesList = page.locator('.issue-content-right .assignees.list .selected .item a'); @@ -182,9 +175,8 @@ test('Issue: Assignees', async ({browser}, workerInfo) => { await expect(page.locator('.ui.assignees.list .item.no-select')).toBeHidden(); }); -test('New Issue: Assignees', async ({browser}, workerInfo) => { +test('New Issue: Assignees', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); // select label list in sidebar only const assigneesList = page.locator('.issue-content-right .assignees.list .selected .item'); @@ -224,9 +216,8 @@ test('New Issue: Assignees', async ({browser}, workerInfo) => { await save_visual(page); }); -test('Issue: Milestone', async ({browser}, workerInfo) => { +test('Issue: Milestone', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); const response = await page.goto('/user2/repo1/issues/1'); expect(response?.status()).toBe(200); @@ -248,9 +239,8 @@ test('Issue: Milestone', async ({browser}, workerInfo) => { await expect(page.locator('.timeline-item.event').last()).toContainText('user2 removed this from the milestone1 milestone'); }); -test('New Issue: Milestone', async ({browser}, workerInfo) => { +test('New Issue: Milestone', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Unable to get tests working on Safari Mobile, see https://codeberg.org/forgejo/forgejo/pulls/3445#issuecomment-1789636'); - const page = await login({browser}, workerInfo); const response = await page.goto('/user2/repo1/issues/new'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/markdown-editor.test.e2e.ts b/tests/e2e/markdown-editor.test.e2e.ts index ca2d6e01b6..762113d563 100644 --- a/tests/e2e/markdown-editor.test.e2e.ts +++ b/tests/e2e/markdown-editor.test.e2e.ts @@ -5,21 +5,16 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, load_logged_in_context, login_user} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('Markdown image preview behaviour', async ({browser}, workerInfo) => { +test('Markdown image preview behaviour', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Flaky behaviour on mobile safari;'); - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - // Editing the root README.md file for image preview const editPath = '/user2/repo1/src/branch/master/README.md'; - const page = await context.newPage(); const response = await page.goto(editPath, {waitUntil: 'domcontentloaded'}); expect(response?.status()).toBe(200); @@ -43,12 +38,9 @@ test('Markdown image preview behaviour', async ({browser}, workerInfo) => { await save_visual(page); }); -test('markdown indentation', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - +test('markdown indentation', async ({page}) => { const initText = `* first\n* second\n* third\n* last`; - const page = await context.newPage(); const response = await page.goto('/user2/repo1/issues/new'); expect(response?.status()).toBe(200); @@ -116,12 +108,9 @@ test('markdown indentation', async ({browser}, workerInfo) => { await expect(textarea).toHaveValue(initText); }); -test('markdown list continuation', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - +test('markdown list continuation', async ({page}) => { const initText = `* first\n* second\n* third\n* last`; - const page = await context.newPage(); const response = await page.goto('/user2/repo1/issues/new'); expect(response?.status()).toBe(200); @@ -213,10 +202,7 @@ test('markdown list continuation', async ({browser}, workerInfo) => { } }); -test('markdown insert table', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - - const page = await context.newPage(); +test('markdown insert table', async ({page}) => { const response = await page.goto('/user2/repo1/issues/new'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/org-settings.test.e2e.ts b/tests/e2e/org-settings.test.e2e.ts index 22a8bc0e2d..df554e0674 100644 --- a/tests/e2e/org-settings.test.e2e.ts +++ b/tests/e2e/org-settings.test.e2e.ts @@ -5,16 +5,13 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, login_user, login} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('org team settings', async ({browser}, workerInfo) => { +test('org team settings', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Cannot get it to work - as usual'); - const page = await login({browser}, workerInfo); const response = await page.goto('/org/org3/teams/team1/edit'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/profile_actions.test.e2e.ts b/tests/e2e/profile_actions.test.e2e.ts index 65090e62b2..a66dc43aab 100644 --- a/tests/e2e/profile_actions.test.e2e.ts +++ b/tests/e2e/profile_actions.test.e2e.ts @@ -5,13 +5,11 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; -test('Follow actions', async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); +test.use({user: 'user2'}); +test('Follow actions', async ({page}) => { await page.goto('/user1'); // Check if following and then unfollowing works. diff --git a/tests/e2e/reaction-selectors.test.e2e.ts b/tests/e2e/reaction-selectors.test.e2e.ts index 3ce71b24d7..54b7d91869 100644 --- a/tests/e2e/reaction-selectors.test.e2e.ts +++ b/tests/e2e/reaction-selectors.test.e2e.ts @@ -4,11 +4,9 @@ // @watch end import {expect, type Locator} from '@playwright/test'; -import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); const assertReactionCounts = (comment: Locator, counts: unknown) => expect(async () => { @@ -26,6 +24,7 @@ const assertReactionCounts = (comment: Locator, counts: unknown) => ]), ), ); + // eslint-disable-next-line playwright/no-standalone-expect return expect(reactions).toStrictEqual(counts); }).toPass(); @@ -35,10 +34,7 @@ async function toggleReaction(menu: Locator, reaction: string) { await menu.locator(`[role=menuitem][data-reaction-content="${reaction}"]`).click(); } -test('Reaction Selectors', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); - +test('Reaction Selectors', async ({page}) => { const response = await page.goto('/user2/repo1/issues/1'); expect(response?.status()).toBe(200); diff --git a/tests/e2e/release.test.e2e.ts b/tests/e2e/release.test.e2e.ts index fefa446c59..49c67793e6 100644 --- a/tests/e2e/release.test.e2e.ts +++ b/tests/e2e/release.test.e2e.ts @@ -9,24 +9,18 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, save_visual, load_logged_in_context} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); test.describe.configure({ timeout: 30000, }); -test('External Release Attachments', async ({browser, isMobile}, workerInfo) => { +test('External Release Attachments', async ({page, isMobile}) => { test.skip(isMobile); - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - /** @type {import('@playwright/test').Page} */ - const page = await context.newPage(); - // Click "New Release" await page.goto('/user2/repo2/releases'); await page.click('.button.small.primary'); diff --git a/tests/e2e/repo-code.test.e2e.ts b/tests/e2e/repo-code.test.e2e.ts index 264dd3a8e0..335fb5b7f5 100644 --- a/tests/e2e/repo-code.test.e2e.ts +++ b/tests/e2e/repo-code.test.e2e.ts @@ -5,13 +5,9 @@ // @watch end import {expect, type Page} from '@playwright/test'; -import {test, save_visual, login_user, login} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; import {accessibilityCheck} from './shared/accessibility.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); - async function assertSelectedLines(page: Page, nums: string[]) { const pageAssertions = async () => { expect( @@ -81,20 +77,23 @@ test('Readable diff', async ({page}, workerInfo) => { } }); -test('Username highlighted in commits', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); - await page.goto('/user2/mentions-highlighted/commits/branch/main'); - // check first commit - await page.getByRole('link', {name: 'A commit message which'}).click(); - await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); - await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); - await accessibilityCheck({page}, ['.commit-header'], [], []); - await save_visual(page); - // check second commit - await page.goto('/user2/mentions-highlighted/commits/branch/main'); - await page.locator('tbody').getByRole('link', {name: 'Another commit which mentions'}).click(); - await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); - await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); - await accessibilityCheck({page}, ['.commit-header'], [], []); - await save_visual(page); +test.describe('As authenticated user', () => { + test.use({user: 'user2'}); + + test('Username highlighted in commits', async ({page}) => { + await page.goto('/user2/mentions-highlighted/commits/branch/main'); + // check first commit + await page.getByRole('link', {name: 'A commit message which'}).click(); + await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); + await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); + await accessibilityCheck({page}, ['.commit-header'], [], []); + await save_visual(page); + // check second commit + await page.goto('/user2/mentions-highlighted/commits/branch/main'); + await page.locator('tbody').getByRole('link', {name: 'Another commit which mentions'}).click(); + await expect(page.getByRole('link', {name: '@user2'})).toHaveCSS('background-color', /(.*)/); + await expect(page.getByRole('link', {name: '@user1'})).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)'); + await accessibilityCheck({page}, ['.commit-header'], [], []); + await save_visual(page); + }); }); diff --git a/tests/e2e/repo-migrate.test.e2e.ts b/tests/e2e/repo-migrate.test.e2e.ts index a0f9ab6c80..428c2cb171 100644 --- a/tests/e2e/repo-migrate.test.e2e.ts +++ b/tests/e2e/repo-migrate.test.e2e.ts @@ -3,15 +3,13 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test, save_visual, test_context} from './utils_e2e.ts'; -test.beforeAll(({browser}, workerInfo) => login_user(browser, workerInfo, 'user2')); +test.use({user: 'user2'}); -test('Migration Progress Page', async ({page: unauthedPage, browser}, workerInfo) => { +test('Migration Progress Page', async ({page, browser}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Flaky actionability checks on Mobile Safari'); - const page = await (await load_logged_in_context(browser, workerInfo, 'user2')).newPage(); - expect((await page.goto('/user2/invalidrepo'))?.status(), 'repo should not exist yet').toBe(404); await page.goto('/repo/migrate?service_type=1'); @@ -23,10 +21,12 @@ test('Migration Progress Page', async ({page: unauthedPage, browser}, workerInfo await form.locator('button.primary').click({timeout: 5000}); await expect(page).toHaveURL('user2/invalidrepo'); await save_visual(page); - // page screenshot of unauthedPage is checked automatically after the test + // page screenshot of unauthenticatedPage is checked automatically after the test - expect((await unauthedPage.goto('/user2/invalidrepo'))?.status(), 'public migration page should be accessible').toBe(200); - await expect(unauthedPage.locator('#repo_migrating_progress')).toBeVisible(); + const ctx = await test_context(browser); + const unauthenticatedPage = await ctx.newPage(); + expect((await unauthenticatedPage.goto('/user2/invalidrepo'))?.status(), 'public migration page should be accessible').toBe(200); + await expect(unauthenticatedPage.locator('#repo_migrating_progress')).toBeVisible(); await page.reload(); await expect(page.locator('#repo_migrating_failed')).toBeVisible(); diff --git a/tests/e2e/repo-new.test.e2e.ts b/tests/e2e/repo-new.test.e2e.ts index c9cc29ad56..ad202825a0 100644 --- a/tests/e2e/repo-new.test.e2e.ts +++ b/tests/e2e/repo-new.test.e2e.ts @@ -4,15 +4,12 @@ // @watch end import {expect} from '@playwright/test'; -import {test, dynamic_id, save_visual, login_user, login} from './utils_e2e.ts'; +import {test, dynamic_id, save_visual} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('New repo: invalid', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('New repo: invalid', async ({page}) => { const response = await page.goto('/repo/create'); expect(response?.status()).toBe(200); // check that relevant form content is hidden or available @@ -28,8 +25,7 @@ test('New repo: invalid', async ({browser}, workerInfo) => { await save_visual(page); }); -test('New repo: initialize', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('New repo: initialize', async ({page}, workerInfo) => { const response = await page.goto('/repo/create'); expect(response?.status()).toBe(200); // check that relevant form content is hidden or available @@ -62,8 +58,7 @@ test('New repo: initialize', async ({browser}, workerInfo) => { await save_visual(page); }); -test('New repo: initialize later', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('New repo: initialize later', async ({page}) => { const response = await page.goto('/repo/create'); expect(response?.status()).toBe(200); @@ -97,9 +92,8 @@ test('New repo: initialize later', async ({browser}, workerInfo) => { await save_visual(page); }); -test('New repo: from template', async ({browser}, workerInfo) => { +test('New repo: from template', async ({page}, workerInfo) => { test.skip(['Mobile Safari', 'webkit'].includes(workerInfo.project.name), 'WebKit browsers seem to have CORS issues with localhost here.'); - const page = await login({browser}, workerInfo); const response = await page.goto('/repo/create'); expect(response?.status()).toBe(200); @@ -114,8 +108,7 @@ test('New repo: from template', async ({browser}, workerInfo) => { await save_visual(page); }); -test('New repo: label set', async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); +test('New repo: label set', async ({page}) => { await page.goto('/repo/create'); const reponame = dynamic_id(); diff --git a/tests/e2e/repo-settings.test.e2e.ts b/tests/e2e/repo-settings.test.e2e.ts index 113b15181b..3d260866fb 100644 --- a/tests/e2e/repo-settings.test.e2e.ts +++ b/tests/e2e/repo-settings.test.e2e.ts @@ -7,16 +7,13 @@ // @watch end import {expect} from '@playwright/test'; -import {test, save_visual, login_user, login} from './utils_e2e.ts'; +import {test, save_visual} from './utils_e2e.ts'; import {validate_form} from './shared/forms.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); +test.use({user: 'user2'}); -test('repo webhook settings', async ({browser}, workerInfo) => { +test('repo webhook settings', async ({page}, workerInfo) => { test.skip(workerInfo.project.name === 'Mobile Safari', 'Cannot get it to work - as usual'); - const page = await login({browser}, workerInfo); const response = await page.goto('/user2/repo1/settings/hooks/forgejo/new'); expect(response?.status()).toBe(200); @@ -35,9 +32,8 @@ test('repo webhook settings', async ({browser}, workerInfo) => { }); test.describe('repo branch protection settings', () => { - test('form', async ({browser}, workerInfo) => { - test.skip(workerInfo.project.name === 'Mobile Safari', 'Cannot get it to work - as usual'); - const page = await login({browser}, workerInfo); + test('form', async ({page}, {project}) => { + test.skip(project.name === 'Mobile Safari', 'Cannot get it to work - as usual'); const response = await page.goto('/user2/repo1/settings/branches/edit'); expect(response?.status()).toBe(200); @@ -56,8 +52,7 @@ test.describe('repo branch protection settings', () => { await save_visual(page); }); - test.afterEach(async ({browser}, workerInfo) => { - const page = await login({browser}, workerInfo); + test.afterEach(async ({page}) => { // delete the rule for the next test await page.goto('/user2/repo1/settings/branches/'); await page.waitForLoadState('domcontentloaded'); diff --git a/tests/e2e/right-settings-button.test.e2e.ts b/tests/e2e/right-settings-button.test.e2e.ts index bfb1800a27..e1c40fdd4d 100644 --- a/tests/e2e/right-settings-button.test.e2e.ts +++ b/tests/e2e/right-settings-button.test.e2e.ts @@ -5,19 +5,12 @@ // @watch end import {expect} from '@playwright/test'; -import {test, login_user, load_logged_in_context} from './utils_e2e.ts'; +import {test} from './utils_e2e.ts'; -test.beforeAll(async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user2'); -}); - -test.describe('desktop viewport', () => { - test.use({viewport: {width: 1920, height: 300}}); - - test('Settings button on right of repo header', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); +test.describe('desktop viewport as user 2', () => { + test.use({user: 'user2', viewport: {width: 1920, height: 300}}); + test('Settings button on right of repo header', async ({page}) => { await page.goto('/user2/repo1'); const settingsBtn = page.locator('.overflow-menu-items>#settings-btn'); @@ -27,24 +20,7 @@ test.describe('desktop viewport', () => { await expect(page.locator('.overflow-menu-button')).toHaveCount(0); }); - test('Settings button on right of repo header also when add more button is shown', async ({browser}, workerInfo) => { - await login_user(browser, workerInfo, 'user12'); - const context = await load_logged_in_context(browser, workerInfo, 'user12'); - const page = await context.newPage(); - - await page.goto('/user12/repo10'); - - const settingsBtn = page.locator('.overflow-menu-items>#settings-btn'); - await expect(settingsBtn).toBeVisible(); - await expect(settingsBtn).toHaveClass(/right/); - - await expect(page.locator('.overflow-menu-button')).toHaveCount(0); - }); - - test('Settings button on right of org header', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); - + test('Settings button on right of org header', async ({page}) => { await page.goto('/org3'); const settingsBtn = page.locator('.overflow-menu-items>#settings-btn'); @@ -53,6 +29,24 @@ test.describe('desktop viewport', () => { await expect(page.locator('.overflow-menu-button')).toHaveCount(0); }); +}); + +test.describe('desktop viewport as user12', () => { + test.use({user: 'user12', viewport: {width: 1920, height: 300}}); + + test('Settings button on right of repo header also when add more button is shown', async ({page}) => { + await page.goto('/user12/repo10'); + + const settingsBtn = page.locator('.overflow-menu-items>#settings-btn'); + await expect(settingsBtn).toBeVisible(); + await expect(settingsBtn).toHaveClass(/right/); + + await expect(page.locator('.overflow-menu-button')).toHaveCount(0); + }); +}); + +test.describe('desktop viewport, unauthenticated', () => { + test.use({viewport: {width: 1920, height: 300}}); test('User overview overflow menu should not be influenced', async ({page}) => { await page.goto('/user2'); @@ -64,12 +58,9 @@ test.describe('desktop viewport', () => { }); test.describe('small viewport', () => { - test.use({viewport: {width: 800, height: 300}}); - - test('Settings button in overflow menu of repo header', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); + test.use({user: 'user2', viewport: {width: 800, height: 300}}); + test('Settings button in overflow menu of repo header', async ({page}) => { await page.goto('/user2/repo1'); await expect(page.locator('.overflow-menu-items>#settings-btn')).toHaveCount(0); @@ -89,10 +80,7 @@ test.describe('small viewport', () => { expect(Array.from(new Set(items))).toHaveLength(items.length); }); - test('Settings button in overflow menu of org header', async ({browser}, workerInfo) => { - const context = await load_logged_in_context(browser, workerInfo, 'user2'); - const page = await context.newPage(); - + test('Settings button in overflow menu of org header', async ({page}) => { await page.goto('/org3'); await expect(page.locator('.overflow-menu-items>#settings-btn')).toHaveCount(0); @@ -111,6 +99,10 @@ test.describe('small viewport', () => { const items = shownItems.concat(overflowItems); expect(Array.from(new Set(items))).toHaveLength(items.length); }); +}); + +test.describe('small viewport, unauthenticated', () => { + test.use({viewport: {width: 800, height: 300}}); test('User overview overflow menu should not be influenced', async ({page}) => { await page.goto('/user2'); diff --git a/tests/e2e/utils_e2e.ts b/tests/e2e/utils_e2e.ts index 7e25441ea3..80412e437d 100644 --- a/tests/e2e/utils_e2e.ts +++ b/tests/e2e/utils_e2e.ts @@ -1,9 +1,31 @@ import {expect, test as baseTest, type Browser, type BrowserContextOptions, type APIRequestContext, type TestInfo, type Page} from '@playwright/test'; -export const test = baseTest.extend({ - context: async ({browser}, use) => { - return use(await test_context(browser)); +import * as path from 'node:path'; + +const AUTH_PATH = 'tests/e2e/.auth'; + +type AuthScope = 'logout' | 'shared' | 'webauthn'; + +export type TestOptions = { + forEachTest: void + user: string | null; + authScope: AuthScope; +}; + +export const test = baseTest.extend({ + context: async ({browser, user, authScope, contextOptions}, use, {project}) => { + if (user && authScope) { + const browserName = project.name.toLowerCase().replace(' ', '-'); + contextOptions.storageState = path.join(AUTH_PATH, `state-${browserName}-${user}-${authScope}.json`); + } else { + // if no user is given, ensure to have clean state + contextOptions.storageState = {cookies: [], origins: []}; + } + + return use(await test_context(browser, contextOptions)); }, + user: null, + authScope: 'shared', // see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks forEachTest: [async ({page}, use) => { await use(); @@ -15,7 +37,7 @@ export const test = baseTest.extend({ }, {auto: true}], }); -async function test_context(browser: Browser, options?: BrowserContextOptions) { +export async function test_context(browser: Browser, options?: BrowserContextOptions) { const context = await browser.newContext(options); context.on('page', (page) => { diff --git a/tests/e2e/utils_e2e_test.go b/tests/e2e/utils_e2e_test.go index bf1a8a418c..96fd905363 100644 --- a/tests/e2e/utils_e2e_test.go +++ b/tests/e2e/utils_e2e_test.go @@ -5,17 +5,27 @@ package e2e import ( "context" + "crypto/rand" + "encoding/hex" + "fmt" "net" "net/http" "net/url" "os" + "path/filepath" "regexp" + "strings" "testing" "time" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/json" + modules_session "code.gitea.io/gitea/modules/session" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/tests" + "code.forgejo.org/go-chi/session" "github.com/stretchr/testify/require" ) @@ -25,6 +35,8 @@ func onForgejoRunTB(t testing.TB, callback func(testing.TB, *url.URL), prepare . if len(prepare) == 0 || prepare[0] { defer tests.PrepareTestEnv(t, 1)() } + createSessions(t) + s := http.Server{ Handler: testE2eWebRoutes, } @@ -64,3 +76,118 @@ func onForgejoRun(t *testing.T, callback func(*testing.T, *url.URL), prepare ... callback(t.(*testing.T), u) }, prepare...) } + +func createSessions(t testing.TB) { + t.Helper() + // copied from playwright.config.ts + browsers := []string{ + "chromium", + "firefox", + "webkit", + "Mobile Chrome", + "Mobile Safari", + } + scopes := []string{ + "shared", + } + users := []string{ + "user1", + "user2", + "user12", + "user40", + } + + authState := filepath.Join(filepath.Dir(setting.AppPath), "tests", "e2e", ".auth") + err := os.RemoveAll(authState) + require.NoError(t, err) + + err = os.MkdirAll(authState, os.ModePerm) + require.NoError(t, err) + + createSessionCookie := stateHelper(t) + + for _, user := range users { + u := unittest.AssertExistsAndLoadBean(t, &user_model.User{LowerName: strings.ToLower(user)}) + for _, browser := range browsers { + for _, scope := range scopes { + stateFile := strings.ReplaceAll(strings.ToLower(fmt.Sprintf("state-%s-%s-%s.json", browser, user, scope)), " ", "-") + createSessionCookie(filepath.Join(authState, stateFile), u) + } + } + } +} + +func stateHelper(t testing.TB) func(stateFile string, user *user_model.User) { + type Cookie struct { + Name string `json:"name"` + Value string `json:"value"` + Domain string `json:"domain"` + Path string `json:"path"` + Expires int `json:"expires"` + HTTPOnly bool `json:"httpOnly"` + Secure bool `json:"secure"` + SameSite string `json:"sameSite"` + } + + type BrowserState struct { + Cookies []Cookie `json:"cookies"` + Origins []string `json:"origins"` + } + + options := session.Options{ + Provider: setting.SessionConfig.Provider, + ProviderConfig: setting.SessionConfig.ProviderConfig, + CookieName: setting.SessionConfig.CookieName, + CookiePath: setting.SessionConfig.CookiePath, + Gclifetime: setting.SessionConfig.Gclifetime, + Maxlifetime: setting.SessionConfig.Maxlifetime, + Secure: setting.SessionConfig.Secure, + SameSite: setting.SessionConfig.SameSite, + Domain: setting.SessionConfig.Domain, + } + + opt := session.PrepareOptions([]session.Options{options}) + + vsp := modules_session.VirtualSessionProvider{} + err := vsp.Init(opt.Maxlifetime, opt.ProviderConfig) + require.NoError(t, err) + + return func(stateFile string, user *user_model.User) { + buf := make([]byte, opt.IDLength/2) + _, err = rand.Read(buf) + require.NoError(t, err) + + sessionID := hex.EncodeToString(buf) + + s, err := vsp.Read(sessionID) + require.NoError(t, err) + + err = s.Set("uid", user.ID) + require.NoError(t, err) + + err = s.Release() + require.NoError(t, err) + + state := BrowserState{ + Cookies: []Cookie{ + { + Name: opt.CookieName, + Value: sessionID, + Domain: setting.Domain, + Path: "/", + Expires: -1, + HTTPOnly: true, + Secure: false, + SameSite: "Lax", + }, + }, + Origins: []string{}, + } + + jsonData, err := json.Marshal(state) + require.NoError(t, err) + + err = os.WriteFile(stateFile, jsonData, 0o644) + require.NoError(t, err) + } +} From 26b7c6b86a8031c5d2cb2c27c7fafc057f46165b Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Fri, 17 Jan 2025 01:07:22 +0000 Subject: [PATCH 002/125] [v10.0/forgejo] tests(e2e): Various fixes to visual testing (#6587) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6569 Co-authored-by: Otto Richter Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6587 Reviewed-by: Otto Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- tests/e2e/actions.test.e2e.ts | 1 + tests/e2e/clipboard-copy.test.e2e.ts | 5 ++++- tests/e2e/dashboard-ci-status.test.e2e.ts | 5 +++-- tests/e2e/e2e_test.go | 7 ++++++- tests/e2e/example.test.e2e.ts | 3 ++- tests/e2e/explore.test.e2e.ts | 3 ++- tests/e2e/markup.test.e2e.ts | 3 ++- tests/e2e/repo-code.test.e2e.ts | 2 ++ tests/e2e/repo-commitgraph.test.e2e.ts | 3 ++- tests/e2e/repo-migrate.test.e2e.ts | 3 ++- tests/e2e/repo-wiki.test.e2e.ts | 4 +++- tests/e2e/right-settings-button.test.e2e.ts | 5 ++++- tests/e2e/utils_e2e.ts | 10 +--------- 13 files changed, 34 insertions(+), 20 deletions(-) diff --git a/tests/e2e/actions.test.e2e.ts b/tests/e2e/actions.test.e2e.ts index 6236fe70d3..4e93b89ee0 100644 --- a/tests/e2e/actions.test.e2e.ts +++ b/tests/e2e/actions.test.e2e.ts @@ -71,4 +71,5 @@ test('workflow dispatch box not available for unauthenticated users', async ({pa await page.goto('/user2/test_workflows/actions?workflow=test-dispatch.yml&actor=0&status=0'); await expect(page.locator('body')).not.toContainText(workflow_trigger_notification_text); + await save_visual(page); }); diff --git a/tests/e2e/clipboard-copy.test.e2e.ts b/tests/e2e/clipboard-copy.test.e2e.ts index 70a3425868..2ae0e0dfff 100644 --- a/tests/e2e/clipboard-copy.test.e2e.ts +++ b/tests/e2e/clipboard-copy.test.e2e.ts @@ -8,7 +8,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test('copy src file path to clipboard', async ({page}, workerInfo) => { test.skip(['Mobile Safari', 'webkit'].includes(workerInfo.project.name), 'Apple clipboard API addon - starting at just $499!'); @@ -19,6 +19,7 @@ test('copy src file path to clipboard', async ({page}, workerInfo) => { await page.click('[data-clipboard-text]'); const clipboardText = await page.evaluate(() => navigator.clipboard.readText()); expect(clipboardText).toContain('README.md'); + await save_visual(page); }); test('copy diff file path to clipboard', async ({page}, workerInfo) => { @@ -30,4 +31,6 @@ test('copy diff file path to clipboard', async ({page}, workerInfo) => { await page.click('[data-clipboard-text]'); const clipboardText = await page.evaluate(() => navigator.clipboard.readText()); expect(clipboardText).toContain('README.md'); + await expect(page.getByText('Copied')).toBeVisible(); + await save_visual(page); }); diff --git a/tests/e2e/dashboard-ci-status.test.e2e.ts b/tests/e2e/dashboard-ci-status.test.e2e.ts index 800fc951e6..d35fe299ff 100644 --- a/tests/e2e/dashboard-ci-status.test.e2e.ts +++ b/tests/e2e/dashboard-ci-status.test.e2e.ts @@ -3,7 +3,7 @@ // @watch end import {expect} from '@playwright/test'; -import {save_visual, test} from './utils_e2e.ts'; +import {test} from './utils_e2e.ts'; test.use({user: 'user2'}); @@ -23,5 +23,6 @@ test('Correct link and tooltip', async ({page}, testInfo) => { const repoStatus = page.locator('.dashboard-repos .repo-owner-name-list > li:nth-child(1) > a:nth-child(2)'); await expect(repoStatus).toHaveAttribute('href', '/user2/test_workflows/actions', {timeout: 10000}); await expect(repoStatus).toHaveAttribute('data-tooltip-content', /^(Error|Failure)$/); - await save_visual(page); + // ToDo: Ensure stable screenshot of dashboard. Known to be flaky: https://code.forgejo.org/forgejo/visual-browser-testing/commit/206d4cfb7a4af6d8d7043026cdd4d63708798b2a + // await save_visual(page); }); diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index b8c89625c0..245bd347b8 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -89,6 +89,7 @@ func TestE2e(t *testing.T) { runArgs := []string{"npx", "playwright", "test"} + _, testVisual := os.LookupEnv("VISUAL_TEST") // To update snapshot outputs if _, set := os.LookupEnv("ACCEPT_VISUAL"); set { runArgs = append(runArgs, "--update-snapshots") @@ -112,6 +113,10 @@ func TestE2e(t *testing.T) { onForgejoRun(t, func(*testing.T, *url.URL) { defer DeclareGitRepos(t)() thisTest := runArgs + // when all tests are run, use unique artifacts directories per test to preserve artifacts from other tests + if testVisual { + thisTest = append(thisTest, "--output=tests/e2e/test-artifacts/"+testname) + } thisTest = append(thisTest, path) cmd := exec.Command(runArgs[0], thisTest...) cmd.Env = os.Environ() @@ -121,7 +126,7 @@ func TestE2e(t *testing.T) { cmd.Stderr = os.Stderr err := cmd.Run() - if err != nil { + if err != nil && !testVisual { log.Fatal("Playwright Failed: %s", err) } }) diff --git a/tests/e2e/example.test.e2e.ts b/tests/e2e/example.test.e2e.ts index b2a679a82d..97c5b8684b 100644 --- a/tests/e2e/example.test.e2e.ts +++ b/tests/e2e/example.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test('Load Homepage', async ({page}) => { const response = await page.goto('/'); @@ -26,6 +26,7 @@ test('Register Form', async ({page}, workerInfo) => { expect(page.url()).toBe(`${workerInfo.project.use.baseURL}/`); await expect(page.locator('.secondary-nav span>img.ui.avatar')).toBeVisible(); await expect(page.locator('.ui.positive.message.flash-success')).toHaveText('Account was successfully created. Welcome!'); + await save_visual(page); }); // eslint-disable-next-line playwright/no-skipped-test diff --git a/tests/e2e/explore.test.e2e.ts b/tests/e2e/explore.test.e2e.ts index 44c9b21f58..1bb5af3cc6 100644 --- a/tests/e2e/explore.test.e2e.ts +++ b/tests/e2e/explore.test.e2e.ts @@ -7,7 +7,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test('Explore view taborder', async ({page}) => { await page.goto('/explore/repos'); @@ -42,4 +42,5 @@ test('Explore view taborder', async ({page}) => { } } expect(res).toBe(exp); + await save_visual(page); }); diff --git a/tests/e2e/markup.test.e2e.ts b/tests/e2e/markup.test.e2e.ts index 2726942d57..398a0a6300 100644 --- a/tests/e2e/markup.test.e2e.ts +++ b/tests/e2e/markup.test.e2e.ts @@ -3,7 +3,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test('markup with #xyz-mode-only', async ({page}) => { const response = await page.goto('/user2/repo1/issues/1'); @@ -13,4 +13,5 @@ test('markup with #xyz-mode-only', async ({page}) => { await expect(comment).toBeVisible(); await expect(comment.locator('[src$="#gh-light-mode-only"]')).toBeVisible(); await expect(comment.locator('[src$="#gh-dark-mode-only"]')).toBeHidden(); + await save_visual(page); }); diff --git a/tests/e2e/repo-code.test.e2e.ts b/tests/e2e/repo-code.test.e2e.ts index 335fb5b7f5..11b710c956 100644 --- a/tests/e2e/repo-code.test.e2e.ts +++ b/tests/e2e/repo-code.test.e2e.ts @@ -49,6 +49,7 @@ test('Line Range Selection', async ({page}) => { // out-of-bounds end line await page.goto(`${filePath}#L1-L100`); await assertSelectedLines(page, ['1', '2', '3']); + await save_visual(page); }); test('Readable diff', async ({page}, workerInfo) => { @@ -75,6 +76,7 @@ test('Readable diff', async ({page}, workerInfo) => { await expect(page.getByText(thisDiff.added, {exact: true})).toHaveCSS('background-color', 'rgb(134, 239, 172)'); } } + await save_visual(page); }); test.describe('As authenticated user', () => { diff --git a/tests/e2e/repo-commitgraph.test.e2e.ts b/tests/e2e/repo-commitgraph.test.e2e.ts index 5f0cad117a..39c5661900 100644 --- a/tests/e2e/repo-commitgraph.test.e2e.ts +++ b/tests/e2e/repo-commitgraph.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test('Commit graph overflow', async ({page}) => { await page.goto('/user2/diff-test/graph'); @@ -28,4 +28,5 @@ test('Switch branch', async ({page}) => { await expect(page.locator('#loading-indicator')).toBeHidden(); await expect(page.locator('#rel-container')).toBeVisible(); await expect(page.locator('#rev-container')).toBeVisible(); + await save_visual(page); }); diff --git a/tests/e2e/repo-migrate.test.e2e.ts b/tests/e2e/repo-migrate.test.e2e.ts index 428c2cb171..5e67f89ed1 100644 --- a/tests/e2e/repo-migrate.test.e2e.ts +++ b/tests/e2e/repo-migrate.test.e2e.ts @@ -21,7 +21,6 @@ test('Migration Progress Page', async ({page, browser}, workerInfo) => { await form.locator('button.primary').click({timeout: 5000}); await expect(page).toHaveURL('user2/invalidrepo'); await save_visual(page); - // page screenshot of unauthenticatedPage is checked automatically after the test const ctx = await test_context(browser); const unauthenticatedPage = await ctx.newPage(); @@ -37,4 +36,6 @@ test('Migration Progress Page', async ({page, browser}, workerInfo) => { await save_visual(page); await deleteModal.getByRole('button', {name: 'Delete repository'}).click(); await expect(page).toHaveURL('/'); + // checked last to preserve the order of screenshots from first run + await save_visual(unauthenticatedPage); }); diff --git a/tests/e2e/repo-wiki.test.e2e.ts b/tests/e2e/repo-wiki.test.e2e.ts index f32fe3fc91..4ce66da8bc 100644 --- a/tests/e2e/repo-wiki.test.e2e.ts +++ b/tests/e2e/repo-wiki.test.e2e.ts @@ -4,7 +4,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; for (const searchTerm of ['space', 'consectetur']) { for (const width of [null, 2560, 4000]) { @@ -23,6 +23,7 @@ for (const searchTerm of ['space', 'consectetur']) { await page.getByPlaceholder('Search wiki').dispatchEvent('keyup'); // timeout is necessary because HTMX search could be slow await expect(page.locator('#wiki-search a[href]')).toBeInViewport({ratio: 1}); + await save_visual(page); }); } } @@ -36,4 +37,5 @@ test(`Search results show titles (and not file names)`, async ({page}, workerInf // so we manually "type" the last letter await page.getByPlaceholder('Search wiki').dispatchEvent('keyup'); await expect(page.locator('#wiki-search a[href] b')).toHaveText('Page With Spaced Name'); + await save_visual(page); }); diff --git a/tests/e2e/right-settings-button.test.e2e.ts b/tests/e2e/right-settings-button.test.e2e.ts index e1c40fdd4d..3bea329ba0 100644 --- a/tests/e2e/right-settings-button.test.e2e.ts +++ b/tests/e2e/right-settings-button.test.e2e.ts @@ -5,7 +5,7 @@ // @watch end import {expect} from '@playwright/test'; -import {test} from './utils_e2e.ts'; +import {save_visual, test} from './utils_e2e.ts'; test.describe('desktop viewport as user 2', () => { test.use({user: 'user2', viewport: {width: 1920, height: 300}}); @@ -54,6 +54,7 @@ test.describe('desktop viewport, unauthenticated', () => { await expect(page.locator('.overflow-menu-items>#settings-btn')).toHaveCount(0); await expect(page.locator('.overflow-menu-button')).toHaveCount(0); + await save_visual(page); }); }); @@ -78,6 +79,7 @@ test.describe('small viewport', () => { const items = shownItems.concat(overflowItems); expect(Array.from(new Set(items))).toHaveLength(items.length); + await save_visual(page); }); test('Settings button in overflow menu of org header', async ({page}) => { @@ -121,5 +123,6 @@ test.describe('small viewport, unauthenticated', () => { const items = shownItems.concat(overflowItems); expect(Array.from(new Set(items))).toHaveLength(items.length); + await save_visual(page); }); }); diff --git a/tests/e2e/utils_e2e.ts b/tests/e2e/utils_e2e.ts index 80412e437d..ff921a2cf3 100644 --- a/tests/e2e/utils_e2e.ts +++ b/tests/e2e/utils_e2e.ts @@ -26,15 +26,6 @@ export const test = baseTest.extend({ }, user: null, authScope: 'shared', - // see https://playwright.dev/docs/test-fixtures#adding-global-beforeeachaftereach-hooks - forEachTest: [async ({page}, use) => { - await use(); - // some tests create a new page which is not yet available here - // only operate on tests that make the URL available - if (page.url() !== 'about:blank') { - await save_visual(page); - } - }, {auto: true}], }); export async function test_context(browser: Browser, options?: BrowserContextOptions) { @@ -128,6 +119,7 @@ export async function save_visual(page: Page) { // update order of recently created repos is not fully deterministic page.locator('.flex-item-main').filter({hasText: 'relative time in repo'}), page.locator('#activity-feed'), + page.locator('#user-heatmap'), // dynamic IDs in fixed-size inputs page.locator('input[value*="dyn-id-"]'), ], From 2d1e1639131cdf5a54a4c847cf5efa6e50c18095 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Fri, 17 Jan 2025 08:15:16 +0000 Subject: [PATCH 003/125] [v10.0/forgejo] fix: reduce noise for the v303 migration (#6594) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6591 Using SELECT `%s` FROM `%s` WHERE 0 = 1 to assert the existence of a column is simple but noisy: it shows errors in the migrations that are confusing for Forgejo admins because they are not actual errors. Use introspection instead, which is more complicated but leads to the same result. Add a test that ensures it works as expected, for all database types. Although the migration is run for all database types, it does not account for various scenarios and is never tested in the case a column does not exist. Refs: https://codeberg.org/forgejo/forgejo/issues/6583 Co-authored-by: Earl Warren Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6594 Reviewed-by: Earl Warren Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- models/migrations/v1_23/v303.go | 35 ++++++++++++++++++------ models/migrations/v1_23/v303_test.go | 41 ++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 models/migrations/v1_23/v303_test.go diff --git a/models/migrations/v1_23/v303.go b/models/migrations/v1_23/v303.go index e3ee180539..2fb37ac843 100644 --- a/models/migrations/v1_23/v303.go +++ b/models/migrations/v1_23/v303.go @@ -1,23 +1,27 @@ -// Copyright 2024 The Forgejo Authors. -// SPDX-License-Identifier: MIT +// Copyright 2025 The Forgejo Authors. +// SPDX-License-Identifier: GPL-3.0-or-later package v1_23 //nolint import ( - "fmt" - "code.gitea.io/gitea/models/migrations/base" "xorm.io/xorm" + "xorm.io/xorm/schemas" ) func GiteaLastDrop(x *xorm.Engine) error { + tables, err := x.DBMetas() + if err != nil { + return err + } + sess := x.NewSession() defer sess.Close() for _, drop := range []struct { - table string - field string + table string + column string }{ {"badge", "slug"}, {"oauth2_application", "skip_secondary_authorization"}, @@ -29,10 +33,25 @@ func GiteaLastDrop(x *xorm.Engine) error { {"protected_branch", "force_push_allowlist_team_i_ds"}, {"protected_branch", "force_push_allowlist_deploy_keys"}, } { - if _, err := sess.Exec(fmt.Sprintf("SELECT `%s` FROM `%s` WHERE 0 = 1", drop.field, drop.table)); err != nil { + var table *schemas.Table + found := false + + for _, table = range tables { + if table.Name == drop.table { + found = true + break + } + } + + if !found { continue } - if err := base.DropTableColumns(sess, drop.table, drop.field); err != nil { + + if table.GetColumn(drop.column) == nil { + continue + } + + if err := base.DropTableColumns(sess, drop.table, drop.column); err != nil { return err } } diff --git a/models/migrations/v1_23/v303_test.go b/models/migrations/v1_23/v303_test.go new file mode 100644 index 0000000000..752eacee0c --- /dev/null +++ b/models/migrations/v1_23/v303_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 The Forgejo Authors. +// SPDX-License-Identifier: GPL-3.0-or-later + +package v1_23 //nolint + +import ( + "testing" + + migration_tests "code.gitea.io/gitea/models/migrations/test" + + "github.com/stretchr/testify/require" + "xorm.io/xorm/schemas" +) + +func Test_GiteaLastDrop(t *testing.T) { + type Badge struct { + ID int64 `xorm:"pk autoincr"` + Slug string + } + + x, deferable := migration_tests.PrepareTestEnv(t, 0, new(Badge)) + defer deferable() + if x == nil || t.Failed() { + return + } + + getColumn := func() *schemas.Column { + tables, err := x.DBMetas() + require.NoError(t, err) + require.Len(t, tables, 1) + table := tables[0] + require.Equal(t, "badge", table.Name) + return table.GetColumn("slug") + } + + require.NotNil(t, getColumn(), "slug column exists") + require.NoError(t, GiteaLastDrop(x)) + require.Nil(t, getColumn(), "slug column was deleted") + // idempotent + require.NoError(t, GiteaLastDrop(x)) +} From 6d0bf55f0501e6c82671c8b746908b19f33462e5 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Fri, 17 Jan 2025 20:48:35 +0000 Subject: [PATCH 004/125] [v10.0/forgejo] fix: Reset content of comment edit field on cancel (#6601) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6595 Currently, the content of the text field is not reset when you cancel editing. This change resets the content of the text field when editing is canceled. If this is not done and you click on cancel and then on edit again, you can no longer return to the initial content without completely reloading the page. Co-authored-by: Beowulf Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6601 Reviewed-by: Gusted Reviewed-by: Beowulf Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- tests/e2e/issue-comment.test.e2e.ts | 21 +++++++++++++++++++++ web_src/js/features/repo-legacy.js | 1 + 2 files changed, 22 insertions(+) diff --git a/tests/e2e/issue-comment.test.e2e.ts b/tests/e2e/issue-comment.test.e2e.ts index 933e65fa32..1c19f98c48 100644 --- a/tests/e2e/issue-comment.test.e2e.ts +++ b/tests/e2e/issue-comment.test.e2e.ts @@ -77,6 +77,27 @@ test('Always focus edit tab first on edit', async ({page}) => { await save_visual(page); }); +test('Reset content of comment edit field on cancel', async ({page}) => { + const response = await page.goto('/user2/repo1/issues/1'); + expect(response?.status()).toBe(200); + + const editorTextarea = page.locator('[id="_combo_markdown_editor_1"]'); + + // Change the content of the edit field + await page.click('#issue-1 .comment-container .context-menu'); + await page.click('#issue-1 .comment-container .menu>.edit-content'); + await expect(editorTextarea).toHaveValue('content for the first issue'); + await editorTextarea.fill('some random string'); + await expect(editorTextarea).toHaveValue('some random string'); + await page.click('#issue-1 .comment-container .edit .cancel'); + + // Edit again and assert that the edit field should be reset to the initial content + await page.click('#issue-1 .comment-container .context-menu'); + await page.click('#issue-1 .comment-container .menu>.edit-content'); + await expect(editorTextarea).toHaveValue('content for the first issue'); + await save_visual(page); +}); + test('Quote reply', async ({page}, workerInfo) => { test.skip(workerInfo.project.name !== 'firefox', 'Uses Firefox specific selection quirks'); const response = await page.goto('/user2/repo1/issues/1'); diff --git a/web_src/js/features/repo-legacy.js b/web_src/js/features/repo-legacy.js index a4606aa3b5..7ce464c970 100644 --- a/web_src/js/features/repo-legacy.js +++ b/web_src/js/features/repo-legacy.js @@ -404,6 +404,7 @@ async function onEditContent(event) { e.preventDefault(); showElem(renderContent); hideElem(editContentZone); + comboMarkdownEditor.value(rawContent.textContent); comboMarkdownEditor.attachedDropzoneInst?.emit('reload'); }; From 28db11f2e79b8d03e27a00b460d860034b9af9b4 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Sat, 18 Jan 2025 19:43:08 +0000 Subject: [PATCH 005/125] [v10.0/forgejo] fix(ui): hide git note add button for commit if commit already has a note (#6614) Backport: https://codeberg.org/forgejo/forgejo/pulls/6613 Regression from f5c0570533b0a835a88eb7337da841d071f2de6b Co-authored-by: Beowulf Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6614 Reviewed-by: Beowulf Reviewed-by: 0ko <0ko@noreply.codeberg.org> Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- templates/repo/commit_page.tmpl | 8 +++++--- tests/e2e/git-notes.test.e2e.ts | 3 +++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/templates/repo/commit_page.tmpl b/templates/repo/commit_page.tmpl index 66be0c143d..36de789dd1 100644 --- a/templates/repo/commit_page.tmpl +++ b/templates/repo/commit_page.tmpl @@ -128,9 +128,11 @@ -
- {{ctx.Locale.Tr "repo.diff.git-notes.add"}} -
+ {{if not .NoteRendered}} +
+ {{ctx.Locale.Tr "repo.diff.git-notes.add"}} +
+ {{end}} {{end}} diff --git a/tests/e2e/git-notes.test.e2e.ts b/tests/e2e/git-notes.test.e2e.ts index 4245853b24..1e2cbe76fc 100644 --- a/tests/e2e/git-notes.test.e2e.ts +++ b/tests/e2e/git-notes.test.e2e.ts @@ -8,6 +8,9 @@ test('Change git note', async ({page}) => { let response = await page.goto('/user2/repo1/commit/65f1bf27bc3bf70f64657658635e66094edbcb4d'); expect(response?.status()).toBe(200); + // An add button should not be present, because the commit already has a commit note + await expect(page.locator('#commit-notes-add-button')).toHaveCount(0); + await page.locator('#commit-notes-edit-button').click(); let textarea = page.locator('textarea[name="notes"]'); From 627634a76ea6520dfdc393f775e10efd34fa91f0 Mon Sep 17 00:00:00 2001 From: Beowulf Date: Fri, 17 Jan 2025 18:22:43 +0100 Subject: [PATCH 006/125] Prevent prefix continuation if currently a text expander popup is open This fixes that mentions and emoji autocompletion was broken in e.g. a list, because the list handling take presidency over the text expansion. (cherry picked from commit 276ef10dd5a1167a8bcc20599197f89ff7f9b8a4) --- tests/e2e/markdown-editor.test.e2e.ts | 26 +++++++++++++++++++ .../js/features/comp/ComboMarkdownEditor.js | 2 ++ 2 files changed, 28 insertions(+) diff --git a/tests/e2e/markdown-editor.test.e2e.ts b/tests/e2e/markdown-editor.test.e2e.ts index 762113d563..1e30b8d3b9 100644 --- a/tests/e2e/markdown-editor.test.e2e.ts +++ b/tests/e2e/markdown-editor.test.e2e.ts @@ -224,3 +224,29 @@ test('markdown insert table', async ({page}) => { await expect(textarea).toHaveValue('| Header | Header |\n|---------|---------|\n| Content | Content |\n| Content | Content |\n| Content | Content |\n'); await save_visual(page); }); + +test('text expander has higher prio then prefix continuation', async ({page}) => { + const response = await page.goto('/user2/repo1/issues/new'); + expect(response?.status()).toBe(200); + + const textarea = page.locator('textarea[name=content]'); + const initText = `* first`; + await textarea.fill(initText); + await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.indexOf('rst'), it.value.indexOf('rst'))); + await textarea.press('End'); + + // Test emoji completion + await textarea.press('Enter'); + await textarea.pressSequentially(':smile_c'); + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`* first\n* 😸`); + + // Test username completion + await textarea.press('Enter'); + await textarea.pressSequentially('@user'); + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`* first\n* 😸\n* @user2 `); + + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`* first\n* 😸\n* @user2 \n* `); +}); diff --git a/web_src/js/features/comp/ComboMarkdownEditor.js b/web_src/js/features/comp/ComboMarkdownEditor.js index 8ae5defa47..707101190c 100644 --- a/web_src/js/features/comp/ComboMarkdownEditor.js +++ b/web_src/js/features/comp/ComboMarkdownEditor.js @@ -99,6 +99,8 @@ class ComboMarkdownEditor { e.target._shiftDown = true; } if (e.key === 'Enter' && !e.shiftKey && !e.ctrlKey && !e.altKey) { + // Prevent special line break handling if currently a text expander popup is open + if (this.textarea.hasAttribute('aria-expanded')) return; if (!this.breakLine()) return; // Nothing changed, let the default handler work. this.options?.onContentChanged?.(this, e); e.preventDefault(); From 348e0e1face01e952b59bdefc28ac48fdcf07d19 Mon Sep 17 00:00:00 2001 From: Beowulf Date: Fri, 17 Jan 2025 18:42:42 +0100 Subject: [PATCH 007/125] Leave list/quote expanison with double enter When editing a list or similar syntax elements, pressing enter starts a new line with the line introducer (e.g. `- ` for a plain list). But currently it's uncomfortable when someone wants to leave the list. Pressing enter again simply adds more and more lines with the prefix. With this change the list is terminated if enter is pressed on a line which contains the introducer but nothing else. This behavior is known from other markdown editors like the on used by GitLab or GitHub. Additionally I changed the regex for detecting a prefix. - Why: With the change you can add a single whitespace at the end if you want to keep an "empty" line. So if you want to write: ``` - First - - Third ``` You just need to add a whitespace in the second line to prevent that the prefix will be removed. - Changes in detail: - ordered bullet list prefix detection: nothing changed - todo list and unordered list prefix detection: have been split up: - todo list: Changed that only 1 to 4 whitespaces can be between the list char (`-`,`*`,`+`) and the checkbox (`[ ]`,`[x]`) - Why? If more then 4 spaces are between the list char and the checkbox, this is no longer detected as a prefix for a todo item based on the markdown standard. Due to the amount of spaces it is instead parsed as code. - unordered list: The prefix now needs to have exactly one space after the list char (`-`,`*`,`+`). More spaces will not be taken into account for detecting the prefix. - quote prefix detection: nothing changed The current e2e-tests where simplified and duplicated tests where removed. Test cases for the new functionality where added. (cherry picked from commit 7ea62c5ce475db81f2930a569c98190a52e6cae6) --- tests/e2e/markdown-editor.test.e2e.ts | 51 +++++++++---------- .../js/features/comp/ComboMarkdownEditor.js | 22 ++++++-- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/tests/e2e/markdown-editor.test.e2e.ts b/tests/e2e/markdown-editor.test.e2e.ts index 1e30b8d3b9..35e9de2ea6 100644 --- a/tests/e2e/markdown-editor.test.e2e.ts +++ b/tests/e2e/markdown-editor.test.e2e.ts @@ -109,7 +109,7 @@ test('markdown indentation', async ({page}) => { }); test('markdown list continuation', async ({page}) => { - const initText = `* first\n* second\n* third\n* last`; + const initText = `* first\n* second`; const response = await page.goto('/user2/repo1/issues/new'); expect(response?.status()).toBe(200); @@ -119,25 +119,20 @@ test('markdown list continuation', async ({page}) => { const indent = page.locator('button[data-md-action="indent"]'); await textarea.fill(initText); - // Test continuation of '* ' prefix - await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.indexOf('cond'), it.value.indexOf('cond'))); + // Test continuation of ' * ' prefix + await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.indexOf('rst'), it.value.indexOf('rst'))); + await indent.click(); await textarea.press('End'); await textarea.press('Enter'); - await textarea.pressSequentially('middle'); - await expect(textarea).toHaveValue(`* first\n* second\n* middle\n* third\n* last`); - - // Test continuation of ' * ' prefix - await indent.click(); - await textarea.press('Enter'); await textarea.pressSequentially('muddle'); - await expect(textarea).toHaveValue(`* first\n* second\n${tab}* middle\n${tab}* muddle\n* third\n* last`); + await expect(textarea).toHaveValue(`${tab}* first\n${tab}* muddle\n* second`); // Test breaking in the middle of a line await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.lastIndexOf('ddle'), it.value.lastIndexOf('ddle'))); await textarea.pressSequentially('tate'); await textarea.press('Enter'); await textarea.pressSequentially('me'); - await expect(textarea).toHaveValue(`* first\n* second\n${tab}* middle\n${tab}* mutate\n${tab}* meddle\n* third\n* last`); + await expect(textarea).toHaveValue(`${tab}* first\n${tab}* mutate\n${tab}* meddle\n* second`); // Test not triggering when Shift held await textarea.fill(initText); @@ -145,35 +140,36 @@ test('markdown list continuation', async ({page}) => { await textarea.press('Shift+Enter'); await textarea.press('Enter'); await textarea.pressSequentially('...but not least'); - await expect(textarea).toHaveValue(`* first\n* second\n* third\n* last\n\n...but not least`); + await expect(textarea).toHaveValue(`* first\n* second\n\n...but not least`); // Test continuation of ordered list - await textarea.fill(`1. one\n2. two`); + await textarea.fill(`1. one`); await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.length, it.value.length)); await textarea.press('Enter'); + await textarea.pressSequentially(' '); + await textarea.press('Enter'); await textarea.pressSequentially('three'); - await expect(textarea).toHaveValue(`1. one\n2. two\n3. three`); + await textarea.press('Enter'); + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`1. one\n2. \n3. three\n\n`); // Test continuation of alternative ordered list syntax - await textarea.fill(`1) one\n2) two`); + await textarea.fill(`1) one`); await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.length, it.value.length)); await textarea.press('Enter'); + await textarea.pressSequentially(' '); + await textarea.press('Enter'); await textarea.pressSequentially('three'); - await expect(textarea).toHaveValue(`1) one\n2) two\n3) three`); - - // Test continuation of blockquote - await textarea.fill(`> knowledge is power`); - await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.length, it.value.length)); await textarea.press('Enter'); - await textarea.pressSequentially('france is bacon'); - await expect(textarea).toHaveValue(`> knowledge is power\n> france is bacon`); + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`1) one\n2) \n3) three\n\n`); // Test continuation of checklists - await textarea.fill(`- [ ] have a problem\n- [x] create a solution`); + await textarea.fill(`- [ ]have a problem\n- [x]create a solution`); await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.length, it.value.length)); await textarea.press('Enter'); await textarea.pressSequentially('write a test'); - await expect(textarea).toHaveValue(`- [ ] have a problem\n- [x] create a solution\n- [ ] write a test`); + await expect(textarea).toHaveValue(`- [ ]have a problem\n- [x]create a solution\n- [ ]write a test`); // Test all conceivable syntax (except ordered lists) const prefixes = [ @@ -189,7 +185,6 @@ test('markdown list continuation', async ({page}) => { '> ', '> > ', '- [ ] ', - '- [ ]', // This does seem to render, so allow. '* [ ] ', '+ [ ] ', ]; @@ -197,8 +192,12 @@ test('markdown list continuation', async ({page}) => { await textarea.fill(`${prefix}one`); await textarea.evaluate((it:HTMLTextAreaElement) => it.setSelectionRange(it.value.length, it.value.length)); await textarea.press('Enter'); + await textarea.pressSequentially(' '); + await textarea.press('Enter'); await textarea.pressSequentially('two'); - await expect(textarea).toHaveValue(`${prefix}one\n${prefix}two`); + await textarea.press('Enter'); + await textarea.press('Enter'); + await expect(textarea).toHaveValue(`${prefix}one\n${prefix} \n${prefix}two\n\n`); } }); diff --git a/web_src/js/features/comp/ComboMarkdownEditor.js b/web_src/js/features/comp/ComboMarkdownEditor.js index 707101190c..89a252f6f3 100644 --- a/web_src/js/features/comp/ComboMarkdownEditor.js +++ b/web_src/js/features/comp/ComboMarkdownEditor.js @@ -409,13 +409,27 @@ class ComboMarkdownEditor { // Find the beginning of the current line. const lineStart = Math.max(0, value.lastIndexOf('\n', start - 1) + 1); // Find the end and extract the line. - const lineEnd = value.indexOf('\n', start); - const line = value.slice(lineStart, lineEnd === -1 ? value.length : lineEnd); + const nextLF = value.indexOf('\n', start); + const lineEnd = nextLF === -1 ? value.length : nextLF; + const line = value.slice(lineStart, lineEnd); // Match any whitespace at the start + any repeatable prefix + exactly one space after. - const prefix = line.match(/^\s*((\d+)[.)]\s|[-*+]\s+(\[[ x]\]\s?)?|(>\s+)+)?/); + const prefix = line.match(/^\s*((\d+)[.)]\s|[-*+]\s{1,4}\[[ x]\]\s?|[-*+]\s|(>\s?)+)?/); // Defer to browser if we can't do anything more useful, or if the cursor is inside the prefix. - if (!prefix || !prefix[0].length || lineStart + prefix[0].length > start) return false; + if (!prefix) return false; + const prefixLength = prefix[0].length; + if (!prefixLength || lineStart + prefixLength > start) return false; + // If the prefix is just indentation (which should always be an even number of spaces or tabs), check if a single whitespace is added to the end of the line. + // If this is the case do not leave the indentation and continue with the prefix. + if ((prefixLength % 2 === 1 && /^ +$/.test(prefix[0])) || /^\t+ $/.test(prefix[0])) { + prefix[0] = prefix[0].slice(0, prefixLength - 1); + } else if (prefixLength === lineEnd - lineStart) { + this.textarea.setSelectionRange(lineStart, lineEnd); + if (!document.execCommand('insertText', false, '\n')) { + this.textarea.setRangeText('\n'); + } + return true; + } // Insert newline + prefix. let text = `\n${prefix[0]}`; From 6020be2c8a88e27c052fc4dfbce37d985bd34416 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 22 Nov 2024 16:38:23 +0100 Subject: [PATCH 008/125] Run testing workflow unconditionally --- .forgejo/workflows/testing.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index eb3163d3ae..3d8ad443da 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -10,7 +10,6 @@ on: jobs: backend-checks: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker container: image: 'data.forgejo.org/oci/node:20-bookworm' @@ -27,7 +26,6 @@ jobs: - run: su forgejo -c 'make --always-make -j$(nproc) lint-backend tidy-check swagger-check fmt-check swagger-validate' # ensure the "go-licenses" make target runs - uses: ./.forgejo/workflows-composite/build-backend frontend-checks: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker container: image: 'data.forgejo.org/oci/node:20-bookworm' @@ -176,7 +174,6 @@ jobs: TAGS: bindata TEST_REDIS_SERVER: cacher:${{ matrix.cacher.port }} test-mysql: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -207,7 +204,6 @@ jobs: env: USE_REPO_TEST_DIR: 1 test-pgsql: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -246,7 +242,6 @@ jobs: USE_REPO_TEST_DIR: 1 TEST_LDAP: 1 test-sqlite: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: [backend-checks, frontend-checks] container: @@ -269,14 +264,11 @@ jobs: TEST_TAGS: sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 security-check: - if: vars.ROLE == 'forgejo-coding' || vars.ROLE == 'forgejo-testing' runs-on: docker needs: - test-sqlite - test-pgsql - test-mysql - - test-remote-cacher - - test-unit container: image: 'data.forgejo.org/oci/node:20-bookworm' options: --tmpfs /tmp:exec,noatime From b3f62a9af2248ad1e02eab9db68374cc90171ece Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 12:33:18 +0200 Subject: [PATCH 009/125] Fix name in package-lock.json The frontend-checks job started failing because the declared name does not match the repository name. --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index e081796a52..c5f1950e59 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "forgejo", + "name": "forgejo-aneksajo", "lockfileVersion": 3, "requires": true, "packages": { From d1788bd26743cd613d8cb597913a8bb5847937d3 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sat, 30 Jul 2022 21:45:03 -0400 Subject: [PATCH 010/125] git-annex support [git-annex](https://git-annex.branchable.com/) is a more complicated cousin to git-lfs, storing large files in an optional-download side content. Unlike lfs, it allows mixing and matching storage remotes, so the content remote(s) doesn't need to be on the same server as the git remote, making it feasible to scatter a collection across cloud storage, old harddrives, or anywhere else storage can be scavenged. Since this can get complicated, fast, it has a content-tracking database (`git annex whereis`) to help find everything later. The use-case we imagine for including it in Gitea is just the simple case, where we're primarily emulating git-lfs: each repo has its large content at the same URL. Our motivation is so we can self-host https://www.datalad.org/ datasets, which currently are only hostable by fragilely scrounging together cloud storage -- and having to manage all the credentials associated with all the pieces -- or at https://openneuro.org which is fragile in its own ways. Supporting git-annex also allows multiple Gitea instance to be annex remotes for each other, mirroring the content or otherwise collaborating the split up the hosting costs. Enabling -------- TODO HTTP ---- TODO Permission Checking ------------------- This tweaks the API in routers/private/serv.go to expose the calling user's computed permission, instead of just returning HTTP 403. This doesn't fit in super well. It's the opposite from how the git-lfs support is done, where there's a complete list of possible subcommands and their matching permission levels, and then the API compares the requested with the actual level and returns HTTP 403 if the check fails. But it's necessary. The main git-annex verbs, 'git-annex-shell configlist' and 'git-annex-shell p2pstdio' are both either read-only or read-write operations, depending on the state on disk on either end of the connection and what the user asked it to ask for, with no way to know before git-annex examines the situation. So tell the level via GIT_ANNEX_READONLY and trust it to handle itself. In the older Gogs version, the permission was directly read in cmd/serv.go: ``` mode, err = db.UserAccessMode(user.ID, repo) ``` - https://github.com/G-Node/gogs/blob/966e925cf320beff768b192276774d9265706df5/internal/cmd/serv.go#L334 but in Gitea permission enforcement has been centralized in the API layer. (perhaps so the cmd layer can avoid making direct DB connections?) Deletion -------- git-annex has this "lockdown" feature where it tries really quite very hard to prevent you deleting its data, to the point that even an rm -rf won't do it: each file in annex/objects/ is nested inside a folder with read-only permissions. The recommended workaround is to run chmod -R +w when you're sure you actually want to delete a repo. See https://git-annex.branchable.com/internals/lockdown So we edit util.RemoveAll() to do just that, so now it's `chmod -R +w && rm -rf` instead of just `rm -rf`. --- cmd/serv.go | 72 ++++++++++++++++++++++++++++++++++++++--- modules/private/serv.go | 1 + modules/util/remove.go | 34 ++++++++++++++++++- routers/private/serv.go | 12 +++++-- 4 files changed, 110 insertions(+), 9 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index db67e36fa3..7d16547053 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -38,6 +38,7 @@ import ( const ( lfsAuthenticateVerb = "git-lfs-authenticate" + gitAnnexShellVerb = "git-annex-shell" ) // CmdServ represents the available serv sub-command. @@ -79,6 +80,7 @@ var ( "git-upload-archive": perm.AccessModeRead, "git-receive-pack": perm.AccessModeWrite, lfsAuthenticateVerb: perm.AccessModeNone, + gitAnnexShellVerb: perm.AccessModeNone, // annex permissions are enforced by GIT_ANNEX_SHELL_READONLY, rather than the Gitea API } alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -212,6 +214,29 @@ func runServ(c *cli.Context) error { } } + if verb == gitAnnexShellVerb { + // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 + if false { + return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") + } + + if len(words) < 3 { + return fail(ctx, "Too few arguments", "Too few arguments in cmd: %s", cmd) + } + + // git-annex always puts the repo in words[2], unlike most other + // git subcommands; and it sometimes names repos like /~/, as if + // $HOME should get expanded while also being rooted. e.g.: + // git-annex-shell 'configlist' '/~/user/repo' + // git-annex-shell 'sendkey' '/user/repo 'key' + repoPath = words[2] + repoPath = strings.TrimPrefix(repoPath, "/") + repoPath = strings.TrimPrefix(repoPath, "~/") + } + + // prevent directory traversal attacks + repoPath = filepath.Clean("/" + repoPath)[1:] + rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { return fail(ctx, "Invalid repository path", "Invalid repository path: %v", repoPath) @@ -225,6 +250,18 @@ func runServ(c *cli.Context) error { // so that username and reponame are not affected. repoPath = strings.ToLower(strings.TrimSpace(repoPath)) + // put the sanitized repoPath back into the argument list for later + if verb == gitAnnexShellVerb { + // git-annex-shell demands an absolute path + absRepoPath, err := filepath.Abs(filepath.Join(setting.RepoRootPath, repoPath)) + if err != nil { + return fail(ctx, "Error locating repoPath", "%v", err) + } + words[2] = absRepoPath + } else { + words[1] = repoPath + } + if alphaDashDotPattern.MatchString(reponame) { return fail(ctx, "Invalid repo name", "Invalid repo name: %s", reponame) } @@ -303,21 +340,46 @@ func runServ(c *cli.Context) error { return nil } - var gitcmd *exec.Cmd gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack if _, err := os.Stat(gitBinVerb); err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way + // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, + // but '{gitBinPath}/git annex-shell' should be able to find them on $PATH. verbFields := strings.SplitN(verb, "-", 2) if len(verbFields) == 2 { // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ... - gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath) + gitBinVerb = git.GitExecutable + words = append([]string{verbFields[1]}, words...) } } - if gitcmd == nil { - // by default, use the verb (it has been checked above by allowedCommands) - gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath) + + // by default, use the verb (it has been checked above by allowedCommands) + gitcmd := exec.CommandContext(ctx, gitBinVerb, words[1:]...) + + if verb == gitAnnexShellVerb { + // This doesn't get its own isolated section like LFS does, because LFS + // is handled by internal Gitea routines, but git-annex has to be shelled out + // to like other git subcommands, so we need to build up gitcmd. + + // TODO: does this work on Windows? + gitcmd.Env = append(gitcmd.Env, + // "If set, disallows running git-shell to handle unknown commands." + // - git-annex-shell(1) + "GIT_ANNEX_SHELL_LIMITED=True", + // "If set, git-annex-shell will refuse to run commands + // that do not operate on the specified directory." + // - git-annex-shell(1) + fmt.Sprintf("GIT_ANNEX_SHELL_DIRECTORY=%s", words[2]), + ) + if results.UserMode < perm.AccessModeWrite { + // "If set, disallows any action that could modify the git-annex repository." + // - git-annex-shell(1) + // We set this when the backend API has told us that we don't have write permission to this repo. + log.Debug("Setting GIT_ANNEX_SHELL_READONLY=True") + gitcmd.Env = append(gitcmd.Env, "GIT_ANNEX_SHELL_READONLY=True") + } } process.SetSysProcAttribute(gitcmd) diff --git a/modules/private/serv.go b/modules/private/serv.go index 480a446954..6c7c753cf0 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -40,6 +40,7 @@ type ServCommandResults struct { UserName string UserEmail string UserID int64 + UserMode perm.AccessMode OwnerName string RepoName string RepoID int64 diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f..0f41471fcc 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -4,7 +4,9 @@ package util import ( + "io/fs" "os" + "path/filepath" "runtime" "syscall" "time" @@ -41,10 +43,40 @@ func Remove(name string) error { return err } -// RemoveAll removes the named file or (empty) directory with at most 5 attempts. +// RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error + for i := 0; i < 5; i++ { + // Do chmod -R +w to help ensure the removal succeeds. + // In particular, in the git-annex case, this handles + // https://git-annex.branchable.com/internals/lockdown/ : + // + // > (The only bad consequence of this is that rm -rf .git + // > doesn't work unless you first run chmod -R +w .git) + + err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + // try again + <-time.After(100 * time.Millisecond) + continue + } + err = os.RemoveAll(name) if err == nil { break diff --git a/routers/private/serv.go b/routers/private/serv.go index ef3920d359..b42fbc04f4 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -81,12 +81,14 @@ func ServCommand(ctx *context.PrivateContext) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") mode := perm.AccessMode(ctx.FormInt("mode")) + verbs := ctx.FormStrings("verb") // Set the basic parts of the results to return results := private.ServCommandResults{ RepoName: repoName, OwnerName: ownerName, KeyID: keyID, + UserMode: perm.AccessModeNone, } // Now because we're not translating things properly let's just default some English strings here @@ -287,8 +289,10 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey + ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { + results.UserMode = deployKey.Mode if deployKey.Mode < mode { ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), @@ -310,9 +314,9 @@ func ServCommand(ctx *context.PrivateContext) { return } - userMode := perm.UnitAccessMode(unitType) + results.UserMode = perm.UnitAccessMode(unitType) - if userMode < mode { + if results.UserMode < mode { log.Warn("Failed authentication attempt for %s with key %s (not authorized to %s %s/%s) from %s", user.Name, key.Name, modeString, ownerName, repoName, ctx.RemoteAddr()) ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("User: %d:%s with Key: %d:%s is not authorized to %s %s/%s.", user.ID, user.Name, key.ID, key.Name, modeString, ownerName, repoName), @@ -353,6 +357,7 @@ func ServCommand(ctx *context.PrivateContext) { }) return } + results.UserMode = perm.AccessModeWrite results.RepoID = repo.ID } @@ -381,13 +386,14 @@ func ServCommand(ctx *context.PrivateContext) { return } } - log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", + log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nUserMode: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", results.IsWiki, results.DeployKeyID, results.KeyID, results.KeyName, results.UserName, results.UserID, + results.UserMode, results.OwnerName, results.RepoName, results.RepoID) From 054537989fb738c561d8e37494654a8f81455299 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Mon, 20 Jan 2025 20:28:39 +0000 Subject: [PATCH 011/125] [v10.0/forgejo] fix(ui): prevent overflow of branch selector in commit graph (#6636) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6617 Fix that the branch selector in the commit graph can overflow | Previous | Now | | :----: | :----: | | ![grafik](/attachments/ab303490-2abc-46d8-8715-0750886fd111) | ![grafik](/attachments/63f919a9-bcc2-4969-8c8c-d265c1917e07) | | ![grafik](/attachments/c0e6636f-52eb-4bf0-bf07-0139ec407e33) | ![grafik](/attachments/752aef87-9250-4bf6-b74a-5a1813394dbe) | | ![grafik](/attachments/e61842dd-29c1-4517-86db-f068de9ff6e8) | ![grafik](/attachments/bf251b43-80fa-4e1a-9fbe-fd27e5f8d195) | Fixes #6615 Co-authored-by: Beowulf Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6636 Reviewed-by: Otto Reviewed-by: Beowulf Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- tests/e2e/repo-commitgraph.test.e2e.ts | 19 ++++++++++++++++++- web_src/css/features/gitgraph.css | 19 +++++++++++++------ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tests/e2e/repo-commitgraph.test.e2e.ts b/tests/e2e/repo-commitgraph.test.e2e.ts index 39c5661900..e8b85c5997 100644 --- a/tests/e2e/repo-commitgraph.test.e2e.ts +++ b/tests/e2e/repo-commitgraph.test.e2e.ts @@ -8,10 +8,27 @@ import {expect} from '@playwright/test'; import {save_visual, test} from './utils_e2e.ts'; test('Commit graph overflow', async ({page}) => { - await page.goto('/user2/diff-test/graph'); + const response = await page.goto('/user2/repo1/graph'); + expect(response?.status()).toBe(200); + + await page.click('#flow-select-refs-dropdown'); + const input = page.locator('#flow-select-refs-dropdown'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + await input.press('Enter'); + + await expect(page.locator('#flow-select-refs-dropdown')).toBeInViewport({ratio: 1}); await expect(page.getByRole('button', {name: 'Mono'})).toBeInViewport({ratio: 1}); await expect(page.getByRole('button', {name: 'Color'})).toBeInViewport({ratio: 1}); await expect(page.locator('.selection.search.dropdown')).toBeInViewport({ratio: 1}); + await save_visual(page); }); test('Switch branch', async ({page}) => { diff --git a/web_src/css/features/gitgraph.css b/web_src/css/features/gitgraph.css index 4da871da61..726ac7e9e2 100644 --- a/web_src/css/features/gitgraph.css +++ b/web_src/css/features/gitgraph.css @@ -23,6 +23,18 @@ #git-graph-heading { align-items: center; } + + #git-graph-heading-left { + margin-right: 1rem; + } + + #git-graph-heading h2 { + flex-shrink: 0; + } + + #git-graph-container #flow-select-refs-dropdown { + min-width: 250px; + } } @media (max-width: 767.98px) { @@ -34,15 +46,10 @@ #git-graph-heading-left { margin-bottom: 1rem; } - - h2, - #flow-select-refs-dropdown { - max-width: 100%; - } } #git-graph-container #flow-select-refs-dropdown { - min-width: 250px; + flex-wrap: wrap; } #git-graph-container #flow-select-refs-dropdown .ui.label { From 7546c4acf35c727e1f4c58e5c602ca1ca79c3afa Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 21 Jan 2025 09:32:40 +0000 Subject: [PATCH 012/125] Update dependency go to v1.23.5 (v10.0/forgejo) (#6644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [go](https://go.dev/) ([source](https://github.com/golang/go)) | toolchain | patch | `1.23.4` -> `1.23.5` | --- ### Configuration 📅 **Schedule**: Branch creation - At any time (no schedule defined), Automerge - "* 0-3 * * *" (UTC). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate). Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6644 Reviewed-by: Earl Warren Co-authored-by: Renovate Bot Co-committed-by: Renovate Bot --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index d5f48716ff..19bec3f81f 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module code.gitea.io/gitea go 1.23 -toolchain go1.23.4 +toolchain go1.23.5 require ( code.forgejo.org/f3/gof3/v3 v3.10.2 From 5c5e1c87ba4ac6d36fe37c1975056530caa58481 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Tue, 21 Jan 2025 10:40:00 +0000 Subject: [PATCH 013/125] [v10.0/forgejo] fix: listing tokens must not require basic auth (#6643) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6633 When the change is reverted, the test fails as follows: ```sh === TestAPIGetTokens (tests/integration/api_token_test.go:34) --- FAIL: TestAPIGetTokens (0.17s) testlogger.go:405: 2025/01/20 14:05:22 ...les/storage/local.go:33:NewLocalStorage() [I] Creating new Local Storage at /home/earl-warren/software/forgejo/tests/gitea-lfs-meta testlogger.go:405: 2025/01/20 14:05:22 ...eb/routing/logger.go:102:func1() [I] router: completed GET /api/v1/users/user2/tokens for test-mock:12345, 200 OK in 2.5ms @ user/app.go:24(user.ListAccessTokens) testlogger.go:405: 2025/01/20 14:05:22 ...eb/routing/logger.go:102:func1() [I] router: completed POST /api/v1/users/user1/tokens for test-mock:12345, 201 Created in 4.7ms @ user/app.go:75(user.CreateAccessToken) testlogger.go:405: 2025/01/20 14:05:22 ...eb/routing/logger.go:102:func1() [I] router: completed GET /api/v1/users/user2/tokens for test-mock:12345, 401 Unauthorized in 4.9ms @ v1/api.go:413(v1.Routes.func2.5.1.reqBasicOrRevProxyAuth.6) api_token_test.go:46: Error Trace: /home/earl-warren/software/forgejo/tests/integration/integration_test.go:556 /home/earl-warren/software/forgejo/tests/integration/api_token_test.go:46 Error: Not equal: expected: 200 actual : 401 Test: TestAPIGetTokens Messages: Request: GET /api/v1/users/user2/tokens api_token_test.go:46: Response: {"message":"auth required","url":"http://localhost:3003/api/swagger"} testlogger.go:405: 2025/01/20 14:05:22 ...eb/routing/logger.go:102:func1() [I] router: completed DELETE /api/v1/users/user1/tokens/94 for test-mock:12345, 204 No Content in 1.4ms @ user/app.go:145(user.DeleteAccessToken) ``` ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [x] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. ## Release notes - Bug fixes - [PR](https://codeberg.org/forgejo/forgejo/pulls/6633): listing tokens must not require basic auth Co-authored-by: Earl Warren Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6643 Reviewed-by: Michael Kriese Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- routers/api/v1/api.go | 6 +++--- tests/integration/api_token_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go index 4928c9ff58..18ab6ce287 100644 --- a/routers/api/v1/api.go +++ b/routers/api/v1/api.go @@ -907,9 +907,9 @@ func Routes() *web.Route { m.Get("/repos", tokenRequiresScopes(auth_model.AccessTokenScopeCategoryRepository), reqExploreSignIn(), user.ListUserRepos) m.Group("/tokens", func() { m.Combo("").Get(user.ListAccessTokens). - Post(bind(api.CreateAccessTokenOption{}), reqToken(), user.CreateAccessToken) - m.Combo("/{id}").Delete(reqToken(), user.DeleteAccessToken) - }, reqSelfOrAdmin(), reqBasicOrRevProxyAuth()) + Post(bind(api.CreateAccessTokenOption{}), reqBasicOrRevProxyAuth(), reqToken(), user.CreateAccessToken) + m.Combo("/{id}").Delete(reqBasicOrRevProxyAuth(), reqToken(), user.DeleteAccessToken) + }, reqSelfOrAdmin()) m.Get("/activities/feeds", user.ListUserActivityFeeds) }, context.UserAssignmentAPI(), checkTokenPublicOnly(), individualPermsChecker) diff --git a/tests/integration/api_token_test.go b/tests/integration/api_token_test.go index 01d18ef6f1..f94a0986f2 100644 --- a/tests/integration/api_token_test.go +++ b/tests/integration/api_token_test.go @@ -30,6 +30,23 @@ func TestAPICreateAndDeleteToken(t *testing.T) { deleteAPIAccessToken(t, newAccessToken, user) } +func TestAPIGetTokens(t *testing.T) { + defer tests.PrepareTestEnv(t)() + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) + + // with basic auth... + req := NewRequest(t, "GET", "/api/v1/users/user2/tokens"). + AddBasicAuth(user.Name) + MakeRequest(t, req, http.StatusOK) + + // ... or with a token. + newAccessToken := createAPIAccessTokenWithoutCleanUp(t, "test-key-1", user, []auth_model.AccessTokenScope{auth_model.AccessTokenScopeAll}) + req = NewRequest(t, "GET", "/api/v1/users/user2/tokens"). + AddTokenAuth(newAccessToken.Token) + MakeRequest(t, req, http.StatusOK) + deleteAPIAccessToken(t, newAccessToken, user) +} + // TestAPIDeleteMissingToken ensures that error is thrown when token not found func TestAPIDeleteMissingToken(t *testing.T) { defer tests.PrepareTestEnv(t)() From 61e345cd362db4968a2f5b5043e34387207749aa Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Wed, 22 Jan 2025 07:47:34 +0000 Subject: [PATCH 014/125] [v10.0/forgejo] fix: teach the doctor about orphaned two_factor rows (#6651) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6639 If a row in the two_factor table references a non existent user, it may contain a secret that has an invalid format. Such an orphaned row is never used and should be removed. Improve the error message to suggest using the doctor to remove it. Fixes: https://codeberg.org/forgejo/forgejo/issues/6637 ## Testing - make TAGS='sqlite sqlite_unlock_notify' watch - make TAGS='sqlite sqlite_unlock_notify' forgejo - sqlite3 data/gitea.db 'INSERT INTO two_factor VALUES( 0, 500, "", "", "", "", 0, 0)' - ./forgejo doctor check --run check-db-consistency ``` [1] Check consistency of database - [W] Found 1 Orphaned TwoFactor without existing User OK All done (checks: 1). ``` - ./forgejo doctor check --run check-db-consistency --fix ``` [1] Check consistency of database - [I] Deleted 1 Orphaned TwoFactor without existing User OK All done (checks: 1). ``` ## Checklist The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org). ### Tests - I added test coverage for Go changes... - [ ] in their respective `*_test.go` for unit tests. - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [x] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. ## Release notes - Bug fixes - [PR](https://codeberg.org/forgejo/forgejo/pulls/6651): fix: teach the doctor about orphaned two_factor rows Co-authored-by: Earl Warren Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6651 Reviewed-by: Earl Warren Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- modules/secret/secret.go | 2 +- release-notes/6639.md | 1 + services/doctor/dbconsistency.go | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 release-notes/6639.md diff --git a/modules/secret/secret.go b/modules/secret/secret.go index e70ae1839c..e3557b91b9 100644 --- a/modules/secret/secret.go +++ b/modules/secret/secret.go @@ -47,7 +47,7 @@ func AesDecrypt(key, text []byte) ([]byte, error) { cfb.XORKeyStream(text, text) data, err := base64.StdEncoding.DecodeString(string(text)) if err != nil { - return nil, fmt.Errorf("AesDecrypt invalid decrypted base64 string: %w", err) + return nil, fmt.Errorf("AesDecrypt invalid decrypted base64 string: %w - it can be caused by a change of the [security].SECRET_KEY setting or a database corruption - `forgejo doctor check --run check-db-consistency --fix` will get rid of orphaned rows found in the `two_factor` table and may fix this problem if they are the one with the invalid content", err) } return data, nil } diff --git a/release-notes/6639.md b/release-notes/6639.md new file mode 100644 index 0000000000..1bc01c12a3 --- /dev/null +++ b/release-notes/6639.md @@ -0,0 +1 @@ +Teach the doctor to remove orphaned two_factor with `forgejo doctor check --run check-db-consistency --fix`. Such rows may contain invalid data and [block the migration to v10](https://codeberg.org/forgejo/forgejo/issues/6637) with a message such as `failed: AesDecrypt invalid decrypted base64 string: illegal base64 data at input byte 0`. diff --git a/services/doctor/dbconsistency.go b/services/doctor/dbconsistency.go index 80f538d670..9e2fcb645f 100644 --- a/services/doctor/dbconsistency.go +++ b/services/doctor/dbconsistency.go @@ -246,6 +246,9 @@ func checkDBConsistency(ctx context.Context, logger log.Logger, autofix bool) er // find authorization tokens without existing user genericOrphanCheck("Authorization token without existing User", "forgejo_auth_token", "user", "forgejo_auth_token.uid=`user`.id"), + // find two_factor without existing user + genericOrphanCheck("Orphaned TwoFactor without existing User", + "two_factor", "user", "`two_factor`.uid=`user`.id"), ) for _, c := range consistencyChecks { From eb83b054302b1cfc827f801503764d3c2c8210da Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Wed, 22 Jan 2025 14:52:04 +0100 Subject: [PATCH 015/125] chore(security): update security.txt with new expiration date Same as https://forgejo.org/.well-known/security.txt (cherry picked from commit 955f99b6a4f017ad8ff3c311ba234a48065dee51) --- public/.well-known/security.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/.well-known/security.txt b/public/.well-known/security.txt index 2a75a8dcd2..0ac9f09d34 100644 --- a/public/.well-known/security.txt +++ b/public/.well-known/security.txt @@ -5,4 +5,4 @@ Policy: https://codeberg.org/forgejo/governance/src/commit/5c07b3801537212ed6be1 Contact: mailto:security@forgejo.org Encryption: https://keys.openpgp.org/vks/v1/by-fingerprint/1B638BDF10969D627926B8D9F585D0F99E1FB56F Preferred-Languages: en -Expires: 2025-10-25T00:00:00Z +Expires: 2026-07-16T23:59:59.000Z From 553fc3cc42b3d50c1a69c74f3db61b801337cdcf Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Fri, 24 Jan 2025 12:25:00 +0000 Subject: [PATCH 016/125] [v10.0/forgejo] fix: load settings for valid user and email check (#6678) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6674 - The doctor commands to check the validity of existing usernames and email addresses depend on functionality that have configurable behavior depending on the values of the `[service]` settings, so load them when running the doctor command. - Resolves #6664 - No unit test due to the architecture of doctor commands. Co-authored-by: Gusted Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6678 Reviewed-by: Gusted Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- modules/setting/service.go | 5 +++++ services/doctor/breaking.go | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/modules/setting/service.go b/modules/setting/service.go index 5a6cc254e0..74ed5cd3c9 100644 --- a/modules/setting/service.go +++ b/modules/setting/service.go @@ -138,6 +138,11 @@ func CompileEmailGlobList(sec ConfigSection, keys ...string) (globs []glob.Glob) return globs } +// LoadServiceSetting loads the service settings +func LoadServiceSetting() { + loadServiceFrom(CfgProvider) +} + func loadServiceFrom(rootCfg ConfigProvider) { sec := rootCfg.Section("service") Service.ActiveCodeLives = sec.Key("ACTIVE_CODE_LIVE_MINUTES").MustInt(180) diff --git a/services/doctor/breaking.go b/services/doctor/breaking.go index 683ec97389..ec8433b8de 100644 --- a/services/doctor/breaking.go +++ b/services/doctor/breaking.go @@ -10,6 +10,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/validation" "xorm.io/builder" @@ -30,6 +31,8 @@ func iterateUserAccounts(ctx context.Context, each func(*user.User) error) error // addresses would be currently facing a error due to their invalid email address. // Ref: https://github.com/go-gitea/gitea/pull/19085 & https://github.com/go-gitea/gitea/pull/17688 func checkUserEmail(ctx context.Context, logger log.Logger, _ bool) error { + setting.LoadServiceSetting() + // We could use quirky SQL to get all users that start without a [a-zA-Z0-9], but that would mean // DB provider-specific SQL and only works _now_. So instead we iterate through all user accounts // and use the validation.ValidateEmail function to be future-proof. @@ -61,6 +64,8 @@ func checkUserEmail(ctx context.Context, logger log.Logger, _ bool) error { // are allowed for various reasons. This check helps with detecting users that, according // to our reserved names, don't have a valid username. func checkUserName(ctx context.Context, logger log.Logger, _ bool) error { + setting.LoadServiceSetting() + var invalidUserCount int64 if err := iterateUserAccounts(ctx, func(u *user.User) error { if err := user.IsUsableUsername(u.Name); err != nil { From d10034f4d8d3af44810849689513fa34515852dd Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Sat, 25 Jan 2025 01:15:27 +0000 Subject: [PATCH 017/125] [v10.0/forgejo] fix: add non allowed domain translation (#6684) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6677 - Was added in 2559c80bec27a41967b355d214253a83b9ee5dad and accidentally removed in 5a16c9d9c0fd25b25d5eaf1b0ab00f1c113e6b32. - Reworded for clarity. - Resolves #6661 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6684 Reviewed-by: Otto Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- options/locale/locale_en-US.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d2f47adab2..cf69f6ef16 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -644,6 +644,7 @@ team_name_been_taken = The team name is already taken. team_no_units_error = Allow access to at least one repository section. email_been_used = The email address is already used. email_invalid = The email address is invalid. +email_domain_is_not_allowed = The domain of the user's email address %s conflicts with EMAIL_DOMAIN_ALLOWLIST or EMAIL_DOMAIN_BLOCKLIST. Make sure you have set the email address correctly. openid_been_used = The OpenID address "%s" is already used. username_password_incorrect = Username or password is incorrect. password_complexity = Password does not pass complexity requirements: From 0ecf28f37f1f472454d9d2f1f5151a18533b14ff Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Sat, 25 Jan 2025 11:38:02 +0000 Subject: [PATCH 018/125] [v10.0/forgejo] Fix inline file preview for rendered files (#6685) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6572 ### What? This fixes the inline file preview for rendered files (e.g., markdown). [Here, a live issue in v11](https://v11.next.forgejo.org/mahlzahn/test-inline-file-preview/issues/1) and [the same in v7 (with even more bugs)](https://v7.next.forgejo.org/mahlzahn/test-inline-file-preview/issues/1). It fixes 1. the inline preview for possibly rendered files, when the link is specified with `?display=source`. This happens, e.g., if you are watching a (e.g., markdown) file in source and then want to link some of its lines. 2. the link to the source file inside the inline preview for possible rendered files (currently it links to the rendered version and then the `#L…` cannot point to the correct lines). This is done by always adding `?display=source` to the link. ### Screenshots
#### Before ![image](/attachments/898f82d5-d116-465a-89e2-ed83da189762)
#### After ![image](/attachments/41058620-47f3-4f6a-b427-66ef33c1a07f)
### Tests - I added test coverage for Go changes... - [x] in their respective `*_test.go` for unit tests. - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server. - I added test coverage for JavaScript changes... - [ ] in `web_src/js/*.test.js` if it can be unit tested. - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)). I think that this minor edit does not need special tests. Some backend tests have been updated to reflect the addition of URL parameters. #### Manual testing - create a repository with a file that can be rendered with couple of lines inside, e.g., a markdown README.md - go to the source of this file (e.g., `…/src/branch/main/README.md`) - click on the `<> View Source` button (or add `?display=source` to the URL) - click on one of the lines, then on the three dots, then on ”Reference in a new issue“ - continue creating the issue ### Documentation - [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change. - [x] I did not document these changes and I do not expect someone else to do it. ### Release notes - [ ] I do not want this change to show in the release notes. - [ ] I want the title to show in the release notes with a link to this pull request. - [ ] I want the content of the `release-notes/.md` to be be used for the release notes instead of the title. Co-authored-by: Robert Wolff Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6685 Reviewed-by: Gusted Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- modules/markup/file_preview.go | 8 +- modules/markup/html_test.go | 103 ++++++++++++++++++ .../0b/b53b56d70d253ce75c257d3cd6334a41ef2b6c | Bin 0 -> 77 bytes .../35/75ed7948fe86ab56b0a76f796f7995222bec65 | Bin 0 -> 44 bytes .../3c/95f14e5a0ab2c5ba9ee9a47ddc261af4968043 | Bin 0 -> 90 bytes .../72/1f0ce13d83f93d431b849a554a62948b85f573 | 1 + .../72/e0a44ea5761c9055995db18019e459576b3b27 | Bin 0 -> 44 bytes .../72/e1c77b65c7baa0e848557089148833fb54705e | Bin 0 -> 90 bytes .../8b/ccd5176c25898b57da2551e076f769054e0d8e | Bin 0 -> 21 bytes .../c9/8762531dd068cd818300a5f5c7dca5da79b510 | Bin 0 -> 80 bytes .../c9/913120ed2c1e27c1d7752ecdb7a504dc7cf6be | Bin 0 -> 170 bytes .../repo/repo1_filepreview/refs/heads/master | 2 +- 12 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/0b/b53b56d70d253ce75c257d3cd6334a41ef2b6c create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/35/75ed7948fe86ab56b0a76f796f7995222bec65 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/3c/95f14e5a0ab2c5ba9ee9a47ddc261af4968043 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/72/1f0ce13d83f93d431b849a554a62948b85f573 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/72/e0a44ea5761c9055995db18019e459576b3b27 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/72/e1c77b65c7baa0e848557089148833fb54705e create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/8b/ccd5176c25898b57da2551e076f769054e0d8e create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/c9/8762531dd068cd818300a5f5c7dca5da79b510 create mode 100644 modules/markup/tests/repo/repo1_filepreview/objects/c9/913120ed2c1e27c1d7752ecdb7a504dc7cf6be diff --git a/modules/markup/file_preview.go b/modules/markup/file_preview.go index 49a5f1e8ba..3caf08f7bb 100644 --- a/modules/markup/file_preview.go +++ b/modules/markup/file_preview.go @@ -77,6 +77,12 @@ func newFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca commitSha := node.Data[m[4]:m[5]] filePath := node.Data[m[6]:m[7]] + urlFullSource := urlFull + if strings.HasSuffix(filePath, "?display=source") { + filePath = strings.TrimSuffix(filePath, "?display=source") + } else if Type(filePath) != "" { + urlFullSource = node.Data[m[0]:m[6]] + filePath + "?display=source#" + node.Data[m[8]:m[1]] + } hash := node.Data[m[8]:m[9]] preview.start = m[0] @@ -113,7 +119,7 @@ func newFilePreview(ctx *RenderContext, node *html.Node, locale translation.Loca titleBuffer.WriteString(" – ") } - err = html.Render(titleBuffer, createLink(urlFull, filePath, "muted")) + err = html.Render(titleBuffer, createLink(urlFullSource, filePath, "muted")) if err != nil { log.Error("failed to render filepathLink: %v", err) } diff --git a/modules/markup/html_test.go b/modules/markup/html_test.go index 50ea70905c..702c5a716d 100644 --- a/modules/markup/html_test.go +++ b/modules/markup/html_test.go @@ -1026,4 +1026,107 @@ func TestRender_FilePreview(t *testing.T) { localMetas, ) }) + + commitFileURL := util.URLJoin(markup.TestRepoURL, "src", "commit", "c9913120ed2c1e27c1d7752ecdb7a504dc7cf6be", "path", "to", "file.md") + + t.Run("rendered file with ?display=source", func(t *testing.T) { + testRender( + commitFileURL+"?display=source"+"#L1-L2", + `

`+ + `
`+ + `
`+ + `
`+ + `path/to/file.md`+ + `
`+ + ``+ + `Lines 1 to 2 in c991312`+ + ``+ + `
`+ + `
`+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + `
# A`+"\n"+`
B`+"\n"+`
`+ + `
`+ + `
`+ + `

`, + localMetas, + ) + }) + + t.Run("rendered file without ?display=source", func(t *testing.T) { + testRender( + commitFileURL+"#L1-L2", + `

`+ + `
`+ + `
`+ + `
`+ + `path/to/file.md`+ + `
`+ + ``+ + `Lines 1 to 2 in c991312`+ + ``+ + `
`+ + `
`+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + `
# A`+"\n"+`
B`+"\n"+`
`+ + `
`+ + `
`+ + `

`, + localMetas, + ) + }) + + commitFileURL = util.URLJoin(markup.TestRepoURL, "src", "commit", "190d9492934af498c3f669d6a2431dc5459e5b20", "path", "to", "file.go") + + t.Run("normal file with ?display=source", func(t *testing.T) { + testRender( + commitFileURL+"?display=source"+"#L2-L3", + `

`+ + `
`+ + `
`+ + `
`+ + `path/to/file.go`+ + `
`+ + ``+ + `Lines 2 to 3 in 190d949`+ + ``+ + `
`+ + `
`+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + ``+ + `
B`+"\n"+`
C`+"\n"+`
`+ + `
`+ + `
`+ + `

`, + localMetas, + ) + }) } diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/0b/b53b56d70d253ce75c257d3cd6334a41ef2b6c b/modules/markup/tests/repo/repo1_filepreview/objects/0b/b53b56d70d253ce75c257d3cd6334a41ef2b6c new file mode 100644 index 0000000000000000000000000000000000000000..1ab268b76c99e8c8617f0db6e89b040ef9510c65 GIT binary patch literal 77 zcmV-T0J8sh0V^p=O;s>AU@$Z=Ff%bxNXyJg)l1K3Xi>VtFH~43w>Rd!is!cjbLGn; jGm(|#rZ9A$xhkHc+Swg`OEvI8+4oFVKi)n7{P-Lc(|IE6 literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/35/75ed7948fe86ab56b0a76f796f7995222bec65 b/modules/markup/tests/repo/repo1_filepreview/objects/35/75ed7948fe86ab56b0a76f796f7995222bec65 new file mode 100644 index 0000000000000000000000000000000000000000..1493caa3dfe2d9f627884805316bb754bc739b4c GIT binary patch literal 44 zcmb)5VqnO?)H-??ybM8{~)M!?Q_e=RB0B$)E A_y7O^ literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/3c/95f14e5a0ab2c5ba9ee9a47ddc261af4968043 b/modules/markup/tests/repo/repo1_filepreview/objects/3c/95f14e5a0ab2c5ba9ee9a47ddc261af4968043 new file mode 100644 index 0000000000000000000000000000000000000000..3e9c0c0d8bf7f7aa61ae01ad1474dad2cb75d81e GIT binary patch literal 90 zcmV-g0HyzU0V^p=O;s>AVK6ZO0)>Lak_?8T2TS~xmdQ*Aof*5aLGnptc(%2=p@D&! wiHSmSW?p(us%}nZUaDS6MF~SsT~u1Vbh(84zm5YkwvSze7j9Aj0FzA}xNOQLJOBUy literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/72/1f0ce13d83f93d431b849a554a62948b85f573 b/modules/markup/tests/repo/repo1_filepreview/objects/72/1f0ce13d83f93d431b849a554a62948b85f573 new file mode 100644 index 0000000000..d781d4d248 --- /dev/null +++ b/modules/markup/tests/repo/repo1_filepreview/objects/72/1f0ce13d83f93d431b849a554a62948b85f573 @@ -0,0 +1 @@ +xK1@]$JazJR@w+s۲"@VL&J3%f-GDq2>FjBOEݹ:g\1ꦒkEM6D,Ÿ\Ǹ:\6Olmȩ;ϭ|!GE6ZzYβ mwٛi.x-o"L \ No newline at end of file diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/72/e0a44ea5761c9055995db18019e459576b3b27 b/modules/markup/tests/repo/repo1_filepreview/objects/72/e0a44ea5761c9055995db18019e459576b3b27 new file mode 100644 index 0000000000000000000000000000000000000000..7b926dc0d8324cfc92001c553b87fe03262fd3f0 GIT binary patch literal 44 zcmV+{0Mq|?0V^p=O;s?mWH2!R0)>)%2JWraVb^(8ZJx)d*4kV%_Hul$odW>PCkzCG C!V+Tu literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/72/e1c77b65c7baa0e848557089148833fb54705e b/modules/markup/tests/repo/repo1_filepreview/objects/72/e1c77b65c7baa0e848557089148833fb54705e new file mode 100644 index 0000000000000000000000000000000000000000..0bbca73af27a469407e7241c2dad9e0a2504d531 GIT binary patch literal 90 zcmV-g0HyzU0V^p=O;s>AVK6ZO0)>Lak_-mZ(zlf!|JqiEZCIXPnO`|oN&8Kzp@D&! wiHSmSW?p(us%}nZUaDS6MF~SsT~u1Vbh(84zm5YkwvSze7j9Aj0M2b5?sYFEz5oCK literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/8b/ccd5176c25898b57da2551e076f769054e0d8e b/modules/markup/tests/repo/repo1_filepreview/objects/8b/ccd5176c25898b57da2551e076f769054e0d8e new file mode 100644 index 0000000000000000000000000000000000000000..394a7bb50d7dcebdf407950c6e579b2a8ff18744 GIT binary patch literal 21 ccmbAVlXr?Ff%bxNXyJgRZ!8(O=0Lhb5%S?wX-|?mTKUGvhSI! me!P81iuBU+8CsOC@Cy~z$?c7Kuj2Xbz+CzA$V>nw>l|8&RVW<* literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/objects/c9/913120ed2c1e27c1d7752ecdb7a504dc7cf6be b/modules/markup/tests/repo/repo1_filepreview/objects/c9/913120ed2c1e27c1d7752ecdb7a504dc7cf6be new file mode 100644 index 0000000000000000000000000000000000000000..9fc2b7c3125937c08e184007dfd05822729a0e55 GIT binary patch literal 170 zcmV;b09F5Z0gaAJ3PLdq0A2SK*$dJp{a6t35PE>TG{u5H`l?>v4<5kPz`(%B^?Ysv zkZ>`&Dv;z*o!7vYCzLR8R?X~FDT2{)^*OGsCv)SjmjPZJa}9BlDObuxY7CVs2AeRh zgJms_q(sB_alCdo%-S7n?jP*tHgwf44?eZB1(zr#au^aUt+Uq1_igAO6(RaxW%fD` YsO_Y1>-uQ=g!gIDuH|dZ3;EhgOutoAssI20 literal 0 HcmV?d00001 diff --git a/modules/markup/tests/repo/repo1_filepreview/refs/heads/master b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master index df25bf45f0..f3d5d39dd5 100644 --- a/modules/markup/tests/repo/repo1_filepreview/refs/heads/master +++ b/modules/markup/tests/repo/repo1_filepreview/refs/heads/master @@ -1 +1 @@ -4c1aaf56bcb9f39dcf65f3f250726850aed13cd6 +c9913120ed2c1e27c1d7752ecdb7a504dc7cf6be From b3fe6dc4a1b9cea3021fe9d17b1d906316d2fc82 Mon Sep 17 00:00:00 2001 From: Nick Date: Fri, 19 Aug 2022 02:49:18 -0400 Subject: [PATCH 019/125] git-annex tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes https://github.com/neuropoly/gitea/issues/11 Tests: * `git annex init` * `git annex copy --from origin` * `git annex copy --to origin` over: * ssh for: * the owner * a collaborator * a read-only collaborator * a stranger in a * public repo * private repo And then confirms: * Deletion of the remote repo (to ensure lockdown isn't messing with us: https://git-annex.branchable.com/internals/lockdown/#comment-0cc5225dc5abe8eddeb843bfd2fdc382) ------ To support all this: * Add util.FileCmp() * Patch withKeyFile() so it can be nested in other copies of itself ------- Many thanks to Mathieu for giving style tips and catching several bugs, including a subtle one in util.filecmp() which neutered it. Co-authored-by: Mathieu Guay-Paquet Co-authored-by: Matthias Riße --- Makefile | 2 +- modules/util/filecmp.go | 87 ++ modules/util/remove.go | 37 +- .../api_helper_for_declarative_test.go | 25 + tests/integration/git_annex_test.go | 760 ++++++++++++++++++ .../git_helper_for_declarative_test.go | 22 + 6 files changed, 916 insertions(+), 17 deletions(-) create mode 100644 modules/util/filecmp.go create mode 100644 tests/integration/git_annex_test.go diff --git a/Makefile b/Makefile index a9de57e523..95658a7842 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ self := $(location) @tmpdir=`mktemp --tmpdir -d` ; \ echo Using temporary directory $$tmpdir for test repositories ; \ USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \ - STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS + STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS else diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go new file mode 100644 index 0000000000..76e7705cc1 --- /dev/null +++ b/modules/util/filecmp.go @@ -0,0 +1,87 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "io" + "os" +) + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} diff --git a/modules/util/remove.go b/modules/util/remove.go index 0f41471fcc..f2a61ae467 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -43,6 +43,26 @@ func Remove(name string) error { return err } +// MakeWritable recursively makes the named directory writable. +func MakeWritable(name string) error { + return filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) +} + // RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error @@ -55,22 +75,7 @@ func RemoveAll(name string) error { // > (The only bad consequence of this is that rm -rf .git // > doesn't work unless you first run chmod -R +w .git) - err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { - // NB: this is called WalkDir but it works on a single file too - if err == nil { - info, err := d.Info() - if err != nil { - return err - } - - // 0200 == u+w, in octal unix permission notation - err = os.Chmod(path, info.Mode()|0o200) - if err != nil { - return err - } - } - return nil - }) + err = MakeWritable(name) if err != nil { // try again <-time.After(100 * time.Millisecond) diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index dae71ca8ef..5bd8dc5567 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -22,6 +22,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/forms" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -461,3 +462,27 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +// generate and activate an ssh key for the user attached to the APITestContext +// TODO: pick a better name; golang doesn't do method overriding. +func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + // we need to have write:public_key to do this step + // the easiest way is to create a throwaway ctx that is identical but only has that permission + ctxKeyWriter := ctx + ctxKeyWriter.Token = getTokenForLoggedInUser(t, ctx.Session, auth.AccessTokenScopeWriteUser) + + keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String() + withKeyFile(t, keyName, func(keyFile string) { + var key api.PublicKey + + doAPICreateUserKey(ctxKeyWriter, keyName, keyFile, + func(t *testing.T, _key api.PublicKey) { + // save the key ID so we can delete it at the end + key = _key + })(t) + + defer doAPIDeleteUserKey(ctxKeyWriter, key.ID)(t) + + callback() + }) +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go new file mode 100644 index 0000000000..49f26924e3 --- /dev/null +++ b/tests/integration/git_annex_test.go @@ -0,0 +1,760 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "errors" + "fmt" + "math/rand" + "net/url" + "os" + "path" + "regexp" + "strings" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/perm" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/require" +) + +// Some guidelines: +// +// * a APITestContext is an awkward union of session credential + username + target repo +// which is assumed to be owned by that username; if you want to target a different +// repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { + // creating a repo counts as editing the user's profile (is done by POSTing + // to /api/v1/user/repos/) -- which means it needs a User-scoped token and + // both that and editing need a Repo-scoped token because they edit repositories. + rescopedCtx := ctx + rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) + doAPICreateRepository(rescopedCtx, false)(t) + t.Cleanup(func() { util.MakeWritable(setting.RepoRootPath) }) + doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) + + repoURL := createSSHUrl(ctx.GitPath(), u) + + // Fill in fixture data + withAnnexCtxKeyFile(t, ctx, func() { + err = doInitRemoteAnnexRepository(t, repoURL) + }) + if err != nil { + return fmt.Errorf("Unable to initialize remote repo with git-annex fixture: %w", err) + } + return nil +} + +/* +Test that permissions are enforced on git-annex-shell commands. + + Along the way, test that uploading, downloading, and deleting all work. +*/ +func TestGitAnnexPermissions(t *testing.T) { + /* + // TODO: look into how LFS did this + if !setting.Annex.Enabled { + t.Skip() + } + */ + + // Each case below is split so that 'clone' is done as + // the repo owner, but 'copy' as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + t.Run("Public", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + + t.Run("Private", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + }) +} + +/* +Test that 'git annex init' works. + + precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). +*/ +func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { + _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex init`: %w", err) + } + + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) + } + readAnnexUUID = strings.TrimSpace(readAnnexUUID) + + match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) + if !match { + return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) + } + + remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + if err != nil { + return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) + } + + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + match = regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(remoteAnnexUUID) + if !match { + return fmt.Errorf("'git annex init' should have been able to download the remote's uuid; but instead read '%s'", remoteAnnexUUID) + } + + if readAnnexUUID != remoteAnnexUUID { + return fmt.Errorf("'git annex init' should have read the expected annex UUID '%s', but instead got '%s'", remoteAnnexUUID, readAnnexUUID) + } + + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands the annexed file can be found in the remote annex. + annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + } + // Note: this regex is unanchored because 'whereis' outputs multiple lines containing + // headers and 1+ remotes and we just want to find one of them. + match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) + if !match { + return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + } + + return nil +} + +func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "doAnnexInitTest()": + // "git annex copy" will notice and run "git annex init", silently. + // This shouldn't change any results, but be aware in case it does. + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was downloaded + localObjectPath, err := annexObjectPath(repoPath, "large.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was uploaded + localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +// ---- Helpers ---- + +func generateRandomFile(size int, path string) (err error) { + // Generate random file + + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } + + buffer := make([]byte, bufSize) + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } + + return nil +} + +// ---- Annex-specific helpers ---- + +/* +Initialize a repo with some baseline annexed and non-annexed files. + + TODO: perhaps this generator could be replaced with a fixture (see + integrations/gitea-repositories-meta/ and models/fixtures/repository.yml). + However we reuse this template for -different- repos, so maybe not. +*/ +func doInitAnnexRepository(repoPath string) error { + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + if err != nil { + return err + } + defer f.Close() + + // set up git-annex to store certain filetypes via *annex* pointers + // (https://git-annex.branchable.com/internals/pointer_file/). + // but only when run via 'git add' (see git-annex-smudge(1)) + _, err = f.WriteString("* annex.largefiles=anything\n") + if err != nil { + return err + } + _, err = f.WriteString("*.bin filter=annex\n") + if err != nil { + return err + } + f.Close() + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"}) + if err != nil { + return err + } + + // 'git annex init' + err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add a file to the annex + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + if err != nil { + return err + } + + return nil +} + +/* +Initialize a remote repo with some baseline annexed and non-annexed files. +*/ +func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { + repoPath := path.Join(t.TempDir(), path.Base(repoURL.Path)) + // This clone is immediately thrown away, which + // helps force the tests to be end-to-end. + defer util.RemoveAll(repoPath) + + doGitClone(repoPath, repoURL)(t) // TODO: this call is the only reason for the testing.T; can it be removed? + + err := doInitAnnexRepository(repoPath) + if err != nil { + return err + } + + _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + return nil +} + +/* +Find the path in .git/annex/objects/ of the contents for a given annexed file. + + repoPath: the git repository to examine + file: the path (in the repo's current HEAD) of the annex pointer + + TODO: pass a parameter to allow examining non-HEAD branches +*/ +func annexObjectPath(repoPath, file string) (string, error) { + // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. + annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + } + + // There are two formats an annexed file pointer might be: + // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' + // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge + // This recovers $ANNEX_KEY from either case: + annexKey = path.Base(strings.TrimSpace(annexKey)) + + contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + } + contentPath = strings.TrimSpace(contentPath) + + return path.Join(repoPath, contentPath), nil +} + +/* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ +func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") + defer func() { + // reset + if gitAnnexUseGitSSHExists { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + } + }() + + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + + withCtxKeyFile(t, ctx, callback) +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 83d8177460..1e39b988fd 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -42,6 +42,28 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"IdentitiesOnly=yes\" -i \""+keyFile+"\" \"$@\""), 0o700) require.NoError(t, err) + // reset ssh wrapper afterwards + _gitSSH, gitSSHExists := os.LookupEnv("GIT_SSH") + defer func() { + if gitSSHExists { + os.Setenv("GIT_SSH", _gitSSH) + } + }() + + _gitSSHCommand, gitSSHCommandExists := os.LookupEnv("GIT_SSH_COMMAND") + defer func() { + if gitSSHCommandExists { + os.Setenv("GIT_SSH_COMMAND", _gitSSHCommand) + } + }() + + _gitSSHVariant, gitSSHVariantExists := os.LookupEnv("GIT_SSH_VARIANT") + defer func() { + if gitSSHVariantExists { + os.Setenv("GIT_SSH_VARIANT", _gitSSHVariant) + } + }() + // Setup ssh wrapper t.Setenv("GIT_SSH", path.Join(tmpDir, "ssh")) t.Setenv("GIT_SSH_COMMAND", From b5fc00042b44946d77e52c098ea15673761b2d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 12:57:45 +0200 Subject: [PATCH 020/125] Adapt patch to upstream changes Repository creation now expects an objectFormat to be specified for git. --- tests/integration/git_annex_test.go | 522 ++++++++++++++-------------- 1 file changed, 262 insertions(+), 260 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 49f26924e3..e364e68852 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -34,13 +34,13 @@ import ( // which is assumed to be owned by that username; if you want to target a different // repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" -func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool, objectFormat git.ObjectFormat) (err error) { // creating a repo counts as editing the user's profile (is done by POSTing // to /api/v1/user/repos/) -- which means it needs a User-scoped token and // both that and editing need a Repo-scoped token because they edit repositories. rescopedCtx := ctx rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) - doAPICreateRepository(rescopedCtx, false)(t) + doAPICreateRepository(rescopedCtx, false, objectFormat)(t) t.Cleanup(func() { util.MakeWritable(setting.RepoRootPath) }) doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) @@ -78,365 +78,367 @@ func TestGitAnnexPermissions(t *testing.T) { // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { - t.Run("Public", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) - - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) - - // double-check it's public - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) - require.NoError(t, err) - require.False(t, repo.IsPrivate) - - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL - remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost - - // Different sessions, so we can test different permissions. - // We leave Reponame blank because we don't actually then later add it according to each case if needed - // - // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) - readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) - outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access - - // set up collaborators - doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) - - // tests - t.Run("Owner", func(t *testing.T) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { + ownerCtx := NewAPITestContext(t, "user2", "annex-public"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false, objectFormat)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Writer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Writer", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Reader", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Reader", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Outsider", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ownerCtx)(t) - _, statErr := os.Stat(remoteRepoPath) - require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") - }) - }) - - t.Run("Private", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) - - // create a private repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) - - // double-check it's private - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) - require.NoError(t, err) - require.True(t, repo.IsPrivate) - - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL - remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost - - // Different sessions, so we can test different permissions. - // We leave Reponame blank because we don't actually then later add it according to each case if needed - // - // NB: these usernames need to match appropriate entries in models/fixtures/user.yml - writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) - readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) - outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access - // Note: there's also full anonymous access, which is only available for public HTTP repos; - // it should behave the same as 'outsider' but we (will) test it separately below anyway - - // set up collaborators - doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) - doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) - - // tests - t.Run("Owner", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Delete", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) - - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) - }) + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") }) }) - t.Run("Writer", func(t *testing.T) { + t.Run("Private", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - t.Run("SSH", func(t *testing.T) { + ownerCtx := NewAPITestContext(t, "user2", "annex-private"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true, objectFormat)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + // Remote addresses of the repo + repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Reader", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Writer", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { + t.Run("Reader", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") - }) + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) }) }) }) - }) - t.Run("Delete", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() - // Delete the repo, make sure it's fully gone - doAPIDeleteRepository(ownerCtx)(t) - _, statErr := os.Stat(remoteRepoPath) - require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) }) }) }) From 90f53a4e7c67581e84e65505e495a2625197d904 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 12:12:39 +0200 Subject: [PATCH 021/125] Adapt patch to upstream changes A dead code check started to complain because FileCmp was only used in tests. Moved the function to test_utils. --- modules/util/filecmp.go | 87 ----------------------------- tests/integration/git_annex_test.go | 4 +- tests/test_utils.go | 79 ++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 89 deletions(-) delete mode 100644 modules/util/filecmp.go diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go deleted file mode 100644 index 76e7705cc1..0000000000 --- a/modules/util/filecmp.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package util - -import ( - "bytes" - "io" - "os" -) - -// Decide if two files have the same contents or not. -// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. -// *Follows* symlinks. -// -// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. -// -// derived from https://stackoverflow.com/a/30038571 -// under CC-BY-SA-4.0 by several contributors -func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { - if chunkSize == 0 { - chunkSize = 4 * 1024 - } - - // shortcuts: check file metadata - stat1, err := os.Stat(file1) - if err != nil { - return false, err - } - - stat2, err := os.Stat(file2) - if err != nil { - return false, err - } - - // are inputs are literally the same file? - if os.SameFile(stat1, stat2) { - return true, nil - } - - // do inputs at least have the same size? - if stat1.Size() != stat2.Size() { - return false, nil - } - - // long way: compare contents - f1, err := os.Open(file1) - if err != nil { - return false, err - } - defer f1.Close() - - f2, err := os.Open(file2) - if err != nil { - return false, err - } - defer f2.Close() - - b1 := make([]byte, chunkSize) - b2 := make([]byte, chunkSize) - for { - n1, err1 := io.ReadFull(f1, b1) - n2, err2 := io.ReadFull(f2, b2) - - // https://pkg.go.dev/io#Reader - // > Callers should always process the n > 0 bytes returned - // > before considering the error err. Doing so correctly - // > handles I/O errors that happen after reading some bytes - // > and also both of the allowed EOF behaviors. - - if !bytes.Equal(b1[:n1], b2[:n2]) { - return false, nil - } - - if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { - return true, nil - } - - // some other error, like a dropped network connection or a bad transfer - if err1 != nil { - return false, err1 - } - if err2 != nil { - return false, err2 - } - } -} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index e364e68852..667e57c499 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -522,7 +522,7 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } @@ -575,7 +575,7 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) if err != nil { return err } diff --git a/tests/test_utils.go b/tests/test_utils.go index b3c03a30a1..6e9374db25 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -5,9 +5,11 @@ package tests import ( + "bytes" "context" "database/sql" "fmt" + "io" "os" "path" "path/filepath" @@ -488,3 +490,80 @@ func CreateDeclarativeRepo(t *testing.T, owner *user_model.User, name string, en return CreateDeclarativeRepoWithOptions(t, owner, opts) } + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} From 8cd079ae15b792254621dc055ee39f35d753d9e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 12 Aug 2024 11:34:45 +0200 Subject: [PATCH 022/125] Adapt patch to upstream changes --- tests/integration/git_annex_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 667e57c499..f52f8759b1 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -752,11 +752,11 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { defer func() { // reset if gitAnnexUseGitSSHExists { - os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + t.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) } }() - os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + t.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set withCtxKeyFile(t, ctx, callback) } From 7b7c804ea4f938e581238d3c4ddf667b7ebac2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Apr 2024 17:25:32 +0200 Subject: [PATCH 023/125] Install git-annex in the testing workflow --- .forgejo/workflows/testing.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 3d8ad443da..fb5def2028 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -196,7 +196,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' @@ -232,7 +232,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' @@ -253,7 +253,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-lfs + packages: git git-annex git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' From ca881e7e31310edf7ba1c5ee2d7e34f94ce9d1fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 25 Apr 2024 10:21:08 +0200 Subject: [PATCH 024/125] Fix failing tests Multiple tests that worked fine on v1.20.4-1 started to fail after the rebase onto v1.20.5-1. These tests are: - TestGitAnnexPermissions/Private/Owner/HTTP/Init - TestGitAnnexPermissions/Private/Owner/HTTP/Download - TestGitAnnexPermissions/Private/Writer/HTTP/Init - TestGitAnnexPermissions/Private/Writer/HTTP/Download - TestGitAnnexPermissions/Private/Reader/HTTP/Init - TestGitAnnexPermissions/Private/Reader/HTTP/Download What these tests have in common is that they all operate on a private repository via http with authentication. They broke at some point between v1.20.4-1 and v1.20.5-1, so I did a bisect between these two points running the offending tests. This brought me to the conclusion that ee48c0d5ea8f148c7f4a792b590d93eb51cbf67d introduced the issue. The thing is, this commit does not change any code, it only changes the test environment. Among other things that didn't look as suspicious, it changes the container image from a bespoke test_env image based on debian bullseye to a node image based on debian bookworm. Obviously, this means that there are many version differences between the two. The first one I looked at was git. The previous bullseye image used a manually installed git version 2.40.0, while the bookworm image has 2.39.2 installed. Updating git in the new image did not fix the issue, however. The next thing I looked at was the git-annex version. Bullseye had 8.20210223 installed and worked, while bookworm used 10.20230126 when the tests broke. So I tried my luck upgrading to a more recent version via neurodebian (10.20240227-1~ndall+1). This still worked fine on bullseye and now also works fine on bookworm. I have no idea why this specific version of git-annex broke the tests, but at least there was a commit to pinpoint this to, which isn't always the case with docker images silently changing beneath you... Below are the versions as they are reported by git and git-annex: bullseye (works): git version 2.30.2 git-annex version: 8.20210223 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 bullseye + git-annex from neurodebian (works): git version 2.30.2 git-annex version: 10.20240227-1~ndall+1 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 bookworm (fails): git version 2.39.2 git-annex version: 10.20230126 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 bookworm + git-annex from neurodebian (works): git version 2.39.2 git-annex version: 10.20240227-1~ndall+1 build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1 key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X* remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external operating system: linux x86_64 supported repository versions: 8 9 10 upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10 --- .forgejo/workflows-composite/apt-install-from/action.yaml | 3 +++ .forgejo/workflows/testing.yml | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.forgejo/workflows-composite/apt-install-from/action.yaml b/.forgejo/workflows-composite/apt-install-from/action.yaml index 615e7cb184..ab55883a11 100644 --- a/.forgejo/workflows-composite/apt-install-from/action.yaml +++ b/.forgejo/workflows-composite/apt-install-from/action.yaml @@ -13,6 +13,8 @@ runs: run: | export DEBIAN_FRONTEND=noninteractive echo "deb http://deb.debian.org/debian/ ${RELEASE} main" > "/etc/apt/sources.list.d/${RELEASE}.list" + wget -O- http://neuro.debian.net/lists/bookworm.de-fzj.libre | tee /etc/apt/sources.list.d/neurodebian.sources.list + apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0xA5D32F012649A5A9 env: RELEASE: ${{inputs.release}} - name: install packages @@ -24,6 +26,7 @@ runs: - name: remove temporary package list to prevent using it in other steps run: | rm "/etc/apt/sources.list.d/${RELEASE}.list" + rm "/etc/apt/sources.list.d/neurodebian.sources.list" apt-get update -qq env: RELEASE: ${{inputs.release}} diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index fb5def2028..3d73df5066 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -196,7 +196,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' @@ -232,7 +232,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' @@ -253,7 +253,7 @@ jobs: - name: install dependencies & git >= 2.42 uses: ./.forgejo/workflows-composite/apt-install-from with: - packages: git git-annex git-lfs + packages: git git-annex-standalone git-lfs - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' From ffae358704e389e36cc44e389dd1539515af0e76 Mon Sep 17 00:00:00 2001 From: Nick Date: Mon, 19 Sep 2022 17:22:42 -0400 Subject: [PATCH 025/125] git-annex: add configuration setting [annex].ENABLED Fixes https://github.com/neuropoly/gitea/issues/8 Co-authored-by: Mathieu Guay-Paquet --- cmd/serv.go | 3 +-- cmd/web.go | 4 ++++ custom/conf/app.example.ini | 9 +++++++++ modules/setting/annex.go | 20 ++++++++++++++++++++ modules/setting/setting.go | 1 + routers/private/serv.go | 2 +- tests/integration/git_annex_test.go | 9 +++------ tests/mysql.ini.tmpl | 3 +++ tests/pgsql.ini.tmpl | 3 +++ tests/sqlite.ini.tmpl | 3 +++ 10 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 modules/setting/annex.go diff --git a/cmd/serv.go b/cmd/serv.go index 7d16547053..ac4aa36599 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -215,8 +215,7 @@ func runServ(c *cli.Context) error { } if verb == gitAnnexShellVerb { - // if !setting.Annex.Enabled { // TODO: https://github.com/neuropoly/gitea/issues/8 - if false { + if !setting.Annex.Enabled { return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") } diff --git a/cmd/web.go b/cmd/web.go index 44babd51c5..a817204aa6 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -311,6 +311,10 @@ func listen(m http.Handler, handleRedirector bool) error { log.Info("LFS server enabled") } + if setting.Annex.Enabled { + log.Info("git-annex enabled") + } + var err error switch setting.Protocol { case setting.HTTP: diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index ee8ad66668..486adc60a0 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2678,6 +2678,15 @@ LEVEL = Info ;; Limit the number of concurrent upload/download operations within a batch ;BATCH_OPERATION_CONCURRENCY = 8 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[annex] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Whether git-annex is enabled; defaults to false +;ENABLED = false + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for packages, will override storage setting diff --git a/modules/setting/annex.go b/modules/setting/annex.go new file mode 100644 index 0000000000..a0eeac9bb8 --- /dev/null +++ b/modules/setting/annex.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "code.gitea.io/gitea/modules/log" +) + +// Annex represents the configuration for git-annex +var Annex = struct { + Enabled bool `ini:"ENABLED"` +}{} + +func loadAnnexFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("annex") + if err := sec.MapTo(&Annex); err != nil { + log.Fatal("Failed to map Annex settings: %v", err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index c9d30836ac..9710fb23d8 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -153,6 +153,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error { loadCamoFrom(cfg) loadI18nFrom(cfg) loadGitFrom(cfg) + loadAnnexFrom(cfg) loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadQuotaFrom(cfg) diff --git a/routers/private/serv.go b/routers/private/serv.go index b42fbc04f4..a3a0c6961a 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -289,7 +289,7 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey - ( /*setting.Annex.Enabled && */ len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode + (setting.Annex.Enabled && len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { results.UserMode = deployKey.Mode diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index f52f8759b1..9b8c53ff59 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -62,12 +62,9 @@ Test that permissions are enforced on git-annex-shell commands. Along the way, test that uploading, downloading, and deleting all work. */ func TestGitAnnexPermissions(t *testing.T) { - /* - // TODO: look into how LFS did this - if !setting.Annex.Enabled { - t.Skip() - } - */ + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } // Each case below is split so that 'clone' is done as // the repo owner, but 'copy' as the user under test. diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index e15e79952b..b8320265ab 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -97,6 +97,9 @@ DISABLE_QUERY_AUTH_TOKEN = true [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index 340531fb38..781508a648 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -122,6 +122,9 @@ MINIO_LOCATION = us-east-1 MINIO_USE_SSL = false MINIO_CHECKSUM_ALGORITHM = md5 +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 277916a539..231c0d19c2 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -102,6 +102,9 @@ JWT_SECRET = KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true From 48f21fd213e6ce2b3823714695591d5acf5d68ce Mon Sep 17 00:00:00 2001 From: Nick Date: Tue, 20 Sep 2022 16:17:56 -0400 Subject: [PATCH 026/125] git-annex: support downloading over HTTP This makes HTTP symmetric with SSH clone URLs. This gives us the fancy feature of _anonymous_ downloads, so people can access datasets without having to set up an account or manage ssh keys. Previously, to access "open access" data shared this way, users would need to: 1. Create an account on gitea.example.com 2. Create ssh keys 3. Upload ssh keys (and make sure to find and upload the correct file) 4. `git clone git@gitea.example.com:user/dataset.git` 5. `cd dataset` 6. `git annex get` This cuts that down to just the last three steps: 1. `git clone https://gitea.example.com/user/dataset.git` 2. `cd dataset` 3. `git annex get` This is significantly simpler for downstream users, especially for those unfamiliar with the command line. Unfortunately there's no uploading. While git-annex supports uploading over HTTP to S3 and some other special remotes, it seems to fail on a _plain_ HTTP remote. See https://github.com/neuropoly/gitea/issues/7 and https://git-annex.branchable.com/forum/HTTP_uploads/#comment-ce28adc128fdefe4c4c49628174d9b92. This is not a major loss since no one wants uploading to be anonymous anyway. To support private repos, I had to hunt down and patch a secret extra security corner that Gitea only applies to HTTP for some reason (services/auth/basic.go). This was guided by https://git-annex.branchable.com/tips/setup_a_public_repository_on_a_web_site/ Fixes https://github.com/neuropoly/gitea/issues/3 Co-authored-by: Mathieu Guay-Paquet --- modules/git/command.go | 3 +- routers/web/repo/githttp.go | 31 ++ routers/web/web.go | 13 + services/auth/auth.go | 11 + services/auth/basic.go | 4 +- tests/integration/git_annex_test.go | 360 +++++++++++++++++- .../git_helper_for_declarative_test.go | 7 + 7 files changed, 412 insertions(+), 17 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index a3d43aaec6..d3e6b7b8bc 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -457,12 +457,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests +// It also re-enables git-credential(1), which is used to test git-annex's HTTP support func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run filteredLFSGlobalArgs := make(TrustedCmdArgs, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { - if strings.Contains(string(arg), "lfs") { + if strings.Contains(string(arg), "lfs") || strings.Contains(string(arg), "credential") { j-- } else { filteredLFSGlobalArgs[j] = arg diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index c1adca174f..fc6864ed40 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -597,3 +597,34 @@ func GetIdxFile(ctx *context.Context) { h.sendFile(ctx, "application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx") } } + +// GetAnnexObject implements git-annex dumb HTTP +func GetAnnexObject(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + // git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key} + // where key is a string containing the size and (usually SHA256) checksum of the file, + // and hash1+hash2 are the first few bits of the md5sum of key itself. + // ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory) + // ref: https://git-annex.branchable.com/internals/hashing/ + + // keyDir should = key, but we don't enforce that + object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + + // Sanitize the input against directory traversals. + // + // This works because at the filesystem root, "/.." = "/"; + // So if a path starts rooted ("/"), path.Clean(), which + // path.Join() calls internally, removes all '..' prefixes. + // After, this unroots the path unconditionally ([1:]), which + // works because we know the input is never supposed to be rooted. + // + // The router code probably also disallows "..", so this + // should be redundant, but it's defensive to keep it + // whenever touching filesystem paths with user input. + object = path.Join("/", object)[1:] + + h.setHeaderCacheForever() + h.sendFile("application/octet-stream", "annex/objects/"+object) + } +} diff --git a/routers/web/web.go b/routers/web/web.go index 4d8d280c89..bee637e76b 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -356,6 +356,13 @@ func registerRoutes(m *web.Route) { } } + annexEnabled := func(ctx *context.Context) { + if !setting.Annex.Enabled { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -1635,6 +1642,12 @@ func registerRoutes(m *web.Route) { }) }, ignSignInAndCsrf, lfsServerEnabled) + m.Group("", func() { + // for git-annex + m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + gitHTTPRouters(m) }) }) diff --git a/services/auth/auth.go b/services/auth/auth.go index c10872313f..100e5b25a2 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -61,6 +61,17 @@ func isArchivePath(req *http.Request) bool { return archivePathRe.MatchString(req.URL.Path) } +var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) + +func isAnnexPath(req *http.Request) bool { + if setting.Annex.Enabled { + // "/config" is git's config, not specifically git-annex's; but the only current + // user of it is when git-annex downloads the annex.uuid during 'git annex init'. + return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path) + } + return false +} + // handleSignIn clears existing session variables and stores new ones for the specified user object func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) { // We need to regenerate the session... diff --git a/services/auth/basic.go b/services/auth/basic.go index d489164954..8e8fbfc9c7 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -43,8 +43,8 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + // Basic authentication should only fire on API, Download or on Git, LFSPaths or Git-Annex paths + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) && !isAnnexPath(req) { return nil, nil } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 9b8c53ff59..89a5cb2389 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -59,7 +59,8 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, /* Test that permissions are enforced on git-annex-shell commands. - Along the way, test that uploading, downloading, and deleting all work. + Along the way, this also tests that uploading, downloading, and deleting all work, + so we haven't written separate tests for those. */ func TestGitAnnexPermissions(t *testing.T) { if !setting.Annex.Enabled { @@ -75,6 +76,16 @@ func TestGitAnnexPermissions(t *testing.T) { // 'annex copy' -- potentially leaving a security gap. onGiteaRun(t, func(t *testing.T, u *url.URL) { + // Tell git-annex to allow http://127.0.0.1, http://localhost and http://::1. Without + // this, all `git annex` commands will silently fail when run against http:// remotes + // without explaining what's wrong. + // + // Note: onGiteaRun() sets up an alternate HOME so this actually edits + // tests/integration/gitea-integration-*/data/home/.gitconfig and + // if you're debugging you need to remember to match that. + _, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("annex.security.allowed-ip-addresses", "all").RunStdString(&git.RunOpts{}) + require.NoError(t, err) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -89,8 +100,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.False(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -112,6 +121,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -139,6 +150,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -147,6 +183,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -174,6 +212,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -182,6 +245,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -209,6 +274,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -217,6 +307,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -244,6 +336,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -269,8 +416,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.True(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -294,6 +439,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -321,6 +468,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -329,6 +501,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -356,6 +530,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -364,6 +563,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -391,6 +592,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -399,6 +625,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -426,6 +654,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -447,7 +730,7 @@ Test that 'git annex init' works. precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). */ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { - _, _, err = git.NewCommand(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex init`: %w", err) } @@ -455,7 +738,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 0: 'git config remote.origin.annex-uuid'. // Demonstrates that 'git annex init' successfully contacted // the remote git-annex and was able to learn its ID number. - readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + readAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) } @@ -466,7 +749,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) } - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + remoteAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) if err != nil { return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) } @@ -483,7 +766,7 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { // - method 1: 'git annex whereis'. // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) } @@ -502,7 +785,7 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { // "git annex copy" will notice and run "git annex init", silently. // This shouldn't change any results, but be aware in case it does. - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -550,12 +833,12 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -667,7 +950,7 @@ func doInitAnnexRepository(repoPath string) error { } // 'git annex init' - err = git.NewCommand(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -705,7 +988,7 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -757,3 +1040,52 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { withCtxKeyFile(t, ctx, callback) } + +/* +Like withKeyFile(), but sets HTTP credentials instead of SSH credentials. + + It does this by temporarily arranging through `git config --global` + to use git-credential-store(1) with the password written to a tempfile. + + This is the only reliable way to pass HTTP credentials non-interactively + to git-annex. See https://git-annex.branchable.com/bugs/http_remotes_ignore_annex.web-options_--netrc/#comment-b5a299e9826b322f2d85c96d4929a430 + for joeyh's proclamation on the subject. + + This **is only effective** when used around git.NewCommandContextNoGlobals() calls. + git.NewCommand() disables credential.helper as a precaution (see modules/git/git.go). + + In contrast, the tests in git_test.go put the password in the remote's URL like + `git config remote.origin.url http://user2:password@localhost:3003/user2/repo-name.git`, + writing the password in repoPath+"/.git/config". That would be equally good, except + that git-annex ignores it! +*/ +func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, callback func()) { + credentialedURL := *u + credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + + creds := path.Join(t.TempDir(), "creds") + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + + originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) + if err != nil && !err.IsExitCode(1) { + // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) + // but catch all others + require.NoError(t, err) + } + hasOriginalCredentialHelper := (err == nil) + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper", fmt.Sprintf("store --file=%s", creds)).RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + defer (func() { + // reset + if hasOriginalCredentialHelper { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("credential.helper").AddDynamicArguments(originalCredentialHelper).RunStdString(&git.RunOpts{}) + } else { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddOptionValues("--unset").AddArguments("credential.helper").RunStdString(&git.RunOpts{}) + } + require.NoError(t, err) + })() + + callback() +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 1e39b988fd..575f01dcdb 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -73,6 +73,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(gitPath string, u *url.URL) *url.URL { + // this assumes u contains the HTTP base URL that Gitea is running on + u2 := *u + u2.Path = gitPath + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" From 0bdc572df99f1aea6f7ac068c3dabe3089ebbcc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 1 Mar 2024 14:23:33 +0100 Subject: [PATCH 027/125] Replace m.GetOptions with m.Methods This applies the same changes that were done in 265cd70bdb152291a13e520cff1da70b8c029432 to the git-annex specific routes as well. --- routers/web/web.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index bee637e76b..4480be4498 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1644,8 +1644,8 @@ func registerRoutes(m *web.Route) { m.Group("", func() { // for git-annex - m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` - m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) gitHTTPRouters(m) From f863511a14a9180b666d0f60e135397d4cee079e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 29 Apr 2024 15:52:47 +0200 Subject: [PATCH 028/125] Fix exit code check for git command The err.IsExitCode method was changed to a function IsErrorExitCode taking err as its first argument in 1e7a6483b8322ad5e1183545a6283f137a0546ac. --- tests/integration/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 89a5cb2389..5423d86d34 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1067,7 +1067,7 @@ func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, call require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) - if err != nil && !err.IsExitCode(1) { + if err != nil && !git.IsErrorExitCode(err, 1) { // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) // but catch all others require.NoError(t, err) From 768a3a9c83576f5cc9dd4b6b7ef9f10a5d4041ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 29 Apr 2024 17:44:58 +0200 Subject: [PATCH 029/125] Adapt patch to upstream changes Usage of `path` was replaced by `path/filepath` in upstream forgejo, and it made sense to use that as well where `path` was previously used. The `setHeaderCacheForever` function and the `sendFile` method had their signature changed. --- routers/web/repo/githttp.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index fc6864ed40..d715b6bdf0 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -609,7 +609,7 @@ func GetAnnexObject(ctx *context.Context) { // ref: https://git-annex.branchable.com/internals/hashing/ // keyDir should = key, but we don't enforce that - object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + object := filepath.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) // Sanitize the input against directory traversals. // @@ -622,9 +622,9 @@ func GetAnnexObject(ctx *context.Context) { // The router code probably also disallows "..", so this // should be redundant, but it's defensive to keep it // whenever touching filesystem paths with user input. - object = path.Join("/", object)[1:] + object = filepath.Join(string(filepath.Separator), object)[1:] - h.setHeaderCacheForever() - h.sendFile("application/octet-stream", "annex/objects/"+object) + setHeaderCacheForever(ctx) + h.sendFile(ctx, "application/octet-stream", "annex/objects/"+object) } } From 0371c78a77165a505c875746b63f4ad9d5157c22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 30 Apr 2024 11:25:16 +0200 Subject: [PATCH 030/125] Adapt patch to upstream changes The "context_service" import was changed to use the default name of just "context". The patch set had to be adapted for that. --- routers/web/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/web.go b/routers/web/web.go index 4480be4498..4cf4086aba 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1646,7 +1646,7 @@ func registerRoutes(m *web.Route) { // for git-annex m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) - }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + }, ignSignInAndCsrf, annexEnabled, context.UserAssignmentWeb()) gitHTTPRouters(m) }) From 5651c5a69dea5bdd9e766fc2a7ab34fe8c199d25 Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:28:55 -0500 Subject: [PATCH 031/125] git-annex: create modules/annex This moves the `annexObjectPath()` helper out of the tests and into a dedicated sub-package as `annex.ContentLocation()`, and expands it with `.Pointer()` (which validates using `git annex examinekey`), `.IsAnnexed()` and `.Content()` to make it a more useful module. The tests retain their own wrapper version of `ContentLocation()` because I tried to follow close to the API modules/lfs uses, which in terms of abstract `git.Blob` and `git.TreeEntry` objects, not in terms of `repoPath string`s which are more convenient for the tests. --- modules/annex/annex.go | 154 ++++++++++++++++++++++++++++ modules/git/blob.go | 4 + tests/integration/git_annex_test.go | 41 ++++---- 3 files changed, 181 insertions(+), 18 deletions(-) create mode 100644 modules/annex/annex.go diff --git a/modules/annex/annex.go b/modules/annex/annex.go new file mode 100644 index 0000000000..bb049d77ed --- /dev/null +++ b/modules/annex/annex.go @@ -0,0 +1,154 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Unlike modules/lfs, which operates mainly on git.Blobs, this operates on git.TreeEntrys. +// The motivation for this is that TreeEntrys have an easy pointer to the on-disk repo path, +// while blobs do not (in fact, if building with TAGS=gogit, blobs might exist only in a mock +// filesystem, living only in process RAM). We must have the on-disk path to do anything +// useful with git-annex because all of its interesting data is on-disk under .git/annex/. + +package annex + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +const ( + // > The maximum size of a pointer file is 32 kb. + // - https://git-annex.branchable.com/internals/pointer_file/ + // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: + blobSizeCutoff = 32 * 1024 +) + +// ErrInvalidPointer occurs if the pointer's value doesn't parse +var ErrInvalidPointer = errors.New("Not a git-annex pointer") + +// Gets the content of the blob as raw text, up to n bytes. +// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) +func getBlobContent(b *git.Blob, n int) (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + buf := make([]byte, n) + n, _ = util.ReadAtMost(dataRc, buf) + buf = buf[:n] + return string(buf), nil +} + +func Pointer(blob *git.Blob) (string, error) { + // git-annex doesn't seem fully spec what its pointer are, but + // the fullest description is here: + // https://git-annex.branchable.com/internals/pointer_file/ + + // a pointer can be: + // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' + // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' + // + // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because + // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. + + if blob.Size() > blobSizeCutoff { + // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. + // https://git-annex.branchable.com/internals/pointer_file/ + + // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. + return "", ErrInvalidPointer + } + + pointer, err := getBlobContent(blob, blobSizeCutoff) + if err != nil { + return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) + } + + // the spec says a pointer file can contain multiple lines each with a pointer in them + // but that makes no sense to me, so I'm just ignoring all but the first + lines := strings.Split(pointer, "\n") + if len(lines) < 1 { + return "", ErrInvalidPointer + } + pointer = lines[0] + + // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it + if !strings.Contains(pointer, "/annex/") { + return "", ErrInvalidPointer + } + + // extract $KEY + pointer = path.Base(strings.TrimSpace(pointer)) + + // ask git-annex's opinion on $KEY + // XXX: this is probably a bit slow, especially if this operation gets run often + // and examinekey is not that strict: + // - it doesn't enforce that the "BACKEND" tag is one it knows, + // - it doesn't enforce that the fields and their format fit the "BACKEND" tag + // so maybe this is a wasteful step + _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + // TODO: make ErrInvalidPointer into a type capable of wrapping err + if strings.TrimSpace(examineStderr) == "git-annex: bad key" { + return "", ErrInvalidPointer + } + return "", err + } + + return pointer, nil +} + +// return the absolute path of the content pointed to by the annex pointer stored in the git object +// errors if the content is not found in this repo +func ContentLocation(blob *git.Blob) (string, error) { + pointer, err := Pointer(blob) + if err != nil { + return "", err + } + + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(blob.Repo().Path, contentLocation) + + return contentLocation, nil +} + +// returns a stream open to the annex content +func Content(blob *git.Blob) (*os.File, error) { + contentLocation, err := ContentLocation(blob) + if err != nil { + return nil, err + } + + return os.Open(contentLocation) +} + +// whether the object appears to be a valid annex pointer +// does *not* verify if the content is actually in this repo; +// for that, use ContentLocation() +func IsAnnexed(blob *git.Blob) (bool, error) { + if !setting.Annex.Enabled { + return false, nil + } + + // Pointer() is written to only return well-formed pointers + // so the test is just to see if it errors + _, err := Pointer(blob) + if err != nil { + if errors.Is(err, ErrInvalidPointer) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/modules/git/blob.go b/modules/git/blob.go index 2f02693428..bbfab7d005 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -126,6 +126,10 @@ func (b *blobReader) Close() error { return nil } +func (b *Blob) Repo() *Repository { + return b.repo +} + // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { return b.name diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 5423d86d34..95693219f6 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -19,6 +19,7 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/perm" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" @@ -791,13 +792,13 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was downloaded - localObjectPath, err := annexObjectPath(repoPath, "large.bin") + localObjectPath, err := contentLocation(repoPath, "large.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "large.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") if err != nil { return err } @@ -844,13 +845,13 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { } // verify the file was uploaded - localObjectPath, err := annexObjectPath(repoPath, "contribution.bin") + localObjectPath, err := contentLocation(repoPath, "contribution.bin") if err != nil { return err } // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - remoteObjectPath, err := annexObjectPath(remoteRepoPath, "contribution.bin") + remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") if err != nil { return err } @@ -1004,26 +1005,30 @@ Find the path in .git/annex/objects/ of the contents for a given annexed file. TODO: pass a parameter to allow examining non-HEAD branches */ -func annexObjectPath(repoPath, file string) (string, error) { - // NB: `git annex lookupkey` is more reliable, but doesn't work in bare repos. - annexKey, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "show").AddDynamicArguments("HEAD:" + file).RunStdString(&git.RunOpts{Dir: repoPath}) +func contentLocation(repoPath, file string) (path string, err error) { + path = "" + + repo, err := git.OpenRepository(git.DefaultContext, repoPath) if err != nil { - return "", fmt.Errorf("in %s: %w", repoPath, err) // the error from git prints the filename but not repo + return path, nil } - // There are two formats an annexed file pointer might be: - // * a symlink to .git/annex/objects/$HASHDIR/$ANNEX_KEY/$ANNEX_KEY - used by files created with 'git annex add' - // * a text file containing /annex/objects/$ANNEX_KEY - used by files for which 'git add' was configured to run git-annex-smudge - // This recovers $ANNEX_KEY from either case: - annexKey = path.Base(strings.TrimSpace(annexKey)) - - contentPath, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(annexKey).RunStdString(&git.RunOpts{Dir: repoPath}) + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be annexed: %w", repoPath, file, err) + return path, nil } - contentPath = strings.TrimSpace(contentPath) - return path.Join(repoPath, contentPath), nil + commit, err := repo.GetCommit(commitID) + if err != nil { + return path, nil + } + + treeEntry, err := commit.GetTreeEntryByPath(file) + if err != nil { + return path, nil + } + + return annex.ContentLocation(treeEntry.Blob()) } /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ From cb218de0d2f57aa003af3b3e61ed905c2d960552 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 14 May 2024 11:48:43 +0200 Subject: [PATCH 032/125] Adapt patch to upstream changes The git repository must be closed after using it. Without this change some tests started to fail due to the lingering repository running into a timeout. --- tests/integration/git_annex_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 95693219f6..d8e1d49904 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1012,6 +1012,7 @@ func contentLocation(repoPath, file string) (path string, err error) { if err != nil { return path, nil } + defer repo.Close() commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags if err != nil { From 7ed9add4c5d0d12923eca6253067b2b0bdddd66e Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 00:40:06 -0500 Subject: [PATCH 033/125] git-annex: make /media/ download annexed content Previously, Gitea's LFS support allowed direct-downloads of LFS content, via http://$HOSTNAME:$PORT/$USER/$REPO/media/branch/$BRANCH/$FILE Expand that grace to git-annex too. Now /media should provide the relevant *content* from the .git/annex/objects/ folder. This adds tests too. And expands the tests to try symlink-based annexing, since /media implicitly supports both that and pointer-file-based annexing. --- routers/web/repo/download.go | 21 ++++ tests/integration/git_annex_test.go | 146 ++++++++++++++++++++++++---- 2 files changed, 148 insertions(+), 19 deletions(-) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index 1e87bbf015..aefaa79521 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -9,6 +9,7 @@ import ( "time" git_model "code.gitea.io/gitea/models/git" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" "code.gitea.io/gitea/modules/lfs" @@ -79,6 +80,26 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } closed = true + // check for git-annex files + // (this code is weirdly redundant because I'm trying not to delete any lines in order to make merges easier) + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + ctx.ServerError("annex.IsAnnexed", err) + return err + } + if isAnnexed { + content, err := annex.Content(blob) + if err != nil { + // XXX are there any other possible failure cases here? + // there are, there could be unrelated io errors; those should be ctx.ServerError()s + ctx.NotFound("annex.Content", err) + return err + } + defer content.Close() + common.ServeContentByReadSeeker(ctx.Base, ctx.Repo.TreePath, lastModified, content) + return nil + } + return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified) } diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index d8e1d49904..efdd65d617 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -7,7 +7,9 @@ package integration import ( "errors" "fmt" + "io" "math/rand" + "net/http" "net/url" "os" "path" @@ -57,6 +59,63 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexMedia(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) + }) +} + +func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { + // Make sure that downloading via /media on the website recognizes it should give the annexed content + + // TODO: + // - [ ] roll this into TestGitAnnexPermissions to ensure that permission enforcement works correctly even on /media? + + session := loginUser(t, ctx.Username) // logs in to the http:// site/API, storing a cookie; + // this is a different auth method than the git+ssh:// or git+http:// protocols TestGitAnnexPermissions uses! + + // compute server-side path of the annexed file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + remoteObjectPath, err := contentLocation(remoteRepoPath, file) + require.NoError(t, err) + + // download annexed file + localObjectPath := path.Join(t.TempDir(), file) + fd, err := os.OpenFile(localObjectPath, os.O_CREATE|os.O_WRONLY, 0o777) + defer fd.Close() + require.NoError(t, err) + + mediaLink := path.Join("/", ctx.Username, ctx.Reponame, "/media/branch/master", file) + req := NewRequest(t, "GET", mediaLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + _, err = io.Copy(fd, resp.Body) + require.NoError(t, err) + fd.Close() + + // verify the download + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + require.NoError(t, err) + require.True(t, match, "Annexed files should be the same") +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -766,16 +825,16 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { } // - method 1: 'git annex whereis'. - // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + // Demonstrates that git-annex understands annexed files can be found in the remote annex. + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "annexed.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { - return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) + return fmt.Errorf("Couldn't `git annex whereis`: %w", err) } // Note: this regex is unanchored because 'whereis' outputs multiple lines containing // headers and 1+ remotes and we just want to find one of them. match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) if !match { - return errors.New("'git annex whereis' should report large.bin is known to be in [origin]") + return errors.New("'git annex whereis' should report files are known to be in [origin]") } return nil @@ -791,27 +850,56 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } - // verify the file was downloaded - localObjectPath, err := contentLocation(repoPath, "large.bin") - if err != nil { - return err - } - // localObjectPath := path.Join(repoPath, "large.bin") // or, just compare against the checked-out file + // verify the files downloaded - remoteObjectPath, err := contentLocation(remoteRepoPath, "large.bin") + cmp := func(filename string) error { + localObjectPath, err := contentLocation(repoPath, filename) + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, filename) // or, just compare against the checked-out file + + remoteObjectPath, err := contentLocation(remoteRepoPath, filename) + if err != nil { + return err + } + + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil + } + + // this is the annex-symlink file + stat, err := os.Lstat(path.Join(repoPath, "annexed.tiff")) if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + if !((stat.Mode() & os.ModeSymlink) != 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.tiff should be a symlink") + } + if err = cmp("annexed.tiff"); err != nil { return err } - match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) + // this is the annex-pointer file + stat, err = os.Lstat(path.Join(repoPath, "annexed.bin")) if err != nil { - return err + return fmt.Errorf("Lstat: %w", err) } - if !match { - return errors.New("Annexed files should be the same") + if !((stat.Mode() & os.ModeSymlink) == 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.bin should not be a symlink") } + err = cmp("annexed.bin") - return nil + return err } func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { @@ -956,16 +1044,36 @@ func doInitAnnexRepository(repoPath string) error { return err } - // add a file to the annex - err = generateRandomFile(1024*1024/4, path.Join(repoPath, "large.bin")) + // add files to the annex, stored via annex symlinks + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.tiff")) if err != nil { return err } + + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add files to the annex, stored via git-annex-smudge + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.bin")) + if err != nil { + return err + } + + if err != nil { + return err + } + err = git.AddChanges(repoPath, false, ".") if err != nil { return err } - err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex a file"}) + + // save everything + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"}) if err != nil { return err } From db5eaa1a240f28b20706c4182b23c652c7ca98c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 13:21:20 +0200 Subject: [PATCH 034/125] Adapt patch to upstream changes Test with different objectFormats. --- tests/integration/git_annex_test.go | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index efdd65d617..34dceefb9b 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -65,19 +65,21 @@ func TestGitAnnexMedia(t *testing.T) { } onGiteaRun(t, func(t *testing.T, u *url.URL) { - ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-media-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) - // the filenames here correspond to specific cases defined in doInitAnnexRepository() - t.Run("AnnexSymlink", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doAnnexMediaTest(t, ctx, "annexed.tiff") - }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doAnnexMediaTest(t, ctx, "annexed.bin") + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) }) }) } From ce1660d41bc620957f7365d7d6f290df95eccb4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 14 May 2024 14:34:44 +0200 Subject: [PATCH 035/125] Adapt patch to upstream changes Use tests.FileCmp instead of util.FileCmp. --- tests/integration/git_annex_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 34dceefb9b..b3dc2b974d 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -113,7 +113,7 @@ func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { fd.Close() // verify the download - match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + match, err := tests.FileCmp(localObjectPath, remoteObjectPath, 0) require.NoError(t, err) require.True(t, match, "Annexed files should be the same") } From 9ffebb2f84e2c82d59e55bae4e6f8bec7227c42e Mon Sep 17 00:00:00 2001 From: Nick Date: Sun, 27 Nov 2022 02:13:46 -0500 Subject: [PATCH 036/125] git-annex: views for annex files This updates the repo index/file view endpoints so annex files match the way LFS files are rendered, making annexed files accessible via the web instead of being black boxes only accessible by git clone. This mostly just duplicates the existing LFS logic. It doesn't try to combine itself with the existing logic, to make merging with upstream easier. If upstream ever decides to accept, I would like to try to merge the redundant logic. The one bit that doesn't directly copy LFS is my choice to hide annex-symlinks. LFS files are always _pointer files_ and therefore always render with the "file" icon and no special label, but annex files come in two flavours: symlinks or pointer files. I've conflated both kinds to try to give a consistent experience. The tests in here ensure the correct download link (/media, from the last PR) renders in both the toolbar and, if a binary file (like most annexed files will be), in the main pane, but it also adds quite a bit of code to make sure text files that happen to be annexed are dug out and rendered inline like LFS files are. --- modules/base/tool.go | 7 ++ options/locale/locale_cs-CZ.ini | 2 + options/locale/locale_de-DE.ini | 2 + options/locale/locale_el-GR.ini | 2 + options/locale/locale_en-US.ini | 2 + options/locale/locale_es-ES.ini | 2 + options/locale/locale_fa-IR.ini | 2 + options/locale/locale_fr-FR.ini | 2 + options/locale/locale_hu-HU.ini | 2 + options/locale/locale_id-ID.ini | 2 + options/locale/locale_is-IS.ini | 1 + options/locale/locale_it-IT.ini | 2 + options/locale/locale_ja-JP.ini | 2 + options/locale/locale_ko-KR.ini | 1 + options/locale/locale_lv-LV.ini | 4 +- options/locale/locale_nl-NL.ini | 2 + options/locale/locale_pl-PL.ini | 2 + options/locale/locale_pt-BR.ini | 2 + options/locale/locale_pt-PT.ini | 2 + options/locale/locale_ru-RU.ini | 2 + options/locale/locale_si-LK.ini | 2 + options/locale/locale_sk-SK.ini | 1 + options/locale/locale_sv-SE.ini | 2 + options/locale/locale_tr-TR.ini | 2 + options/locale/locale_uk-UA.ini | 2 + options/locale/locale_zh-CN.ini | 2 + options/locale/locale_zh-HK.ini | 3 +- options/locale/locale_zh-TW.ini | 2 + routers/web/repo/view.go | 66 +++++++++++--- templates/repo/file_info.tmpl | 1 + tests/integration/git_annex_test.go | 133 ++++++++++++++++++++++++++++ 31 files changed, 248 insertions(+), 13 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index 02f1db59d3..3d0fe73823 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -16,6 +16,7 @@ import ( "strings" "unicode/utf8" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -103,6 +104,12 @@ func Int64sToStrings(ints []int64) []string { func EntryIcon(entry *git.TreeEntry) string { switch { case entry.IsLink(): + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // git-annex files are sometimes stored as symlinks; + // short-circuit that so like LFS they are displayed as regular files + return "file" + } te, _, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/options/locale/locale_cs-CZ.ini b/options/locale/locale_cs-CZ.ini index 7d8fdee165..e050e0db12 100644 --- a/options/locale/locale_cs-CZ.ini +++ b/options/locale/locale_cs-CZ.ini @@ -1315,6 +1315,7 @@ view_git_blame=Zobrazit git blame video_not_supported_in_browser=Váš prohlížeč nepodporuje značku HTML5 „video“. audio_not_supported_in_browser=Váš prohlížeč nepodporuje značku HTML5 „audio“. stored_lfs=Uloženo pomocí Git LFS +stored_annex=Uloženo pomocí Git Annex symbolic_link=Symbolický odkaz executable_file=Spustitelný soubor vendored = Vendorováno @@ -1340,6 +1341,7 @@ editor.upload_file=Nahrát soubor editor.edit_file=Upravit soubor editor.preview_changes=Náhled změn editor.cannot_edit_lfs_files=LFS soubory nemohou být upravovány přes webové rozhraní. +editor.cannot_edit_annex_files=Annex soubory nemohou být upravovány přes webové rozhraní. editor.cannot_edit_non_text_files=Binární soubory nemohou být upravovány přes webové rozhraní. editor.edit_this_file=Upravit soubor editor.this_file_locked=Soubor je uzamčen diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 6a479b7629..7a28d08e49 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1315,6 +1315,7 @@ view_git_blame=„git blame“ ansehen video_not_supported_in_browser=Dein Browser unterstützt das HTML5-„video“-Tag nicht. audio_not_supported_in_browser=Dein Browser unterstützt das HTML5-„audio“-Tag nicht. stored_lfs=Gespeichert mit Git LFS +stored_annex=Gespeichert mit Git Annex symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit-Graph @@ -1338,6 +1339,7 @@ editor.upload_file=Datei hochladen editor.edit_file=Datei bearbeiten editor.preview_changes=Vorschau der Änderungen editor.cannot_edit_lfs_files=LFS-Dateien können im Webinterface nicht bearbeitet werden. +editor.cannot_edit_annex_files=Annex-Dateien können im Webinterface nicht bearbeitet werden. editor.cannot_edit_non_text_files=Binärdateien können nicht im Webinterface bearbeitet werden. editor.edit_this_file=Datei bearbeiten editor.this_file_locked=Datei ist gesperrt diff --git a/options/locale/locale_el-GR.ini b/options/locale/locale_el-GR.ini index cb99563034..3cbf24c3ff 100644 --- a/options/locale/locale_el-GR.ini +++ b/options/locale/locale_el-GR.ini @@ -1314,6 +1314,7 @@ view_git_blame=Προβολή git blame video_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 «video». audio_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 «audio». stored_lfs=Αποθηκεύτηκε με το Git LFS +stored_annex=Αποθηκεύτηκε με το Git Annex symbolic_link=Symbolic link executable_file=Εκτελέσιμο αρχείο commit_graph=Γράφημα υποβολών @@ -1337,6 +1338,7 @@ editor.upload_file=Ανέβασμα αρχείου editor.edit_file=Επεξεργασία αρχείου editor.preview_changes=Προεπισκόπηση αλλαγών editor.cannot_edit_lfs_files=Τα αρχεία LFS δεν μπορούν να επεξεργαστούν στη διεπαφή web. +editor.cannot_edit_annex_files=Τα αρχεία Annex δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.cannot_edit_non_text_files=Τα δυαδικά αρχεία δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.edit_this_file=Επεξεργασία αρχείου editor.this_file_locked=Το αρχείο είναι κλειδωμένο diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d2f47adab2..93c1eaa770 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1336,6 +1336,7 @@ view_git_blame = View git blame video_not_supported_in_browser = Your browser does not support the HTML5 "video" tag. audio_not_supported_in_browser = Your browser does not support the HTML5 "audio" tag. stored_lfs = Stored with Git LFS +stored_annex = Stored with Git Annex symbolic_link = Symbolic link executable_file = Executable file vendored = Vendored @@ -1363,6 +1364,7 @@ editor.upload_file = Upload file editor.edit_file = Edit file editor.preview_changes = Preview changes editor.cannot_edit_lfs_files = LFS files cannot be edited in the web interface. +editor.cannot_edit_annex_files = Annex files cannot be edited in the web interface. editor.cannot_edit_non_text_files = Binary files cannot be edited in the web interface. editor.edit_this_file = Edit file editor.this_file_locked = File is locked diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 4f9d141423..b55ff3181c 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -1310,6 +1310,7 @@ view_git_blame=Ver Git blame video_not_supported_in_browser=Su navegador no soporta el tag "video" de HTML5. audio_not_supported_in_browser=Su navegador no soporta el tag "audio" de HTML5. stored_lfs=Almacenados con Git LFS +stored_annex=Almacenados con Git Annex symbolic_link=Enlace simbólico executable_file=Archivo ejecutable commit_graph=Gráfico de commits @@ -1333,6 +1334,7 @@ editor.upload_file=Subir archivo editor.edit_file=Editar archivo editor.preview_changes=Vista previa de los cambios editor.cannot_edit_lfs_files=Los archivos LFS no se pueden editar en la interfaz web. +editor.cannot_edit_annex_files=Los archivos Annex no se pueden editar en la interfaz web. editor.cannot_edit_non_text_files=Los archivos binarios no se pueden editar en la interfaz web. editor.edit_this_file=Editar archivo editor.this_file_locked=El archivo está bloqueado diff --git a/options/locale/locale_fa-IR.ini b/options/locale/locale_fa-IR.ini index 8960be8bf4..3d5be270a4 100644 --- a/options/locale/locale_fa-IR.ini +++ b/options/locale/locale_fa-IR.ini @@ -951,6 +951,7 @@ file_copy_permalink=پرمالینک را کپی کنید video_not_supported_in_browser=مرورگر شما از تگ video که در HTML5 تعریف شده است، پشتیبانی نمی کند. audio_not_supported_in_browser=مرورگر شما از تگ audio که در HTML5 تعریف شده است، پشتیبانی نمی کند. stored_lfs=ذخیره شده با GIT LFS +stored_annex=ذخیره شده با GIT Annex symbolic_link=پیوند نمادین commit_graph=نمودار کامیت commit_graph.select=انتخاب برنچها @@ -968,6 +969,7 @@ editor.upload_file=بارگذاری پرونده editor.edit_file=ویرایش پرونده editor.preview_changes=پیش نمایش تغییرات editor.cannot_edit_lfs_files=پرونده های LFS در صحفه وب قابل تغییر نیست. +editor.cannot_edit_annex_files=پرونده های Annex در صحفه وب قابل تغییر نیست. editor.cannot_edit_non_text_files=پرونده‎های دودویی در صفحه وب قابل تغییر نیست. editor.edit_this_file=ویرایش پرونده editor.this_file_locked=پرونده قفل شده است diff --git a/options/locale/locale_fr-FR.ini b/options/locale/locale_fr-FR.ini index 2585eb1616..06d594594a 100644 --- a/options/locale/locale_fr-FR.ini +++ b/options/locale/locale_fr-FR.ini @@ -1316,6 +1316,7 @@ view_git_blame=Voir Git blame video_not_supported_in_browser=Votre navigateur ne supporte pas la balise « vidéo » HTML5. audio_not_supported_in_browser=Votre navigateur ne supporte pas la balise « audio » HTML5. stored_lfs=Stocké avec Git LFS +stored_annex=Stocké avec Git Annex symbolic_link=Lien symbolique executable_file=Fichier exécutable vendored = Vendored @@ -1341,6 +1342,7 @@ editor.upload_file=Téléverser un fichier editor.edit_file=Modifier le fichier editor.preview_changes=Aperçu des modifications editor.cannot_edit_lfs_files=Les fichiers LFS ne peuvent pas être modifiés dans l'interface web. +editor.cannot_edit_annex_files=Les fichiers Annex ne peuvent pas être modifiés dans l'interface web. editor.cannot_edit_non_text_files=Les fichiers binaires ne peuvent pas être édités dans l'interface web. editor.edit_this_file=Modifier le fichier editor.this_file_locked=Le fichier est verrouillé diff --git a/options/locale/locale_hu-HU.ini b/options/locale/locale_hu-HU.ini index 57555b90a2..ef94d2de0c 100644 --- a/options/locale/locale_hu-HU.ini +++ b/options/locale/locale_hu-HU.ini @@ -787,6 +787,7 @@ file_too_large=Ez a fájl túl nagy ahhoz, hogy megjelenítsük. video_not_supported_in_browser=A böngésző nem támogatja a HTML5 video tag-et. audio_not_supported_in_browser=A böngésző nem támogatja a HTML5 audio tag-et. stored_lfs=Git LFS-el eltárolva +stored_annex=Git Annex-el eltárolva symbolic_link=Szimbolikus hivatkozás commit_graph=Commit gráf commit_graph.hide_pr_refs=Pull request-ek elrejtése @@ -799,6 +800,7 @@ editor.upload_file=Fájl feltöltése editor.edit_file=Fájl szerkesztése editor.preview_changes=Változások előnézete editor.cannot_edit_lfs_files=LFS fájlok nem szerkeszthetőek a webes felületen. +editor.cannot_edit_annex_files=Annex fájlok nem szerkeszthetőek a webes felületen. editor.cannot_edit_non_text_files=Bináris fájlok nem szerkeszthetőek a webes felületen. editor.edit_this_file=Fájl szerkesztése editor.this_file_locked=Zárolt állomány diff --git a/options/locale/locale_id-ID.ini b/options/locale/locale_id-ID.ini index 1e0044e4ce..8cf3457aae 100644 --- a/options/locale/locale_id-ID.ini +++ b/options/locale/locale_id-ID.ini @@ -596,6 +596,7 @@ file_permalink=Permalink file_too_large=Berkas terlalu besar untuk ditampilkan. stored_lfs=Tersimpan dengan GIT LFS +stored_annex=Tersimpan dengan GIT Annex commit_graph=Grafik Komit blame=Salahkan normal_view=Pandangan Normal @@ -607,6 +608,7 @@ editor.upload_file=Unggah Berkas editor.edit_file=Sunting Berkas editor.preview_changes=Tinjau Perubahan editor.cannot_edit_lfs_files=Berkas LFS tidak dapat disunting dalam antarmuka web. +editor.cannot_edit_annex_files=Berkas Annex tidak dapat disunting dalam antarmuka web. editor.cannot_edit_non_text_files=Berkas biner tidak dapat disunting dalam antarmuka web. editor.edit_this_file=Sunting Berkas editor.this_file_locked=Berkas terkunci diff --git a/options/locale/locale_is-IS.ini b/options/locale/locale_is-IS.ini index 3a6e844de2..b3cf17fccc 100644 --- a/options/locale/locale_is-IS.ini +++ b/options/locale/locale_is-IS.ini @@ -680,6 +680,7 @@ file_view_rendered=Skoða Unnið file_copy_permalink=Afrita Varanlega Slóð stored_lfs=Geymt með Git LFS +stored_annex=Geymt með Git Annex commit_graph.hide_pr_refs=Fela Sameiningarbeiðnir commit_graph.monochrome=Einlitað commit_graph.color=Litað diff --git a/options/locale/locale_it-IT.ini b/options/locale/locale_it-IT.ini index 2f5bf6f0c1..1e4aeab768 100644 --- a/options/locale/locale_it-IT.ini +++ b/options/locale/locale_it-IT.ini @@ -1267,6 +1267,7 @@ view_git_blame=Visualizza git incolpa video_not_supported_in_browser=Il tuo browser non supporta le etichette "video" di HTML5. audio_not_supported_in_browser=Il tuo browser non supporta le etichette "audio" di HTML5. stored_lfs=Memorizzati con Git LFS +stored_annex=Memorizzati con Git Annex symbolic_link=Link Simbolico commit_graph=Grafico dei commit commit_graph.select=Seleziona rami @@ -1285,6 +1286,7 @@ editor.upload_file=Carica file editor.edit_file=Modifica file editor.preview_changes=Anteprima modifiche editor.cannot_edit_lfs_files=I file LFS non possono essere modificati nell'interfaccia web. +editor.cannot_edit_annex_files=I file Annex non possono essere modificati nell'interfaccia web. editor.cannot_edit_non_text_files=I file binari non possono essere modificati tramite interfaccia web. editor.edit_this_file=Modifica file editor.this_file_locked=Il file è bloccato diff --git a/options/locale/locale_ja-JP.ini b/options/locale/locale_ja-JP.ini index b0fc38d911..4ea433106e 100644 --- a/options/locale/locale_ja-JP.ini +++ b/options/locale/locale_ja-JP.ini @@ -1305,6 +1305,7 @@ view_git_blame=Git Blameを表示 video_not_supported_in_browser=このブラウザはHTML5のvideoタグをサポートしていません。 audio_not_supported_in_browser=このブラウザーはHTML5のaudioタグをサポートしていません。 stored_lfs=Git LFSで保管されています +stored_annex=Git Annexで保管されています symbolic_link=シンボリック リンク executable_file=実行ファイル commit_graph=コミットグラフ @@ -1328,6 +1329,7 @@ editor.upload_file=ファイルをアップロード editor.edit_file=ファイルを編集 editor.preview_changes=変更をプレビュー editor.cannot_edit_lfs_files=LFSのファイルはWebインターフェースで編集できません。 +editor.cannot_edit_annex_files=AnnexのファイルはWebインターフェースで編集できません。 editor.cannot_edit_non_text_files=バイナリファイルはWebインターフェースで編集できません。 editor.edit_this_file=ファイルを編集 editor.this_file_locked=ファイルはロックされています diff --git a/options/locale/locale_ko-KR.ini b/options/locale/locale_ko-KR.ini index a2a50d7440..72b4ed3588 100644 --- a/options/locale/locale_ko-KR.ini +++ b/options/locale/locale_ko-KR.ini @@ -821,6 +821,7 @@ file_too_large=보여주기에는 파일이 너무 큽니다. video_not_supported_in_browser=당신의 브라우저가 HTML5의 "video" 태그를 지원하지 않습니다. audio_not_supported_in_browser=당신의 브라우저가 HTML5의 "audio" 태그를 지원하지 않습니다. stored_lfs=Git LFS에 저장되어 있습니다 +stored_annex=Git Annex에 저장되어 있습니다 commit_graph=커밋 그래프 editor.new_file=새 파일 diff --git a/options/locale/locale_lv-LV.ini b/options/locale/locale_lv-LV.ini index c179158213..96108677fd 100644 --- a/options/locale/locale_lv-LV.ini +++ b/options/locale/locale_lv-LV.ini @@ -1314,6 +1314,7 @@ view_git_blame=Apskatīt Git izmaiņu veicējus video_not_supported_in_browser=Pārlūks neatbalsta HTML5 tagu "video". audio_not_supported_in_browser=Pārlūks neatbalsta HTML5 tagu "audio". stored_lfs=Saglabāts Git LFS +stored_annex=Saglabāts Git Annex symbolic_link=Simboliska saite executable_file=Izpildāma datne commit_graph=Iesūtījumu karte @@ -1337,6 +1338,7 @@ editor.upload_file=Augšupielādēt datni editor.edit_file=Labot datni editor.preview_changes=Priekšskatīt izmaiņas editor.cannot_edit_lfs_files=LFS datnes tīmekļa saskarnē nevar labot. +editor.cannot_edit_annex_files=Annex datnes tīmekļa saskarnē nevar labot. editor.cannot_edit_non_text_files=Binārās datnes tīmekļa saskarnē nevar labot. editor.edit_this_file=Labot datni editor.this_file_locked=Datne ir slēgta @@ -4023,4 +4025,4 @@ filepreview.lines = %[1]d. līdz %[2]d. rinda %[3]s filepreview.truncated = Priekšskatījums tika saīsināts [translation_meta] -test = Šī ir pārbaudes virkne. Tā netiek attēlota Forgejo saskarnē, bet tiek izmantota pārbaudes nolūkiem. Droši var ievadīt "ok", lai ietaupītu laiku (vai kādu jautru faktu pēc izvēles), lai sasniegtu to saldo 100% pabeigšanas atzīmi. \ No newline at end of file +test = Šī ir pārbaudes virkne. Tā netiek attēlota Forgejo saskarnē, bet tiek izmantota pārbaudes nolūkiem. Droši var ievadīt "ok", lai ietaupītu laiku (vai kādu jautru faktu pēc izvēles), lai sasniegtu to saldo 100% pabeigšanas atzīmi. diff --git a/options/locale/locale_nl-NL.ini b/options/locale/locale_nl-NL.ini index c884160c56..3f89d43053 100644 --- a/options/locale/locale_nl-NL.ini +++ b/options/locale/locale_nl-NL.ini @@ -1283,6 +1283,7 @@ view_git_blame=Bekijk git blame video_not_supported_in_browser=Uw browser ondersteunt de HTML5 "video" element niet. audio_not_supported_in_browser=Uw browser ondersteunt de HTML5 "audio" element niet. stored_lfs=Opgeslagen met Git LFS +stored_annex=Opgeslagen met Git Annex symbolic_link=Symbolische link commit_graph=Commit grafiek commit_graph.select=Selecteer branches @@ -1301,6 +1302,7 @@ editor.upload_file=Upload bestand editor.edit_file=Bewerk bestand editor.preview_changes=Voorbeeld tonen editor.cannot_edit_lfs_files=LFS-bestanden kunnen niet worden bewerkt in de webinterface. +editor.cannot_edit_annex_files=Annex-bestanden kunnen niet worden bewerkt in de webinterface. editor.cannot_edit_non_text_files=Binaire bestanden kunnen niet worden bewerkt in de webinterface. editor.edit_this_file=Bewerk bestand editor.this_file_locked=Bestand is vergrendeld diff --git a/options/locale/locale_pl-PL.ini b/options/locale/locale_pl-PL.ini index 2af7648f8d..51bdca6464 100644 --- a/options/locale/locale_pl-PL.ini +++ b/options/locale/locale_pl-PL.ini @@ -1148,6 +1148,7 @@ file_copy_permalink=Kopiuj bezpośredni odnośnik video_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "video". audio_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "audio". stored_lfs=Przechowane za pomocą Git LFS +stored_annex=Przechowane za pomocą Git Annex symbolic_link=Dowiązanie symboliczne commit_graph=Wykres commitów commit_graph.select=Wybierz gałęzie @@ -1165,6 +1166,7 @@ editor.upload_file=Wyślij plik editor.edit_file=Edytuj plik editor.preview_changes=Podgląd zmian editor.cannot_edit_lfs_files=Pliki LFS nie mogą być edytowane poprzez interfejs przeglądarkowy. +editor.cannot_edit_annex_files=Pliki Annex nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.cannot_edit_non_text_files=Pliki binarne nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.edit_this_file=Edytuj plik editor.this_file_locked=Plik jest zablokowany diff --git a/options/locale/locale_pt-BR.ini b/options/locale/locale_pt-BR.ini index d826e60417..d85e00234b 100644 --- a/options/locale/locale_pt-BR.ini +++ b/options/locale/locale_pt-BR.ini @@ -1308,6 +1308,7 @@ view_git_blame=Ver git blame video_not_supported_in_browser=Seu navegador não tem suporte para a tag "video" do HTML5. audio_not_supported_in_browser=Seu navegador não tem suporte para a tag "audio" do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Link simbólico executable_file=Arquivo executável commit_graph=Gráfico de commits @@ -1331,6 +1332,7 @@ editor.upload_file=Enviar arquivo editor.edit_file=Editar arquivo editor.preview_changes=Pré-visualizar alterações editor.cannot_edit_lfs_files=Arquivos LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Arquivos Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Arquivos binários não podem ser editados na interface web. editor.edit_this_file=Editar arquivo editor.this_file_locked=Arquivo está bloqueado diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 309fdbe54d..23158b8215 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -1319,6 +1319,7 @@ view_git_blame=Ver git blame video_not_supported_in_browser=O seu navegador não suporta a etiqueta "video" do HTML5. audio_not_supported_in_browser=O seu navegador não suporta a etiqueta "audio" do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Ligação simbólica executable_file=Ficheiro executável vendored=Externo @@ -1344,6 +1345,7 @@ editor.upload_file=Carregar ficheiro editor.edit_file=Editar ficheiro editor.preview_changes=Pré-visualizar modificações editor.cannot_edit_lfs_files=Ficheiros LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Ficheiros Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Ficheiros binários não podem ser editados na interface da web. editor.edit_this_file=Editar ficheiro editor.this_file_locked=Ficheiro bloqueado diff --git a/options/locale/locale_ru-RU.ini b/options/locale/locale_ru-RU.ini index ec844faea7..18c30e4dc7 100644 --- a/options/locale/locale_ru-RU.ini +++ b/options/locale/locale_ru-RU.ini @@ -1302,6 +1302,7 @@ view_git_blame=Показать git blame video_not_supported_in_browser=Ваш браузер не поддерживает тэг HTML5 «video». audio_not_supported_in_browser=Ваш браузер не поддерживает тэг HTML5 «audio». stored_lfs=Хранится Git LFS +stored_annex=Хранится Git Annex symbolic_link=Символическая ссылка executable_file=Исполняемый файл commit_graph=Граф коммитов @@ -1325,6 +1326,7 @@ editor.upload_file=Загрузить файл editor.edit_file=Редактировать файл editor.preview_changes=Просмотр изменений editor.cannot_edit_lfs_files=LFS файлы невозможно редактировать в веб-интерфейсе. +editor.cannot_edit_annex_files=Annex файлы невозможно редактировать в веб-интерфейсе. editor.cannot_edit_non_text_files=Двоичные файлы нельзя редактировать в веб-интерфейсе. editor.edit_this_file=Редактировать файл editor.this_file_locked=Файл заблокирован diff --git a/options/locale/locale_si-LK.ini b/options/locale/locale_si-LK.ini index ac7627ca80..63c728c48b 100644 --- a/options/locale/locale_si-LK.ini +++ b/options/locale/locale_si-LK.ini @@ -889,6 +889,7 @@ file_copy_permalink=පිටපත් මාමලින්ක් video_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'වීඩියෝ' ටැගය සඳහා සහය නොදක්වයි. audio_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'ශ්රව්ය' ටැගය සඳහා සහය නොදක්වයි. stored_lfs=Git LFS සමඟ ගබඩා +stored_annex=Git Annex සමඟ ගබඩා symbolic_link=සංකේතාත්මක සබැඳිය commit_graph=ප්රස්තාරය කැප commit_graph.select=ශාඛා තෝරන්න @@ -906,6 +907,7 @@ editor.upload_file=ගොනුව උඩුගත කරන්න editor.edit_file=ගොනුව සංස්කරණය editor.preview_changes=වෙනස්කම් පෙරදසුන editor.cannot_edit_lfs_files=LFS ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. +editor.cannot_edit_annex_files=Annex ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.cannot_edit_non_text_files=ද්විමය ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.edit_this_file=ගොනුව සංස්කරණය editor.this_file_locked=ගොනුවට අගුළු ලා ඇත diff --git a/options/locale/locale_sk-SK.ini b/options/locale/locale_sk-SK.ini index bd2ce203dd..cfb3d76a38 100644 --- a/options/locale/locale_sk-SK.ini +++ b/options/locale/locale_sk-SK.ini @@ -1010,6 +1010,7 @@ view_git_blame=Zobraziť Git Blame video_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'video'. audio_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'audio'. stored_lfs=Uložené pomocou Git LFS +stored_annex=Uložené pomocou Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitov line=riadok diff --git a/options/locale/locale_sv-SE.ini b/options/locale/locale_sv-SE.ini index 928d4a3f1e..7358e5a47e 100644 --- a/options/locale/locale_sv-SE.ini +++ b/options/locale/locale_sv-SE.ini @@ -891,6 +891,7 @@ file_too_large=Filen är för stor för att visas. video_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen "video". audio_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen "audio". stored_lfs=Sparad med Git LFS +stored_annex=Sparad med Git Annex symbolic_link=Symbolisk länk commit_graph=Commitgraf commit_graph.monochrome=Mono @@ -904,6 +905,7 @@ editor.upload_file=Ladda upp fil editor.edit_file=Redigera fil editor.preview_changes=Förhandsgranska ändringar editor.cannot_edit_lfs_files=LFS-filer kan inte redigeras i webbgränssnittet. +editor.cannot_edit_annex_files=Annex-filer kan inte redigeras i webbgränssnittet. editor.cannot_edit_non_text_files=Binära filer kan inte redigeras genom webbgränssnittet. editor.edit_this_file=Redigera fil editor.this_file_locked=Filen är låst diff --git a/options/locale/locale_tr-TR.ini b/options/locale/locale_tr-TR.ini index 82476d4ae8..8890f6df95 100644 --- a/options/locale/locale_tr-TR.ini +++ b/options/locale/locale_tr-TR.ini @@ -1290,6 +1290,7 @@ view_git_blame=Git Suç Görüntüle video_not_supported_in_browser=Tarayıcınız HTML5 'video' etiketini desteklemiyor. audio_not_supported_in_browser=Tarayıcınız HTML5 'audio' etiketini desteklemiyor. stored_lfs=Git LFS ile depolandı +stored_annex=Git Annex ile depolandı symbolic_link=Sembolik Bağlantı executable_file=Çalıştırılabilir Dosya commit_graph=İşleme Grafiği @@ -1313,6 +1314,7 @@ editor.upload_file=Dosya Yükle editor.edit_file=Dosyayı Düzenle editor.preview_changes=Değişiklikleri Önizle editor.cannot_edit_lfs_files=LFS dosyaları web arayüzünde düzenlenemez. +editor.cannot_edit_annex_files=Annex dosyaları web arayüzünde düzenlenemez. editor.cannot_edit_non_text_files=Bu tür dosyalar web arayüzünden düzenlenemez. editor.edit_this_file=Dosyayı Düzenle editor.this_file_locked=Dosya kilitlendi diff --git a/options/locale/locale_uk-UA.ini b/options/locale/locale_uk-UA.ini index 2186f44719..c92ac9dcc7 100644 --- a/options/locale/locale_uk-UA.ini +++ b/options/locale/locale_uk-UA.ini @@ -1218,6 +1218,7 @@ file_copy_permalink=Копіювати постійне посилання video_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 «video». audio_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 «audio». stored_lfs=Збережено з Git LFS +stored_annex=Збережено з Git Annex symbolic_link=Символічне посилання commit_graph=Графік комітів commit_graph.select=Виберіть гілки @@ -1235,6 +1236,7 @@ editor.upload_file=Завантажити файл editor.edit_file=Редагувати файл editor.preview_changes=Попередній перегляд змін editor.cannot_edit_lfs_files=Файли LFS не можна редагувати в веб-інтерфейсі. +editor.cannot_edit_annex_files=Файли Annex не можна редагувати в веб-інтерфейсі. editor.cannot_edit_non_text_files=Бінарні файли не можливо редагувати у веб-інтерфейсі. editor.edit_this_file=Редагувати файл editor.this_file_locked=Файл заблоковано diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 7354a4d02b..62021be9fe 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1317,6 +1317,7 @@ view_git_blame=查看 Git Blame video_not_supported_in_browser=您的浏览器不支持 HTML5 “video” 标签。 audio_not_supported_in_browser=您的浏览器不支持 HTML5 “audio” 标签。 stored_lfs=存储到Git LFS +stored_annex=存储到Git Annex symbolic_link=符号链接 executable_file=可执行文件 vendored = Vendored @@ -1342,6 +1343,7 @@ editor.upload_file=上传文件 editor.edit_file=编辑文件 editor.preview_changes=预览变更 editor.cannot_edit_lfs_files=无法在 web 界面中编辑 lfs 文件。 +editor.cannot_edit_annex_files=无法在 web 界面中编辑 lfs 文件。 editor.cannot_edit_non_text_files=网页不能编辑二进制文件。 editor.edit_this_file=编辑文件 editor.this_file_locked=文件已锁定 diff --git a/options/locale/locale_zh-HK.ini b/options/locale/locale_zh-HK.ini index e5080e65bb..25dfcdf913 100644 --- a/options/locale/locale_zh-HK.ini +++ b/options/locale/locale_zh-HK.ini @@ -472,6 +472,7 @@ file_view_raw=查看原始文件 file_permalink=永久連結 stored_lfs=儲存到到 Git LFS +stored_annex=儲存到到 Git Annex editor.preview_changes=預覽更改 editor.or=或 @@ -1132,4 +1133,4 @@ runners.labels = 標籤 [projects] -[git.filemode] \ No newline at end of file +[git.filemode] diff --git a/options/locale/locale_zh-TW.ini b/options/locale/locale_zh-TW.ini index 5caafacce1..3dcce0bedd 100644 --- a/options/locale/locale_zh-TW.ini +++ b/options/locale/locale_zh-TW.ini @@ -1263,6 +1263,7 @@ view_git_blame=檢視 Git Blame video_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「video」標籤。 audio_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「audio」標籤。 stored_lfs=已使用 Git LFS 儲存 +stored_annex=已使用 Git Annex 儲存 symbolic_link=符號連結 commit_graph=提交線圖 commit_graph.select=選擇分支 @@ -1282,6 +1283,7 @@ editor.upload_file=上傳檔案 editor.edit_file=編輯檔案 editor.preview_changes=預覽變更 editor.cannot_edit_lfs_files=無法在 web 介面中編輯 LFS 檔。 +editor.cannot_edit_annex_files=無法在 web 介面中編輯 Annex 檔。 editor.cannot_edit_non_text_files=網站介面不能編輯二進位檔案。 editor.edit_this_file=編輯檔案 editor.this_file_locked=檔案已被鎖定 diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index fd8c1da058..59efa4334f 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -34,6 +34,7 @@ import ( unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/git" @@ -209,14 +210,47 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + return nil, nil, nil, err + } + if isAnnexed { + // TODO: this code could be merged with the LFS case, especially the redundant type sniffer, + // but it is *currently* written this way to make merging with the non-annex upstream easier: + // this way, the git-annex patch is (mostly) pure additions. + + annexContent, err := annex.Content(blob) + if err != nil { + // in the case where annex content is missing, what should happen? + // do we render the page with an error message? + // actually that's not a bad idea, there's some sort of error message situation + // TODO: display an error to the user explaining that their data is missing + return nil, nil, nil, err + } + + stat, err := annexContent.Stat() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(annexContent, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + + return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + } + dataRc, err := blob.DataAsync() if err != nil { return nil, nil, nil, err @@ -231,18 +265,18 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil { // fallback to plain file log.Warn("Unable to access LFS pointer %s in repo %d: %v", pointer.Oid, repoID, err) - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -262,7 +296,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry) { @@ -447,10 +481,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - if fInfo.isLFSFile { + if fInfo.isLFSFile || fInfo.isAnnexFile { ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } + if fInfo.isAnnexFile { + // pre-git-annex v7, all annexed files were represented in-repo as symlinks; + // but we pretend they aren't, since that's a distracting quirk of git-annex + // and not a meaningful choice on the user's part + ctx.Data["FileIsSymlink"] = false + } + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. @@ -458,6 +499,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { isDisplayingRendered = true } ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -492,6 +534,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { // Assume file is not editable first. if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") + } else if fInfo.isAnnexFile { + ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_annex_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") } @@ -599,7 +643,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !fInfo.isLFSFile { + if !fInfo.isLFSFile && !fInfo.isAnnexFile { if ctx.Repo.CanEnableEditor(ctx, ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 6ae7c15a26..05d9825cfb 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -17,6 +17,7 @@ {{if .FileSize}}
{{ctx.Locale.TrSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}}
{{end}} {{if .LFSLock}} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index b3dc2b974d..bc52d2ae1d 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -118,6 +118,111 @@ func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { require.True(t, match, "Annexed files should be the same") } +func TestGitAnnexViews(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + session := loginUser(t, ctx.Username) + + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() + + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + }) + + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files + defer tests.PrintCurrentTest(t)() + + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink + } + + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link + defer tests.PrintCurrentTest(t)() + + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") + }) + }) + + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + }) + }) + }) +} + /* Test that permissions are enforced on git-annex-shell commands. @@ -1029,6 +1134,14 @@ func doInitAnnexRepository(repoPath string) error { if err != nil { return err } + _, err = f.WriteString("*.rst filter=annex\n") + if err != nil { + return err + } + _, err = f.WriteString("*.markdown filter=annex\n") + if err != nil { + return err + } f.Close() err = git.AddChanges(repoPath, false, ".") @@ -1053,6 +1166,18 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.md"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.txt"), []byte("We're going to see the wizard\nThe wonderful\nMonkey of\nBoz\n"), 0o777) + if err != nil { + return err + } + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err @@ -1065,6 +1190,14 @@ func doInitAnnexRepository(repoPath string) error { return err } + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.rst"), []byte("Title\n=====\n\n- this is to test annexing a text file\n- lists are fun\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.markdown"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) if err != nil { return err } From 4c12a2ecee0c496a49b658b08b80f90dfcb10a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 24 May 2024 13:21:42 +0200 Subject: [PATCH 037/125] Adapt patch to upstream changes Test with different objectFormats. --- tests/integration/git_annex_test.go | 148 ++++++++++++++-------------- 1 file changed, 75 insertions(+), 73 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index bc52d2ae1d..4d4a0da88e 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -124,98 +124,100 @@ func TestGitAnnexViews(t *testing.T) { } onGiteaRun(t, func(t *testing.T, u *url.URL) { - ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) - // create a public repo - require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) - session := loginUser(t, ctx.Username) + session := loginUser(t, ctx.Username) - t.Run("Index", func(t *testing.T) { - // test that annex symlinks renders with the _file icon_ on the main list - defer tests.PrintCurrentTest(t)() + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() - repoLink := path.Join("/", ctx.Username, ctx.Reponame) - req := NewRequest(t, "GET", repoLink) - resp := session.MakeRequest(t, req, http.StatusOK) - - htmlDoc := NewHTMLParser(t, resp.Body) - isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") - require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") - }) - - t.Run("View", func(t *testing.T) { - // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files - defer tests.PrintCurrentTest(t)() - - doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { - viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) - // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? - mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) - - req := NewRequest(t, "GET", viewLink) + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) resp := session.MakeRequest(t, req, http.StatusOK) - htmlDoc = NewHTMLParser(t, resp.Body) - // the first button on the toolbar on the view template is the "Raw" button - // this CSS selector is the most precise I can think to use - buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") - require.True(t, exists, "Download button should exist on the file header") - require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") - - return htmlDoc, viewLink, mediaLink - } - - t.Run("Binary", func(t *testing.T) { - // test that annexing a file renders the /media link in /src and NOT the /raw link - defer tests.PrintCurrentTest(t)() - - doBinaryViewTest := func(file string) { - htmlDoc, _, mediaLink := doViewTest(file) - - rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") - require.True(t, exists, "Download link should render instead of content because this is a binary file") - require.EqualValues(t, mediaLink, rawLink) - } - - t.Run("AnnexSymlink", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doBinaryViewTest("annexed.tiff") - }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doBinaryViewTest("annexed.bin") - }) + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") }) - t.Run("Text", func(t *testing.T) { + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files defer tests.PrintCurrentTest(t)() - doTextViewTest := func(file string) { - htmlDoc, _, _ := doViewTest(file) - require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink } - t.Run("AnnexSymlink", func(t *testing.T) { + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link defer tests.PrintCurrentTest(t)() - doTextViewTest("annexed.txt") - t.Run("Markdown", func(t *testing.T) { - // special case: check that markdown can be pulled out of the annex and rendered, too + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - htmlDoc, _, _ := doViewTest("annexed.md") - require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") }) }) - t.Run("AnnexPointer", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - doTextViewTest("annexed.rst") - t.Run("Markdown", func(t *testing.T) { - // special case: check that markdown can be pulled out of the annex and rendered, too + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - htmlDoc, _, _ := doViewTest("annexed.markdown") - require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) }) }) }) From a5e712f002eea4401b28f9c1141519133026c87b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 15 May 2024 10:55:12 +0200 Subject: [PATCH 038/125] Add git-annex to docker image --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index ae21a0821e..d39de78bf9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,7 @@ RUN apk --no-cache add \ sqlite \ su-exec \ gnupg \ + git-annex \ && rm -rf /var/cache/apk/* RUN addgroup \ From 83f8fdb142d18967a33fcfe49071c0d2bdb8b6cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Jul 2024 16:43:42 +0000 Subject: [PATCH 039/125] Error if git-annex is enabled but missing (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copied from https://github.com/neuropoly/gitea/pull/47 This adds a check so that if `setting.Annex.Enabled` is true and git-annex is not in the PATH Forgejo will abort on startup with a reasonable error message. Fixes #15. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/16 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- cmd/web.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/web.go b/cmd/web.go index a817204aa6..661e6d158e 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -9,6 +9,7 @@ import ( "net" "net/http" "os" + "os/exec" "path/filepath" "strconv" "strings" @@ -247,6 +248,12 @@ func runWeb(ctx *cli.Context) error { createPIDFile(ctx.String("pid")) } + if setting.Annex.Enabled { + if _, err := exec.LookPath("git-annex"); err != nil { + log.Fatal("You have enabled git-annex support but git-annex is not installed. Please make sure that Forgejo's PATH contains the git-annex executable.") + } + } + if !setting.InstallLock { if err := serveInstall(ctx); err != nil { return err From 5fdcb8a7a1b17e927eccac7e5de7508149226a9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Thu, 18 Jul 2024 18:18:06 +0000 Subject: [PATCH 040/125] Git-annex web uploads (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This implements support for uploading files into the annex using the web interface. If a repository is a git-annex-enabled repository all files will be added to it using git annex add. This means that the repository's configuration for what to put into the annex (annex.largefiles in gitattributes) will be respected. Plain git repositories without git-annex will work as before, directly uploading to git. Fixes #5. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/21 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 6 ++ modules/util/remove.go | 11 ++-- services/repository/files/temp_repo.go | 20 +++++++ services/repository/files/upload.go | 83 +++++++++++++++++++++++++- tests/integration/git_annex_test.go | 81 +++++++++++++++++++++++++ 5 files changed, 194 insertions(+), 7 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index bb049d77ed..bab5a7e0b2 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -152,3 +152,9 @@ func IsAnnexed(blob *git.Blob) (bool, error) { } return true, nil } + +// IsAnnexRepo determines if repo is a git-annex enabled repository +func IsAnnexRepo(repo *git.Repository) bool { + _, _, err := git.NewCommand(repo.Ctx, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repo.Path}) + return err == nil +} diff --git a/modules/util/remove.go b/modules/util/remove.go index f2a61ae467..39556e5e0b 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -53,10 +53,13 @@ func MakeWritable(name string) error { return err } - // 0200 == u+w, in octal unix permission notation - err = os.Chmod(path, info.Mode()|0o200) - if err != nil { - return err + // Don't try chmod'ing symlinks (will fail with broken symlinks) + if info.Mode()&os.ModeSymlink != os.ModeSymlink { + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } } } return nil diff --git a/services/repository/files/temp_repo.go b/services/repository/files/temp_repo.go index 6e7570b82c..566ae5ff8f 100644 --- a/services/repository/files/temp_repo.go +++ b/services/repository/files/temp_repo.go @@ -202,6 +202,26 @@ func (t *TemporaryUploadRepository) AddObjectToIndex(mode, objectHash, objectPat return nil } +// InitPrivateAnnex initializes a private annex in the repository +func (t *TemporaryUploadRepository) InitPrivateAnnex() error { + if _, _, err := git.NewCommand(t.ctx, "config", "annex.private", "true").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + if _, _, err := git.NewCommand(t.ctx, "annex", "init").RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + return nil +} + +// AddAnnex adds the file at path to the repository using git annex add +// This requires a non-bare repository +func (t *TemporaryUploadRepository) AddAnnex(path string) error { + if _, _, err := git.NewCommand(t.ctx, "annex", "add").AddDynamicArguments(path).RunStdString(&git.RunOpts{Dir: t.basePath}); err != nil { + return err + } + return nil +} + // WriteTree writes the current index as a tree to the object db and returns its hash func (t *TemporaryUploadRepository) WriteTree() (string, error) { stdout, _, err := git.NewCommand(t.ctx, "write-tree").RunStdString(&git.RunOpts{Dir: t.basePath}) diff --git a/services/repository/files/upload.go b/services/repository/files/upload.go index 1330116889..21cd5a8344 100644 --- a/services/repository/files/upload.go +++ b/services/repository/files/upload.go @@ -6,13 +6,16 @@ package files import ( "context" "fmt" + "io" "os" "path" + "path/filepath" "strings" git_model "code.gitea.io/gitea/models/git" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" @@ -89,7 +92,7 @@ func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use defer t.Close() hasOldBranch := true - if err = t.Clone(opts.OldBranch, true); err != nil { + if err = t.Clone(opts.OldBranch, false); err != nil { if !git.IsErrBranchNotExist(err) || !repo.IsEmpty { return err } @@ -105,10 +108,30 @@ func UploadRepoFiles(ctx context.Context, repo *repo_model.Repository, doer *use } } - // Copy uploaded files into repository. - if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + r, err := git.OpenRepository(ctx, repo.RepoPath()) + if err != nil { return err } + if annex.IsAnnexRepo(r) { + // Initialize annex privately in temporary clone + if err := t.InitPrivateAnnex(); err != nil { + return err + } + // Copy uploaded files into git-annex repository + if err := copyUploadedFilesIntoAnnexRepository(infos, t, opts.TreePath); err != nil { + return err + } + // Move all annexed content in the temporary repository, i.e. everything we have just added, to the origin + author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer) + if err := moveAnnexedFilesToOrigin(t, author, committer); err != nil { + return err + } + } else { + // Copy uploaded files into repository. + if err := copyUploadedLFSFilesIntoRepository(infos, t, opts.TreePath); err != nil { + return err + } + } // Now write the tree treeHash, err := t.WriteTree() @@ -246,3 +269,57 @@ func uploadToLFSContentStore(info uploadInfo, contentStore *lfs.ContentStore) er } return nil } + +func copyUploadedFilesIntoAnnexRepository(infos []uploadInfo, t *TemporaryUploadRepository, treePath string) error { + for i := range len(infos) { + if err := copyUploadedFileIntoAnnexRepository(&infos[i], t, treePath); err != nil { + return err + } + } + return nil +} + +func copyUploadedFileIntoAnnexRepository(info *uploadInfo, t *TemporaryUploadRepository, treePath string) error { + pathInRepo := path.Join(t.basePath, treePath, info.upload.Name) + if err := os.MkdirAll(filepath.Dir(pathInRepo), 0o700); err != nil { + return err + } + if err := os.Rename(info.upload.LocalPath(), pathInRepo); err != nil { + // Rename didn't work, try copy and remove + inputFile, err := os.Open(info.upload.LocalPath()) + if err != nil { + return fmt.Errorf("could not open source file: %v", err) + } + defer inputFile.Close() + outputFile, err := os.Create(pathInRepo) + if err != nil { + return fmt.Errorf("could not open dest file: %v", err) + } + defer outputFile.Close() + _, err = io.Copy(outputFile, inputFile) + if err != nil { + return fmt.Errorf("could not copy to dest from source: %v", err) + } + inputFile.Close() + err = os.Remove(info.upload.LocalPath()) + if err != nil { + return fmt.Errorf("could not remove source file: %v", err) + } + } + return t.AddAnnex(pathInRepo) +} + +func moveAnnexedFilesToOrigin(t *TemporaryUploadRepository, author, committer *user_model.User) error { + authorSig := author.NewGitSig() + committerSig := committer.NewGitSig() + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+authorSig.Name, + "GIT_AUTHOR_EMAIL="+authorSig.Email, + "GIT_COMMITTER_NAME="+committerSig.Name, + "GIT_COMMITTER_EMAIL="+committerSig.Email, + ) + if _, _, err := git.NewCommand(t.ctx, "annex", "move", "--to", "origin").RunStdString(&git.RunOpts{Dir: t.basePath, Env: env}); err != nil { + return err + } + return nil +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 4d4a0da88e..ff840c8b68 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -5,14 +5,17 @@ package integration import ( + "bytes" "errors" "fmt" "io" "math/rand" + "mime/multipart" "net/http" "net/url" "os" "path" + "path/filepath" "regexp" "strings" "testing" @@ -59,6 +62,84 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, return nil } +func TestGitAnnexWebUpload(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + forEachObjectFormat(t, func(t *testing.T, objectFormat git.ObjectFormat) { + ctx := NewAPITestContext(t, "user2", "annex-web-upload-test"+objectFormat.Name(), auth_model.AccessTokenScopeWriteRepository) + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false, objectFormat)) + + uploadFile := func(t *testing.T, path string) string { + t.Helper() + + body := &bytes.Buffer{} + mpForm := multipart.NewWriter(body) + err := mpForm.WriteField("_csrf", GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch)) + require.NoError(t, err) + + file, err := mpForm.CreateFormFile("file", filepath.Base(path)) + require.NoError(t, err) + + srcFile, err := os.Open(path) + require.NoError(t, err) + + io.Copy(file, srcFile) + require.NoError(t, mpForm.Close()) + + req := NewRequestWithBody(t, "POST", "/"+ctx.Username+"/"+ctx.Reponame+"/upload-file", body) + req.Header.Add("Content-Type", mpForm.FormDataContentType()) + resp := ctx.Session.MakeRequest(t, req, http.StatusOK) + + respMap := map[string]string{} + DecodeJSON(t, resp, &respMap) + return respMap["uuid"] + } + + // Generate random file + tmpFile := path.Join(t.TempDir(), "web-upload-test-file.bin") + require.NoError(t, generateRandomFile(1024*1024/4, tmpFile)) + expectedContent, err := os.ReadFile(tmpFile) + require.NoError(t, err) + + // Upload generated file + fileUUID := uploadFile(t, tmpFile) + req := NewRequestWithValues(t, "POST", ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch, map[string]string{ + "commit_choice": "direct", + "files": fileUUID, + "_csrf": GetCSRF(t, ctx.Session, ctx.Username+"/"+ctx.Reponame+"/_upload/"+setting.Repository.DefaultBranch), + "commit_mail_id": "-1", + }) + ctx.Session.MakeRequest(t, req, http.StatusSeeOther) + + // Get some handles on the target repository and file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + repo, err := git.OpenRepository(git.DefaultContext, remoteRepoPath) + require.NoError(t, err) + defer repo.Close() + tree, err := repo.GetTree(setting.Repository.DefaultBranch) + require.NoError(t, err) + treeEntry, err := tree.GetTreeEntryByPath(filepath.Base(tmpFile)) + require.NoError(t, err) + blob := treeEntry.Blob() + + // Check that the uploaded file is annexed + isAnnexed, err := annex.IsAnnexed(blob) + require.NoError(t, err) + require.True(t, isAnnexed) + + // Check that the uploaded file has the correct content + annexedFile, err := annex.Content(blob) + require.NoError(t, err) + actualContent, err := io.ReadAll(annexedFile) + require.NoError(t, err) + require.Equal(t, expectedContent, actualContent) + }) + }) +} + func TestGitAnnexMedia(t *testing.T) { if !setting.Annex.Enabled { t.Skip("Skipping since annex support is disabled.") From 6a3858160f8c500c1cf0ebf69e2332c352474afa Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Tue, 30 Jul 2024 13:15:26 +0000 Subject: [PATCH 041/125] Add git-annex also to the rootless container (#24) Same as https://codeberg.org/matrss/forgejo-aneksajo/commit/89f8aa0bf5134f7c986ea25856fa9ea32a0e0e8c, but for the rootless container. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/24 Reviewed-by: matrss Co-authored-by: Michael Hanke Co-committed-by: Michael Hanke --- Dockerfile.rootless | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.rootless b/Dockerfile.rootless index c5d6a13f35..d636e10168 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -71,6 +71,7 @@ RUN apk --no-cache add \ git \ curl \ gnupg \ + git-annex \ && rm -rf /var/cache/apk/* RUN addgroup \ From 5891313b0bac14a1c32b7435bc0e735eaba2efb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 9 Aug 2024 11:51:11 +0000 Subject: [PATCH 042/125] Improve views for annexed but missing files (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, trying to view files that were annexed, but missing, just led to an uninformative error 500. This was rather confusing. With these changes it now shows the pointer target instead of the (missing) content of the file, and also indicates this situation in the "stored with git-annex" message. For semantic correctness views for missing files return a 404 instead of a 200, as they would with the content present. Fixes #7, fixes #13. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/28 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- options/locale/locale_de-DE.ini | 1 + options/locale/locale_en-US.ini | 1 + routers/web/repo/view.go | 54 +++++++++++++++++++++++---------- templates/repo/file_info.tmpl | 2 +- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index 7a28d08e49..22dd5f3eb7 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1316,6 +1316,7 @@ video_not_supported_in_browser=Dein Browser unterstützt das HTML5-„video“-T audio_not_supported_in_browser=Dein Browser unterstützt das HTML5-„audio“-Tag nicht. stored_lfs=Gespeichert mit Git LFS stored_annex=Gespeichert mit Git Annex +stored_annex_not_present = hier nicht vorhanden, versuche git annex whereis symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit-Graph diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 93c1eaa770..45188feb6c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1337,6 +1337,7 @@ video_not_supported_in_browser = Your browser does not support the HTML5 "video" audio_not_supported_in_browser = Your browser does not support the HTML5 "audio" tag. stored_lfs = Stored with Git LFS stored_annex = Stored with Git Annex +stored_annex_not_present = not present here, try using git annex whereis symbolic_link = Symbolic link executable_file = Executable file vendored = Vendored diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 59efa4334f..23177a2a09 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -210,12 +210,13 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - isAnnexFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + isAnnexFilePresent bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { @@ -230,11 +231,22 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, annexContent, err := annex.Content(blob) if err != nil { - // in the case where annex content is missing, what should happen? - // do we render the page with an error message? - // actually that's not a bad idea, there's some sort of error message situation - // TODO: display an error to the user explaining that their data is missing - return nil, nil, nil, err + // If annex.Content returns an error it can mean that the blob does not + // refer to an annexed file or that it is not present here. Since we already + // checked that it is annexed the latter must be the case. So we return the + // content of the blob instead and indicate that the file is indeed annexed, + // but not present here. The template can then communicate the situation. + dataRc, err := blob.DataAsync() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(dataRc, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + return buf, dataRc, &fileInfo{st.IsText(), false, true, false, blob.Size(), nil, st}, nil } stat, err := annexContent.Stat() @@ -248,7 +260,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st := typesniffer.DetectContentType(buf) - return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + return buf, annexContent, &fileInfo{st.IsText(), false, true, true, stat.Size(), nil, st}, nil } dataRc, err := blob.DataAsync() @@ -265,18 +277,18 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil { // fallback to plain file log.Warn("Unable to access LFS pointer %s in repo %d: %v", pointer.Oid, repoID, err) - return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -296,7 +308,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry) { @@ -500,6 +512,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { } ctx.Data["IsLFSFile"] = fInfo.isLFSFile ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile + ctx.Data["IsAnnexFilePresent"] = fInfo.isAnnexFilePresent ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -1203,6 +1216,15 @@ PostRecentBranchCheck: } else { ctx.Data["CodeSearchOptions"] = git.GrepSearchOptions } + isAnnexFile, okAnnexFile := ctx.Data["IsAnnexFile"] + isAnnexFilePresent, okAnnexFilePresent := ctx.Data["IsAnnexFilePresent"] + if okAnnexFile && okAnnexFilePresent && isAnnexFile.(bool) && !isAnnexFilePresent.(bool) { + // If the file to be viewed is annexed but not present then render it normally + // (which will show the plain git blob content, i.e. the symlink or pointer target) + // but make the status code a 404. + ctx.HTML(http.StatusNotFound, tplRepoHome) + return + } ctx.HTML(http.StatusOK, tplRepoHome) } diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 05d9825cfb..8655404394 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -17,7 +17,7 @@ {{if .FileSize}}
{{ctx.Locale.TrSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} - {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}{{if not .IsAnnexFilePresent}} - {{ctx.Locale.Tr "repo.stored_annex_not_present"}}{{end}}){{end}}
{{end}} {{if .LFSLock}} From b643725cb625d3abb2381612cce1560fbb9bb399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 9 Aug 2024 17:17:02 +0000 Subject: [PATCH 043/125] Change the icon for annexed files to file-binary (#29) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #26. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/29 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/base/tool.go | 12 ++++++------ tests/integration/git_annex_test.go | 8 +++++--- web_src/css/repo.css | 1 + 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index 3d0fe73823..a885546fde 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -102,14 +102,14 @@ func Int64sToStrings(ints []int64) []string { // EntryIcon returns the octicon class for displaying files/directories func EntryIcon(entry *git.TreeEntry) string { + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // Show git-annex files as binary files to differentiate them from non-annexed files + // TODO: find a more suitable icon, maybe something related to git-annex + return "file-binary" + } switch { case entry.IsLink(): - isAnnexed, _ := annex.IsAnnexed(entry.Blob()) - if isAnnexed { - // git-annex files are sometimes stored as symlinks; - // short-circuit that so like LFS they are displayed as regular files - return "file" - } te, _, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index ff840c8b68..11d709ca80 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -214,7 +214,7 @@ func TestGitAnnexViews(t *testing.T) { session := loginUser(t, ctx.Username) t.Run("Index", func(t *testing.T) { - // test that annex symlinks renders with the _file icon_ on the main list + // test that annexed files render with the binary file icon on the main list defer tests.PrintCurrentTest(t)() repoLink := path.Join("/", ctx.Username, ctx.Reponame) @@ -222,8 +222,10 @@ func TestGitAnnexViews(t *testing.T) { resp := session.MakeRequest(t, req, http.StatusOK) htmlDoc := NewHTMLParser(t, resp.Body) - isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") - require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + isFileBinaryIconLocked := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file-binary") + require.True(t, isFileBinaryIconLocked, "locked annexed files should render a binary file icon") + isFileBinaryIconUnlocked := htmlDoc.Find("tr[data-entryname='annexed.bin'] > td.name svg").HasClass("octicon-file-binary") + require.True(t, isFileBinaryIconUnlocked, "unlocked annexed files should render a binary file icon") }) t.Run("View", func(t *testing.T) { diff --git a/web_src/css/repo.css b/web_src/css/repo.css index e9cfc1ddde..651c69c703 100644 --- a/web_src/css/repo.css +++ b/web_src/css/repo.css @@ -244,6 +244,7 @@ td .commit-summary { } .repository.file.list #repo-files-table tbody .svg.octicon-file, +.repository.file.list #repo-files-table tbody .svg.octicon-file-binary, .repository.file.list #repo-files-table tbody .svg.octicon-file-symlink-file, .repository.file.list #repo-files-table tbody .svg.octicon-file-directory-symlink { color: var(--color-secondary-dark-7); From 7a03796fe550e31a821bf83c295ca33dba826ae7 Mon Sep 17 00:00:00 2001 From: Michael Hanke Date: Wed, 4 Sep 2024 09:32:40 +0000 Subject: [PATCH 044/125] Elevate external markup renderer interface for annexed file content (#36) Previously, an external renderer that matched on an annexed file would only see its content streamed via `STDIN`, or a temporary file with a copy of its content would be generated and passed-by-filepath (with `IS_INPUT_FILE=true`). Whether that happens, is also subject to `MAX_DISPLAY_FILE_SIZE` (which defaults to 8MB). This was problematic, because annexed files tend to be large. Moreover, if present, they already exist as write-protected files on the file-system. Creating a copy is both expensive and serves no particular purpose. This commit changes how external renderers are called. 1) With `IS_INPUT_FILE=true`, the renderer is passed the true location of an annex key, if present, and an empty path, if not. 2) The original, repository-relative path of the rendering target is made available to the renderer via the `GITEA_RELATIVE_PATH` environment variable. To achieve a lean implementation, the `Blob` of the rendering target is passed on to the `RenderContext` (because the implementation of the annex-related functionality is centered on this dtype. This change makes it less costly to increase `MAX_DISPLAY_FILE_SIZE`, in order to make large, annexed files eligible for markup rendering, because no content copies will be made any longer. External renderers can now use the original file path, with the full original filename, including extensions, for decision making. For example, to detect particular compression formats based in a file name extension, or to alter the rendering based on contextual information encoded in the file path (e.g., a multi-file data structure with a particular organization pattern). Apart from the additional environment variable, there is no change to the handling of renderers that take their input via `STDIN` (i.e., `IS_INPUT_FILE=false`). Fixes #35. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/36 Reviewed-by: matrss Co-authored-by: Michael Hanke Co-committed-by: Michael Hanke --- modules/markup/external/external.go | 25 +++++++++++++++++++++++-- modules/markup/renderer.go | 20 ++++++++++++-------- routers/web/repo/view.go | 3 +++ 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/modules/markup/external/external.go b/modules/markup/external/external.go index 122517ed11..b9760772a1 100644 --- a/modules/markup/external/external.go +++ b/modules/markup/external/external.go @@ -12,6 +12,7 @@ import ( "runtime" "strings" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" @@ -86,8 +87,22 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. commands = strings.Fields(command) args = commands[1:] ) - - if p.IsInputFile { + isAnnexed, _ := annex.IsAnnexed(ctx.Blob) + // if a renderer wants to read a file, and we have annexed content, we can + // provide the annex key file location directly to the renderer. git-annex + // takes care of having that location be read-only, so no critical + // protection layer is needed. Moreover, the file readily exists, and + // expensive temporary files can be avoided, also allowing an operator + // to raise MAX_DISPLAY_FILE_SIZE without much negative impact. + if p.IsInputFile && isAnnexed { + // look for annexed content, will be empty, if there is none + annexContentLocation, _ := annex.ContentLocation(ctx.Blob) + // we call the renderer, even if there is no annex content present. + // showing the pointer file content is not much use, and a topical + // renderer might be able to produce something useful from the + // filename alone (present in ENV) + args = append(args, annexContentLocation) + } else if p.IsInputFile { // write to temp file f, err := os.CreateTemp("", "gitea_input") if err != nil { @@ -130,6 +145,12 @@ func (p *Renderer) Render(ctx *markup.RenderContext, input io.Reader, output io. os.Environ(), "GITEA_PREFIX_SRC="+ctx.Links.SrcLink(), "GITEA_PREFIX_RAW="+ctx.Links.RawLink(), + // also communicate the relative path of the to-be-rendered item. + // this enables the renderer to make use of the original file name + // and path, e.g., to make rendering or dtype-detection decisions + // that go beyond the originally matched extension. Even if the + // content is directly streamed to STDIN + "GITEA_RELATIVE_PATH="+ctx.RelativePath, ) if !p.IsInputFile { cmd.Stdin = input diff --git a/modules/markup/renderer.go b/modules/markup/renderer.go index 2137302f43..c00bd2b56e 100644 --- a/modules/markup/renderer.go +++ b/modules/markup/renderer.go @@ -67,14 +67,18 @@ type Header struct { // RenderContext represents a render context type RenderContext struct { - Ctx context.Context - RelativePath string // relative path from tree root of the branch - Type string - IsWiki bool - Links Links - Metas map[string]string - DefaultLink string - GitRepo *git.Repository + Ctx context.Context + RelativePath string // relative path from tree root of the branch + Type string + IsWiki bool + Links Links + Metas map[string]string + DefaultLink string + GitRepo *git.Repository + // reporting the target blob that is to-be-rendered enables + // deeper inspection in the handler for external renderer + // (i.e., more targeted handling of annexed files) + Blob *git.Blob ShaExistCache map[string]bool cancelFn func() SidebarTocNode ast.Node diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 23177a2a09..6329f5d714 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -371,6 +371,7 @@ func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.Tr }, Metas: ctx.Repo.Repository.ComposeDocumentMetas(ctx), GitRepo: ctx.Repo.GitRepo, + Blob: target.Blob(), }, rd) if err != nil { log.Error("Render failed for %s in %-v: %v Falling back to rendering source", readmeFile.Name(), ctx.Repo.Repository, err) @@ -603,6 +604,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { }, Metas: metas, GitRepo: ctx.Repo.GitRepo, + Blob: entry.Blob(), }, rd) if err != nil { ctx.ServerError("Render", err) @@ -701,6 +703,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry) { }, Metas: ctx.Repo.Repository.ComposeDocumentMetas(ctx), GitRepo: ctx.Repo.GitRepo, + Blob: entry.Blob(), }, rd) if err != nil { ctx.ServerError("Render", err) From ad7e1e7a9f11d4a151f0cca55cc6ee9c1f420549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 30 Sep 2024 14:56:03 +0000 Subject: [PATCH 045/125] Allow anonymous HEAD requests to annex/objects (#41) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git-annex emits HEAD requests for keys while trying to drop them from a repository that was cloned via http. Forgejo asked for authentication for these HEAD requests. This meant that cloning and getting files was possible without authentication, but dropping was not. Since the response to a HEAD request is a subset of the response to a GET request it is safe to make those unauthenticated as well. That is what this change does, although limited to the :username/:reponame/annex/objects endpoint. Fixes #40. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/41 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- routers/web/repo/githttp.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index d715b6bdf0..dfdedb04f3 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -78,7 +78,24 @@ func httpBase(ctx *context.Context) *serviceHandler { strings.HasSuffix(ctx.Req.URL.Path, "git-upload-archive") { isPull = true } else { - isPull = ctx.Req.Method == "GET" + // In addition to GET requests, HEAD requests are also "pull" + // operations (reads), so they should also not require + // authentication. This is necessary for git-annex to operate + // properly, as it emits HEAD requests to check for the + // existence of keys, e.g. before dropping locally, and asking + // for authentication would break unauthenticated http usage in + // this situation. + // It should be safe to make all HEAD requests require no + // authentication, but as it is only necessary for the + // annex/objects endpoints to fix git-annex' drop operations it + // is limited to those for now. + r, err := regexp.Compile("^/?" + username + "/" + reponame + "(.git)?/annex/objects") + if err != nil { + ctx.ServerError("failed to create URL path regex", err) + return nil + } + isPull = ctx.Req.Method == "GET" || + r.MatchString(ctx.Req.URL.Path) && ctx.Req.Method == "HEAD" } var accessMode perm.AccessMode From 8af2afa30f2e934e06277d598e3d87ff3fd73e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 9 Oct 2024 14:44:18 +0000 Subject: [PATCH 046/125] Use PATH when looking for git commands (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes some issues when using a git-annex that is installed in a different location than where git is installed, e.g. when using the git-annex-standalone release or one installed with nix. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/44 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- cmd/serv.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index ac4aa36599..57804034e2 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -339,9 +339,8 @@ func runServ(c *cli.Context) error { return nil } - gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin - gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack - if _, err := os.Stat(gitBinVerb); err != nil { + gitBinVerb, err := exec.LookPath(verb) + if err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, From 989949b5cc4f60d46856e765003643b4e590889a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 9 Oct 2024 19:33:05 +0000 Subject: [PATCH 047/125] Only upload to annex in doAnnexUploadTest (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation both uploaded to the annex and pushed to the git repository. This meant that the tests checking that uploads without permission fail actually could pass when the git push failed but the git-annex upload didn't. The tests didn't catch the situation where unauthorized users could modify the annex. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/46 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 23 +++++----- tests/integration/git_annex_test.go | 68 ++++++++++++++++------------- 2 files changed, 51 insertions(+), 40 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index bab5a7e0b2..ee0843926f 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -104,6 +104,18 @@ func Pointer(blob *git.Blob) (string, error) { return pointer, nil } +func ContentLocationFromPointer(repoPath, pointer string) (string, error) { + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(repoPath, contentLocation) + + return contentLocation, nil +} + // return the absolute path of the content pointed to by the annex pointer stored in the git object // errors if the content is not found in this repo func ContentLocation(blob *git.Blob) (string, error) { @@ -111,16 +123,7 @@ func ContentLocation(blob *git.Blob) (string, error) { if err != nil { return "", err } - - contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) - if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) - } - contentLocation = strings.TrimSpace(contentLocation) - contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals - contentLocation = path.Join(blob.Repo().Path, contentLocation) - - return contentLocation, nil + return ContentLocationFromPointer(blob.Repo().Path, pointer) } // returns a stream open to the annex content diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 11d709ca80..44bc8376d7 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -1119,19 +1119,21 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return err } - _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - return err - } - // verify the file was uploaded - localObjectPath, err := contentLocation(repoPath, "contribution.bin") + blob, err := blobForFile(repoPath, "contribution.bin") + if err != nil { + return err + } + key, err := annex.Pointer(blob) + if err != nil { + return err + } + localObjectPath, err := annex.ContentLocationFromPointer(repoPath, key) if err != nil { return err } - // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file - remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") + remoteObjectPath, err := annex.ContentLocationFromPointer(remoteRepoPath, key) if err != nil { return err } @@ -1325,6 +1327,31 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return nil } +func blobForFile(repoPath, file string) (*git.Blob, error) { + repo, err := git.OpenRepository(git.DefaultContext, repoPath) + if err != nil { + return nil, err + } + defer repo.Close() + + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags + if err != nil { + return nil, err + } + + commit, err := repo.GetCommit(commitID) + if err != nil { + return nil, err + } + + treeEntry, err := commit.GetTreeEntryByPath(file) + if err != nil { + return nil, err + } + + return treeEntry.Blob(), nil +} + /* Find the path in .git/annex/objects/ of the contents for a given annexed file. @@ -1334,30 +1361,11 @@ Find the path in .git/annex/objects/ of the contents for a given annexed file. TODO: pass a parameter to allow examining non-HEAD branches */ func contentLocation(repoPath, file string) (path string, err error) { - path = "" - - repo, err := git.OpenRepository(git.DefaultContext, repoPath) + blob, err := blobForFile(repoPath, file) if err != nil { - return path, nil + return "", err } - defer repo.Close() - - commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags - if err != nil { - return path, nil - } - - commit, err := repo.GetCommit(commitID) - if err != nil { - return path, nil - } - - treeEntry, err := commit.GetTreeEntryByPath(file) - if err != nil { - return path, nil - } - - return annex.ContentLocation(treeEntry.Blob()) + return annex.ContentLocation(blob) } /* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ From 790fc7c792f15c868f37e0d3bf8f608acbccbb10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Mon, 14 Oct 2024 15:59:21 +0000 Subject: [PATCH 048/125] Add git-annex' testremote to the test suite (#48) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `git annex testremote` command runs a built-in set of tests against a remote. It cannot hurt to check our implementation of a git-annex remote against it too. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/48 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- tests/integration/git_annex_test.go | 196 ++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 44bc8376d7..03c8802918 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -395,6 +395,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -425,6 +435,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -457,6 +477,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -487,6 +517,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -519,6 +559,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -549,6 +599,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -581,6 +641,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -611,6 +681,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -642,6 +722,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) @@ -713,6 +803,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -743,6 +843,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -775,6 +885,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -805,6 +925,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -837,6 +967,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -867,6 +1007,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -899,6 +1049,16 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") }) + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -929,6 +1089,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) }) @@ -960,6 +1130,16 @@ func TestGitAnnexPermissions(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) }) }) @@ -1032,6 +1212,22 @@ func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { return nil } +func doAnnexTestremoteReadWriteTest(repoPath string) (err error) { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "testremote", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + return nil +} + +func doAnnexTestremoteReadOnlyTest(repoPath string) (err error) { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "testremote", "origin", "--test-readonly", "annexed.tiff").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + return nil +} + func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { // NB: this test does something slightly different if run separately from "doAnnexInitTest()": // "git annex copy" will notice and run "git annex init", silently. From 319b4fc997c3d4f9a9f6fecd2486eed26cad3f7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 22 Oct 2024 18:33:26 +0000 Subject: [PATCH 049/125] Add tests for git annex drop (#47) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds some rudimentary tests that drop files in a repository's clone as well as from a repository on Forgejo. Fixes #4. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/47 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- tests/integration/git_annex_test.go | 1004 +++++++++++++++++++++------ 1 file changed, 796 insertions(+), 208 deletions(-) diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 03c8802918..68e521397e 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -382,34 +382,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -425,24 +451,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -464,34 +524,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -507,24 +593,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -546,34 +666,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -589,24 +735,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -628,34 +808,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -671,24 +877,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -723,6 +963,26 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) @@ -790,34 +1050,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -833,24 +1119,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -872,34 +1192,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) - }) }) }) @@ -915,24 +1261,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, writerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -954,34 +1334,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") - }) }) }) @@ -997,24 +1403,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, readerCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -1036,34 +1476,60 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxKeyFile(t, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") - }) }) }) @@ -1079,24 +1545,58 @@ func TestGitAnnexPermissions(t *testing.T) { doGitClone(repoPath, repoURL)(t) }) - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + }) - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + }) - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) + }) - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) @@ -1131,6 +1631,26 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) @@ -1290,6 +1810,40 @@ func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { return err } +func doAnnexLocalDropTest(repoPath string) (err error) { + // This test assumes that files are present in repoPath, i.e. it is run after doAnnexDownloadTest. + // This test drops all files from the repository clone. + binPath, err := contentLocation(repoPath, "annexed.bin") + if err != nil { + return err + } + _, err = os.Stat(binPath) + if err != nil { + return err + } + tiffPath, err := contentLocation(repoPath, "annexed.tiff") + if err != nil { + return err + } + _, err = os.Stat(tiffPath) + if err != nil { + return err + } + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "drop").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + _, err = os.Stat(binPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.bin wasn't dropped properly: %w", err) + } + _, err = os.Stat(tiffPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.tiff wasn't dropped properly: %w", err) + } + return nil +} + func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { // NB: this test does something slightly different if run separately from "Init": // it first runs "git annex init" silently in the background. @@ -1345,6 +1899,40 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { return nil } +func doAnnexRemoteDropTest(remoteRepoPath, repoPath string) (err error) { + // This test assumes that files are present in repoPath, i.e. it is run after doAnnexDownloadTest. + // This test drops all files from the remote repository. + binPath, err := contentLocation(remoteRepoPath, "annexed.bin") + if err != nil { + return err + } + _, err = os.Stat(binPath) + if err != nil { + return err + } + tiffPath, err := contentLocation(remoteRepoPath, "annexed.tiff") + if err != nil { + return err + } + _, err = os.Stat(tiffPath) + if err != nil { + return err + } + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "drop", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + _, err = os.Stat(binPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.bin wasn't dropped properly: %w", err) + } + _, err = os.Stat(tiffPath) + if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("annexed.tiff wasn't dropped properly: %w", err) + } + return nil +} + // ---- Helpers ---- func generateRandomFile(size int, path string) (err error) { From fa7d976b2ee0e0001057a2711620f59325924a44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 25 Oct 2024 09:55:56 +0000 Subject: [PATCH 050/125] Add git-annex p2phttp support (#42) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new endpoint under `/git-annex-p2phttp` which acts as an authenticating proxy to git-annex' p2phttp server. This makes it possible to set `annex+/git-annex-p2phttp` as `remote..annexurl` and use git-annex fully over http(s) with the normal credentials and access tokens provided by Forgejo. Fixes #25. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/42 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/testing.yml | 3 - custom/conf/app.example.ini | 2 + modules/annex/annex.go | 28 + modules/setting/annex.go | 7 +- routers/web/repo/annex.go | 146 +++++ routers/web/repo/githttp.go | 20 + routers/web/web.go | 12 +- services/auth/auth.go | 2 +- tests/integration/git_annex_test.go | 888 +++++++++++++++++++++++++--- 9 files changed, 1012 insertions(+), 96 deletions(-) create mode 100644 routers/web/repo/annex.go diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index 3d73df5066..c4972c96c9 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -200,7 +200,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-mysql-migration test-mysql' - timeout-minutes: 120 env: USE_REPO_TEST_DIR: 1 test-pgsql: @@ -236,7 +235,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-pgsql-migration test-pgsql' - timeout-minutes: 120 env: RACE_ENABLED: true USE_REPO_TEST_DIR: 1 @@ -257,7 +255,6 @@ jobs: - uses: ./.forgejo/workflows-composite/build-backend - run: | su forgejo -c 'make test-sqlite-migration test-sqlite' - timeout-minutes: 120 env: TAGS: sqlite sqlite_unlock_notify RACE_ENABLED: true diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 486adc60a0..3c723fa4c7 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2686,6 +2686,8 @@ LEVEL = Info ;; ;; Whether git-annex is enabled; defaults to false ;ENABLED = false +;; Whether to disable p2phttp support; default is the same as repository.DISABLE_HTTP_GIT +;DISABLE_P2PHTTP = false ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/modules/annex/annex.go b/modules/annex/annex.go index ee0843926f..7deb8ec2fa 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -12,8 +12,11 @@ package annex import ( "errors" "fmt" + "io/fs" "os" "path" + "path/filepath" + "regexp" "strings" "code.gitea.io/gitea/modules/git" @@ -161,3 +164,28 @@ func IsAnnexRepo(repo *git.Repository) bool { _, _, err := git.NewCommand(repo.Ctx, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: repo.Path}) return err == nil } + +var repoConfigFileRe = regexp.MustCompile("[^/]+/[^/]+.git/config$") + +func UUID2RepoPath(uuid string) (string, error) { + var repoPath string + err := filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { + if err == nil && repoConfigFileRe.MatchString(path) { + thisRepoPath := strings.TrimSuffix(path, "/config") + stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: thisRepoPath}) + if err != nil { + return nil + } + repoUUID := strings.TrimSpace(stdout) + if repoUUID == uuid { + repoPath = thisRepoPath + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return "", err + } + return repoPath, nil +} diff --git a/modules/setting/annex.go b/modules/setting/annex.go index a0eeac9bb8..35e9e55c0e 100644 --- a/modules/setting/annex.go +++ b/modules/setting/annex.go @@ -9,7 +9,8 @@ import ( // Annex represents the configuration for git-annex var Annex = struct { - Enabled bool `ini:"ENABLED"` + Enabled bool `ini:"ENABLED"` + DisableP2PHTTP bool `ini:"DISABLE_P2PHTTP"` }{} func loadAnnexFrom(rootCfg ConfigProvider) { @@ -17,4 +18,8 @@ func loadAnnexFrom(rootCfg ConfigProvider) { if err := sec.MapTo(&Annex); err != nil { log.Fatal("Failed to map Annex settings: %v", err) } + if !sec.HasKey("DISABLE_P2PHTTP") { + // If DisableP2PHTTP is not explicitly set then use DisableHTTPGit as its default + Annex.DisableP2PHTTP = Repository.DisableHTTPGit + } } diff --git a/routers/web/repo/annex.go b/routers/web/repo/annex.go new file mode 100644 index 0000000000..852b5a11cc --- /dev/null +++ b/routers/web/repo/annex.go @@ -0,0 +1,146 @@ +package repo + +import ( + "context" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "os/exec" + "strings" + "syscall" + "time" + + "code.gitea.io/gitea/models/perm" + access_model "code.gitea.io/gitea/models/perm/access" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unit" + "code.gitea.io/gitea/modules/annex" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + services_context "code.gitea.io/gitea/services/context" +) + +type p2phttpRecordType struct { + CancelFunc func() + LastUsed time.Time + Port string +} + +var p2phttpRecords = make(map[string]*p2phttpRecordType) + +// AnnexP2PHTTP implements git-annex smart HTTP support by delegating to git annex p2phttp +func AnnexP2PHTTP(ctx *services_context.Context) { + uuid := ctx.Params(":uuid") + repoPath, err := annex.UUID2RepoPath(uuid) + if err != nil { + ctx.PlainText(http.StatusNotFound, "Repository not found") + return + } + + parts := strings.Split(repoPath, "/") + repoName := strings.TrimSuffix(parts[len(parts)-1], ".git") + owner := parts[len(parts)-2] + repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner, repoName) + if err != nil { + ctx.PlainText(http.StatusNotFound, "Repository not found") + return + } + + p, err := access_model.GetUserRepoPermission(ctx, repo, ctx.Doer) + if err != nil { + ctx.ServerError("GetUserRepoPermission", err) + return + } + + if !(ctx.Req.Method == "GET" && p.CanAccess(perm.AccessModeRead, unit.TypeCode) || + ctx.Req.Method == "POST" && p.CanAccess(perm.AccessModeWrite, unit.TypeCode) || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/checkpresent") && p.CanAccess(perm.AccessModeRead, unit.TypeCode) || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/keeplocked") || + ctx.Req.Method == "POST" && strings.HasSuffix(ctx.Req.URL.Path, "/lockcontent")) { + // GET requests require at least read access; POST requests for + // anything but checkpresent, lockcontent, and keeplocked + // require write permissions; POST requests for checkpresent + // only require read permissions, as it really is just a read. + // POST requests for lockcontent and keeplocked require no + // authentication at all, as is also the case for the + // authentication in the git-annex-p2phttp server. See + // https://git-annex.branchable.com/bugs/p2phttp__58___drop_difference_wideopen_unauth-readonly/ + // for reasoning. + ctx.Resp.WriteHeader(http.StatusUnauthorized) + return + } + + p2phttpRecord, p2phttpProcessExists := p2phttpRecords[uuid] + if p2phttpProcessExists { + p2phttpRecord.LastUsed = time.Now() + } else { + // Start a new p2phttp process for the requested repository + // There is a race condition here with the port selection, ideally git annex p2phttp could just listen on a unix socket... + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + log.Error("Failed to listen on a free port: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + hopefullyFreePort := strings.SplitN(lis.Addr().String(), ":", 2)[1] + lis.Close() + p2phttpCtx, p2phttpCtxCancel := context.WithCancel(context.Background()) + go func(ctx context.Context) { + cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "annex", "p2phttp", "-J2", "--bind", "127.0.0.1", "--wideopen", "--port", hopefullyFreePort) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Pdeathsig: syscall.SIGINT, + } + cmd.Cancel = func() error { return cmd.Process.Signal(os.Interrupt) } + _ = cmd.Run() + }(p2phttpCtx) + graceful.GetManager().RunAtTerminate(p2phttpCtxCancel) + + // Wait for the p2phttp server to get ready + start := time.Now() + sleepDuration := 1 * time.Millisecond + for { + if time.Since(start) > 5*time.Second { + p2phttpCtxCancel() + log.Error("Failed to start the p2phttp server in a reasonable amount of time") + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + conn, err := net.Dial("tcp", "127.0.0.1:"+hopefullyFreePort) + if err == nil { + conn.Close() + break + } + time.Sleep(sleepDuration) + sleepDuration *= 2 + if sleepDuration > 1*time.Second { + sleepDuration = 1 * time.Second + } + } + + p2phttpRecord = &p2phttpRecordType{CancelFunc: p2phttpCtxCancel, LastUsed: time.Now(), Port: hopefullyFreePort} + p2phttpRecords[uuid] = p2phttpRecord + } + + // Cleanup p2phttp processes that haven't been used for a while + for uuid, record := range p2phttpRecords { + if time.Since(record.LastUsed) > 5*time.Minute { + record.CancelFunc() + delete(p2phttpRecords, uuid) + } + } + + url, err := url.Parse("http://127.0.0.1:" + p2phttpRecord.Port + strings.TrimPrefix(ctx.Req.RequestURI, "/git-annex-p2phttp")) + if err != nil { + log.Error("Failed to parse URL: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + proxy := httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.Out.URL = url + }, + } + proxy.ServeHTTP(ctx.Resp, ctx.Req) +} diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index dfdedb04f3..b9354dd45a 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -562,6 +562,26 @@ func GetInfoRefs(ctx *context.Context) { } } +// GetConfig implements fetching the git config of a repository +func GetConfig(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + setHeaderNoCache(ctx) + config, err := os.ReadFile(filepath.Join(h.getRepoDir(), "config")) + if err != nil { + log.Error("Failed to read git config file: %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + if !setting.Annex.DisableP2PHTTP { + config = append(config, []byte("[annex]\n\turl = annex+"+setting.AppURL+"git-annex-p2phttp\n")...) + } + ctx.Resp.Header().Set("Content-Type", "text/plain") + ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(config))) + http.ServeContent(ctx.Resp, ctx.Req, "config", time.Now(), bytes.NewReader(config)) + } +} + // GetTextFile implements Git dumb HTTP func GetTextFile(p string) func(*context.Context) { return func(ctx *context.Context) { diff --git a/routers/web/web.go b/routers/web/web.go index 4cf4086aba..db0015fb6a 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -363,6 +363,13 @@ func registerRoutes(m *web.Route) { } } + annexP2PHTTPEnabled := func(ctx *context.Context) { + if setting.Annex.DisableP2PHTTP { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -962,6 +969,9 @@ func registerRoutes(m *web.Route) { // ***** END: Organization ***** // ***** START: Repository ***** + m.Group("", func() { + m.Methods("GET,POST", "/git-annex-p2phttp/git-annex/{uuid}/*", repo.AnnexP2PHTTP) + }, ignSignInAndCsrf, annexEnabled, annexP2PHTTPEnabled) m.Group("/repo", func() { m.Get("/create", repo.Create) m.Post("/create", web.Bind(forms.CreateRepoForm{}), repo.CreatePost) @@ -1644,7 +1654,7 @@ func registerRoutes(m *web.Route) { m.Group("", func() { // for git-annex - m.Methods("GET,OPTIONS", "/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.Methods("GET,OPTIONS", "/config", repo.GetConfig) // needed by clients reading annex.uuid during `git annex initremote` m.Methods("GET,OPTIONS", "/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) }, ignSignInAndCsrf, annexEnabled, context.UserAssignmentWeb()) diff --git a/services/auth/auth.go b/services/auth/auth.go index 100e5b25a2..ddd31917e2 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -61,7 +61,7 @@ func isArchivePath(req *http.Request) bool { return archivePathRe.MatchString(req.URL.Path) } -var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) +var annexPathRe = regexp.MustCompile(`^(/git-annex-p2phttp/|/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/)`) func isAnnexPath(req *http.Request) bool { if setting.Annex.Enabled { diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 68e521397e..89e86cf932 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -458,6 +458,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { @@ -507,6 +511,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -600,6 +673,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, writerCtx, func() { @@ -649,6 +726,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -742,6 +888,79 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, readerCtx, func() { @@ -884,6 +1103,79 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err = git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { @@ -938,7 +1230,7 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("Anonymous", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - // Only HTTP has an anonymous mode + // Only HTTP and P2PHTTP have an anonymous mode t.Run("HTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -958,6 +1250,65 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) @@ -1126,6 +1477,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { @@ -1175,6 +1530,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -1268,6 +1692,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, writerCtx, func() { @@ -1317,6 +1745,75 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + require.NoError(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -1410,6 +1907,10 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) + // Unset annexurl so that git-annex uses the dumb http support + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.NoError(t, err) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() withAnnexCtxHTTPPassword(t, u, readerCtx, func() { @@ -1459,81 +1960,8 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) - }) - t.Run("Outsider", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - t.Run("SSH", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - - repoURL := createSSHUrl(ownerCtx.GitPath(), u) - - repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) - defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions - - withAnnexCtxKeyFile(t, ownerCtx, func() { - doGitClone(repoPath, repoURL)(t) - }) - - t.Run("Init", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") - }) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") - }) - }) - - t.Run("LocalDrop", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexLocalDropTest(repoPath)) - }) - }) - - t.Run("Download", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) - }) - }) - - t.Run("RemoteDrop", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) - }) - }) - - t.Run("Upload", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") - }) - }) - - t.Run("TestremoteReadOnly", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) - }) - }) - - t.Run("TestremoteReadWrite", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - withAnnexCtxKeyFile(t, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) - }) - }) - }) - - t.Run("HTTP", func(t *testing.T) { + t.Run("P2PHTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() repoURL := createHTTPUrl(ownerCtx.GitPath(), u) @@ -1547,66 +1975,281 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("Init", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) }) t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) }) t.Run("LocalDrop", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexLocalDropTest(repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexLocalDropTest(repoPath)) }) }) t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) }) }) t.Run("RemoteDrop", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) }) }) t.Run("Upload", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) }) }) t.Run("TestremoteReadOnly", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { - require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + require.NoError(t, doAnnexTestremoteReadOnlyTest(repoPath)) }) }) t.Run("TestremoteReadWrite", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) }) }) }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxKeyFile(t, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + // Try unsetting annexurl + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + }) + }) }) t.Run("Anonymous", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - // Only HTTP has an anonymous mode + // Only HTTP and P2PHTTP have an anonymous mode t.Run("HTTP", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -1626,6 +2269,65 @@ func TestGitAnnexPermissions(t *testing.T) { require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) }) + // Try unsetting annexurl + _, _, err := git.NewCommand(git.DefaultContext, "config", "--unset", "remote.origin.annexurl").RunStdString(&git.RunOpts{Dir: repoPath}) + require.Error(t, err) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("LocalDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexLocalDropTest(repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("RemoteDrop", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexRemoteDropTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + + t.Run("TestremoteReadOnly", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadOnlyTest(repoPath)) + }) + + t.Run("TestremoteReadWrite", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexTestremoteReadWriteTest(repoPath)) + }) + }) + + t.Run("P2PHTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + t.Run("Download", func(t *testing.T) { defer tests.PrintCurrentTest(t)() require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) @@ -2189,8 +2891,14 @@ func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, call credentialedURL := *u credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + credentialedAnnexURL := *u + credentialedAnnexURL.Host = strings.ReplaceAll(credentialedAnnexURL.Host, "127.0.0.1", "localhost") + credentialedAnnexURL.Scheme = "annex+" + credentialedAnnexURL.Scheme + credentialedAnnexURL.Path += "git-annex-p2phttp" + credentialedAnnexURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + creds := path.Join(t.TempDir(), "creds") - require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()+"\n"+credentialedAnnexURL.String()+"\n"), 0o600)) originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) if err != nil && !git.IsErrorExitCode(err, 1) { From 4bc7f3df34da3f076a2222da714fd77f01e7f7db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Fri, 25 Oct 2024 09:56:36 +0000 Subject: [PATCH 051/125] Simplify git blob to annex key lookup (#43) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #27. Reviewed-on: https://codeberg.org/matrss/forgejo-aneksajo/pulls/43 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 102 ++++------------------------ tests/integration/git_annex_test.go | 6 +- 2 files changed, 18 insertions(+), 90 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 7deb8ec2fa..3f998f06cc 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -21,96 +21,24 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" ) -const ( - // > The maximum size of a pointer file is 32 kb. - // - https://git-annex.branchable.com/internals/pointer_file/ - // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: - blobSizeCutoff = 32 * 1024 -) +// ErrBlobIsNotAnnexed occurs if a blob does not contain a valid annex key +var ErrBlobIsNotAnnexed = errors.New("not a git-annex pointer") -// ErrInvalidPointer occurs if the pointer's value doesn't parse -var ErrInvalidPointer = errors.New("Not a git-annex pointer") - -// Gets the content of the blob as raw text, up to n bytes. -// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) -func getBlobContent(b *git.Blob, n int) (string, error) { - dataRc, err := b.DataAsync() +func LookupKey(blob *git.Blob) (string, error) { + stdout, _, err := git.NewCommand(git.DefaultContext, "annex", "lookupkey", "--ref").AddDynamicArguments(blob.ID.String()).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) if err != nil { - return "", err + return "", ErrBlobIsNotAnnexed } - defer dataRc.Close() - buf := make([]byte, n) - n, _ = util.ReadAtMost(dataRc, buf) - buf = buf[:n] - return string(buf), nil + key := strings.TrimSpace(stdout) + return key, nil } -func Pointer(blob *git.Blob) (string, error) { - // git-annex doesn't seem fully spec what its pointer are, but - // the fullest description is here: - // https://git-annex.branchable.com/internals/pointer_file/ - - // a pointer can be: - // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' - // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' - // - // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because - // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. - - if blob.Size() > blobSizeCutoff { - // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. - // https://git-annex.branchable.com/internals/pointer_file/ - - // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. - return "", ErrInvalidPointer - } - - pointer, err := getBlobContent(blob, blobSizeCutoff) +func ContentLocationFromKey(repoPath, key string) (string, error) { + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(key).RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { - return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) - } - - // the spec says a pointer file can contain multiple lines each with a pointer in them - // but that makes no sense to me, so I'm just ignoring all but the first - lines := strings.Split(pointer, "\n") - if len(lines) < 1 { - return "", ErrInvalidPointer - } - pointer = lines[0] - - // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it - if !strings.Contains(pointer, "/annex/") { - return "", ErrInvalidPointer - } - - // extract $KEY - pointer = path.Base(strings.TrimSpace(pointer)) - - // ask git-annex's opinion on $KEY - // XXX: this is probably a bit slow, especially if this operation gets run often - // and examinekey is not that strict: - // - it doesn't enforce that the "BACKEND" tag is one it knows, - // - it doesn't enforce that the fields and their format fit the "BACKEND" tag - // so maybe this is a wasteful step - _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) - if err != nil { - // TODO: make ErrInvalidPointer into a type capable of wrapping err - if strings.TrimSpace(examineStderr) == "git-annex: bad key" { - return "", ErrInvalidPointer - } - return "", err - } - - return pointer, nil -} - -func ContentLocationFromPointer(repoPath, pointer string) (string, error) { - contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: repoPath}) - if err != nil { - return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, pointer, err) + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", repoPath, key, err) } contentLocation = strings.TrimSpace(contentLocation) contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals @@ -122,11 +50,11 @@ func ContentLocationFromPointer(repoPath, pointer string) (string, error) { // return the absolute path of the content pointed to by the annex pointer stored in the git object // errors if the content is not found in this repo func ContentLocation(blob *git.Blob) (string, error) { - pointer, err := Pointer(blob) + key, err := LookupKey(blob) if err != nil { return "", err } - return ContentLocationFromPointer(blob.Repo().Path, pointer) + return ContentLocationFromKey(blob.Repo().Path, key) } // returns a stream open to the annex content @@ -147,11 +75,11 @@ func IsAnnexed(blob *git.Blob) (bool, error) { return false, nil } - // Pointer() is written to only return well-formed pointers + // LookupKey is written to only return well-formed keys // so the test is just to see if it errors - _, err := Pointer(blob) + _, err := LookupKey(blob) if err != nil { - if errors.Is(err, ErrInvalidPointer) { + if errors.Is(err, ErrBlobIsNotAnnexed) { return false, nil } return false, err diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index 89e86cf932..efcb571aa2 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -2576,16 +2576,16 @@ func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { if err != nil { return err } - key, err := annex.Pointer(blob) + key, err := annex.LookupKey(blob) if err != nil { return err } - localObjectPath, err := annex.ContentLocationFromPointer(repoPath, key) + localObjectPath, err := annex.ContentLocationFromKey(repoPath, key) if err != nil { return err } - remoteObjectPath, err := annex.ContentLocationFromPointer(remoteRepoPath, key) + remoteObjectPath, err := annex.ContentLocationFromKey(remoteRepoPath, key) if err != nil { return err } From 525eadc70eb2a708a2db8c46f379dac7e9520ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Tue, 29 Oct 2024 11:45:40 +0000 Subject: [PATCH 052/125] Add an OCI image build and publish workflow (#50) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #49. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/50 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/build-oci-image.yml | 37 ++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 .forgejo/workflows/build-oci-image.yml diff --git a/.forgejo/workflows/build-oci-image.yml b/.forgejo/workflows/build-oci-image.yml new file mode 100644 index 0000000000..9eabdf3eab --- /dev/null +++ b/.forgejo/workflows/build-oci-image.yml @@ -0,0 +1,37 @@ +on: + push: + branches: + - 'forgejo' + tags: + - '*-git-annex*' + +jobs: + build-oci-image: + runs-on: docker + strategy: + matrix: + type: ["rootful", "rootless"] + steps: + - name: Determine registry and username + id: determine-registry-and-username + run: | + echo "registry=${GITHUB_SERVER_URL#https://}" >> "$GITHUB_OUTPUT" + echo "username=${GITHUB_REPOSITORY%/*}" >> "$GITHUB_OUTPUT" + - name: Install Docker + run: curl -fsSL https://get.docker.com | sh + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ${{ steps.determine-registry-and-username.outputs.registry }} + username: ${{ steps.determine-registry-and-username.outputs.username }} + password: ${{ secrets.REGISTRY_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v6 + with: + file: ${{ (matrix.type == 'rootful' && 'Dockerfile') || (matrix.type == 'rootless' && 'Dockerfile.rootless') }} + push: true + tags: ${{ steps.determine-registry-and-username.outputs.registry }}/${{ github.repository }}:${{ github.ref_name }}${{ (matrix.type == 'rootful' && ' ') || (matrix.type == 'rootless' && '-rootless') }} From ede91562dfda2e87e00743a70c0c430402ee9188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Sat, 2 Nov 2024 15:00:59 +0000 Subject: [PATCH 053/125] Fix Forgejo version in published OCI images (#51) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Forgejo version is derived from the git history, so the image build needs to happen in the context of a full repository clone. Also, the post-processing of the version string needs to remove the second occurrence of "-g", as the first one is now part of the added "-git-annex" part. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/51 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- .forgejo/workflows/build-oci-image.yml | 4 ++++ Makefile | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build-oci-image.yml b/.forgejo/workflows/build-oci-image.yml index 9eabdf3eab..8e843b41ee 100644 --- a/.forgejo/workflows/build-oci-image.yml +++ b/.forgejo/workflows/build-oci-image.yml @@ -12,6 +12,9 @@ jobs: matrix: type: ["rootful", "rootless"] steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # fetch the full history so that the Forgejo version is determined properly - name: Determine registry and username id: determine-registry-and-username run: | @@ -32,6 +35,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v6 with: + context: . file: ${{ (matrix.type == 'rootful' && 'Dockerfile') || (matrix.type == 'rootless' && 'Dockerfile.rootless') }} push: true tags: ${{ steps.determine-registry-and-username.outputs.registry }}/${{ github.repository }}:${{ github.ref_name }}${{ (matrix.type == 'rootful' && ' ') || (matrix.type == 'rootless' && '-rootless') }} diff --git a/Makefile b/Makefile index 95658a7842..561d674198 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ else FORGEJO_VERSION_API ?= $(GITEA_VERSION)+${GITEA_COMPATIBILITY} else # drop the "g" prefix prepended by git describe to the commit hash - FORGEJO_VERSION ?= $(shell git describe --exclude '*-test' --tags --always | sed 's/^v//' | sed 's/\-g/-/')+${GITEA_COMPATIBILITY} + FORGEJO_VERSION ?= $(shell git describe --exclude '*-test' --tags --always | sed 's/^v//' | sed 's/\-g/-/2')+${GITEA_COMPATIBILITY} endif endif FORGEJO_VERSION_MAJOR=$(shell echo $(FORGEJO_VERSION) | sed -e 's/\..*//') From fa5b158a5a5d743b247cece17c4ca8e09034ecb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 6 Nov 2024 14:29:39 +0000 Subject: [PATCH 054/125] Explicitly set http(s) default ports in annex.url (#55) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise, git-annex tries to use its own default port (9417) and fails. Fixes #52. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/55 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- routers/web/repo/githttp.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index b9354dd45a..53f18f9617 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -10,6 +10,7 @@ import ( gocontext "context" "fmt" "net/http" + "net/url" "os" "path/filepath" "regexp" @@ -574,7 +575,23 @@ func GetConfig(ctx *context.Context) { return } if !setting.Annex.DisableP2PHTTP { - config = append(config, []byte("[annex]\n\turl = annex+"+setting.AppURL+"git-annex-p2phttp\n")...) + appURL, err := url.Parse(setting.AppURL) + if err != nil { + log.Error("Could not parse 'setting.AppURL': %v", err) + ctx.Resp.WriteHeader(http.StatusInternalServerError) + return + } + if appURL.Port() == "" { + // If there is no port set then set the http(s) default ports. + // Without this, git-annex would try its own default port (9417) and fail. + if appURL.Scheme == "http" { + appURL.Host += ":80" + } + if appURL.Scheme == "https" { + appURL.Host += ":443" + } + } + config = append(config, []byte("[annex]\n\turl = annex+"+appURL.String()+"git-annex-p2phttp\n")...) } ctx.Resp.Header().Set("Content-Type", "text/plain") ctx.Resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(config))) From 05daede0efa3e02dd25b6b174b72ce6939586bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= Date: Wed, 6 Nov 2024 16:22:29 +0000 Subject: [PATCH 055/125] Cache git-annex UUID to repository path mappings (#54) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Always walking the filesystem and searching for UUIDs slowed p2phttp operations down significantly on a production server with more than a handful of repositories. This caching strategy ensures that only the first call is rather slow, and subsequent ones should be much faster. This should better be implemented as a background job, but for now this is a simple solution to the problem. Fixes #53. Reviewed-on: https://codeberg.org/forgejo-aneksajo/forgejo-aneksajo/pulls/54 Co-authored-by: Matthias Riße Co-committed-by: Matthias Riße --- modules/annex/annex.go | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/modules/annex/annex.go b/modules/annex/annex.go index 3f998f06cc..7b499fe3f1 100644 --- a/modules/annex/annex.go +++ b/modules/annex/annex.go @@ -95,25 +95,43 @@ func IsAnnexRepo(repo *git.Repository) bool { var repoConfigFileRe = regexp.MustCompile("[^/]+/[^/]+.git/config$") -func UUID2RepoPath(uuid string) (string, error) { - var repoPath string - err := filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { +var ( + uuid2repoPathCache = make(map[string]string) + repoPath2uuidCache = make(map[string]string) +) + +func updateUUID2RepoPathCache() error { + return filepath.WalkDir(setting.RepoRootPath, func(path string, d fs.DirEntry, err error) error { if err == nil && repoConfigFileRe.MatchString(path) { thisRepoPath := strings.TrimSuffix(path, "/config") + _, ok := repoPath2uuidCache[thisRepoPath] + if ok { + return nil + } stdout, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: thisRepoPath}) if err != nil { return nil } repoUUID := strings.TrimSpace(stdout) - if repoUUID == uuid { - repoPath = thisRepoPath - return fs.SkipAll + if repoUUID != "" { + uuid2repoPathCache[repoUUID] = thisRepoPath + repoPath2uuidCache[thisRepoPath] = repoUUID } } return nil }) - if err != nil { +} + +func UUID2RepoPath(uuid string) (string, error) { + if repoPath, ok := uuid2repoPathCache[uuid]; ok { + return repoPath, nil + } + // If the cache didn't contain an entry for the UUID then update the cache and try again + if err := updateUUID2RepoPathCache(); err != nil { return "", err } - return repoPath, nil + if repoPath, ok := uuid2repoPathCache[uuid]; ok { + return repoPath, nil + } + return "", fmt.Errorf("no repository known for UUID '%s'", uuid) } From b0a9f5650863be573835617feb52d00fe0120694 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 21 Jan 2025 14:42:04 +0000 Subject: [PATCH 056/125] ci: fix go version check (#6646) Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6646 --- .forgejo/workflows-composite/setup-env/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows-composite/setup-env/action.yaml b/.forgejo/workflows-composite/setup-env/action.yaml index 28216e9b8d..f19569a137 100644 --- a/.forgejo/workflows-composite/setup-env/action.yaml +++ b/.forgejo/workflows-composite/setup-env/action.yaml @@ -19,7 +19,7 @@ runs: set -ex toolchain=$(grep -oP '(?<=toolchain ).+' go.mod) version=$(go version | cut -d' ' -f3) - if [ "$toolchain" != "$version" ]; then - echo "go version mismatch: $toolchain <> $version" + if dpkg --compare-versions ${version#go} lt ${toolchain#go}; then + echo "go version too low: $toolchain >= $version" exit 1 fi From faa263d54a698fb6c61c0b32e2715ecbb1387170 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 28 Jan 2025 11:34:32 +0000 Subject: [PATCH 057/125] Update dependency katex to v0.16.21 [SECURITY] (v10.0/forgejo) (#6694) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR contains the following updates: | Package | Type | Update | Change | |---|---|---|---| | [katex](https://katex.org) ([source](https://github.com/KaTeX/KaTeX)) | dependencies | patch | [`0.16.18` -> `0.16.21`](https://renovatebot.com/diffs/npm/katex/0.16.18/0.16.21) | --- > ⚠️ **Warning** > > Some dependencies could not be looked up. Check the Dependency Dashboard for more information. --- ### KaTeX \htmlData does not validate attribute names [CVE-2025-23207](https://nvd.nist.gov/vuln/detail/CVE-2025-23207) / [GHSA-cg87-wmx4-v546](https://github.com/advisories/GHSA-cg87-wmx4-v546)
More information #### Details ##### Impact KaTeX users who render untrusted mathematical expressions with `renderToString` could encounter malicious input using `\htmlData` that runs arbitrary JavaScript, or generate invalid HTML. ##### Patches Upgrade to KaTeX v0.16.21 to remove this vulnerability. ##### Workarounds - Avoid use of or turn off the `trust` option, or set it to forbid `\htmlData` commands. - Forbid inputs containing the substring `"\\htmlData"`. - Sanitize HTML output from KaTeX. ##### Details `\htmlData` did not validate its attribute name argument, allowing it to generate invalid or malicious HTML that runs scripts. ##### For more information If you have any questions or comments about this advisory: - Open an issue or security advisory in the [KaTeX repository](https://github.com/KaTeX/KaTeX/) - Email us at [katex-security@mit.edu](mailto:katex-security@mit.edu) #### Severity - CVSS Score: 6.3 / 10 (Medium) - Vector String: `CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:L/I:L/A:L` #### References - [https://github.com/KaTeX/KaTeX/security/advisories/GHSA-cg87-wmx4-v546](https://github.com/KaTeX/KaTeX/security/advisories/GHSA-cg87-wmx4-v546) - [https://nvd.nist.gov/vuln/detail/CVE-2025-23207](https://nvd.nist.gov/vuln/detail/CVE-2025-23207) - [https://github.com/KaTeX/KaTeX/commit/ff289955e81aab89086eef09254cbf88573d415c](https://github.com/KaTeX/KaTeX/commit/ff289955e81aab89086eef09254cbf88573d415c) - [https://github.com/KaTeX/KaTeX](https://github.com/KaTeX/KaTeX) This data is provided by [OSV](https://osv.dev/vulnerability/GHSA-cg87-wmx4-v546) and the [GitHub Advisory Database](https://github.com/github/advisory-database) ([CC-BY 4.0](https://github.com/github/advisory-database/blob/main/LICENSE.md)).
--- ### Release Notes
KaTeX/KaTeX (katex) ### [`v0.16.21`](https://github.com/KaTeX/KaTeX/blob/HEAD/CHANGELOG.md#01621-2025-01-17) [Compare Source](https://github.com/KaTeX/KaTeX/compare/v0.16.20...v0.16.21) ##### Bug Fixes - escape \htmlData attribute name ([57914ad](https://github.com/KaTeX/KaTeX/commit/57914ad91eff401357f44bf364b136d37eba04f8)) ### [`v0.16.20`](https://github.com/KaTeX/KaTeX/blob/HEAD/CHANGELOG.md#01620-2025-01-12) [Compare Source](https://github.com/KaTeX/KaTeX/compare/v0.16.19...v0.16.20) ##### Bug Fixes - \providecommand does not overwrite existing macro ([#​4000](https://github.com/KaTeX/KaTeX/issues/4000)) ([6d30fe4](https://github.com/KaTeX/KaTeX/commit/6d30fe47b06f9da9b836fe518d5cbbecf6a6a3a1)), closes [#​3928](https://github.com/KaTeX/KaTeX/issues/3928) ### [`v0.16.19`](https://github.com/KaTeX/KaTeX/blob/HEAD/CHANGELOG.md#01619-2024-12-29) [Compare Source](https://github.com/KaTeX/KaTeX/compare/v0.16.18...v0.16.19) ##### Bug Fixes - **types:** improve `strict` function type ([#​4009](https://github.com/KaTeX/KaTeX/issues/4009)) ([4228b4e](https://github.com/KaTeX/KaTeX/commit/4228b4eb529b8e35def66cc6e4fa467383b98c86))
--- ### Configuration 📅 **Schedule**: Branch creation - "" (UTC), Automerge - "* 0-3 * * *" (UTC). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 🔕 **Ignore**: Close this PR and you won't be reminded about this update again. --- - [ ] If you want to rebase/retry this PR, check this box --- This PR has been generated by [Renovate Bot](https://github.com/renovatebot/renovate). Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6694 Reviewed-by: Gusted Co-authored-by: Renovate Bot Co-committed-by: Renovate Bot --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index e081796a52..7b1c1ce6b3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,7 +30,7 @@ "htmx.org": "1.9.12", "idiomorph": "0.3.0", "jquery": "3.7.1", - "katex": "0.16.18", + "katex": "0.16.21", "mermaid": "11.4.1", "mini-css-extract-plugin": "2.9.2", "minimatch": "10.0.1", @@ -10368,9 +10368,9 @@ "license": "MIT" }, "node_modules/katex": { - "version": "0.16.18", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.18.tgz", - "integrity": "sha512-LRuk0rPdXrecAFwQucYjMiIs0JFefk6N1q/04mlw14aVIVgxq1FO0MA9RiIIGVaKOB5GIP5GH4aBBNraZERmaQ==", + "version": "0.16.21", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.21.tgz", + "integrity": "sha512-XvqR7FgOHtWupfMiigNzmh+MgUVmDGU2kXZm899ZkPfcuoPuFxyHmXsgATDpFZDAXCI8tvinaVcDo8PIIJSo4A==", "funding": [ "https://opencollective.com/katex", "https://github.com/sponsors/katex" diff --git a/package.json b/package.json index fe7c30471b..dbd21f89f0 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "htmx.org": "1.9.12", "idiomorph": "0.3.0", "jquery": "3.7.1", - "katex": "0.16.18", + "katex": "0.16.21", "mermaid": "11.4.1", "mini-css-extract-plugin": "2.9.2", "minimatch": "10.0.1", From 7ee19b4c6c1a879e3a0f2b32642834ec93b8b20b Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Tue, 28 Jan 2025 15:46:07 +0000 Subject: [PATCH 058/125] chore: consistent docker image and action references (#6704) backport of #6703 - replace `code.forgejo.org` ->`data.forgejo.org` on docker images - add `https://data.forgejo.org/` to actions where missing Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6704 Reviewed-by: Earl Warren Co-authored-by: Michael Kriese Co-committed-by: Michael Kriese --- .forgejo/workflows-composite/build-backend/action.yaml | 2 +- .forgejo/workflows-composite/setup-cache-go/action.yaml | 2 +- .forgejo/workflows/build-release-integration.yml | 2 +- .forgejo/workflows/build-release.yml | 2 +- .forgejo/workflows/cascade-setup-end-to-end.yml | 4 ++-- .forgejo/workflows/publish-release.yml | 2 +- .forgejo/workflows/testing.yml | 4 ++-- Dockerfile | 6 +++--- Dockerfile.rootless | 6 +++--- 9 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.forgejo/workflows-composite/build-backend/action.yaml b/.forgejo/workflows-composite/build-backend/action.yaml index ada372b834..68a99ffaf9 100644 --- a/.forgejo/workflows-composite/build-backend/action.yaml +++ b/.forgejo/workflows-composite/build-backend/action.yaml @@ -3,7 +3,7 @@ runs: steps: - run: | su forgejo -c 'make deps-backend' - - uses: actions/cache@v4 + - uses: https://data.forgejo.org/actions/cache@v4 id: cache-backend with: path: ${{github.workspace}}/gitea diff --git a/.forgejo/workflows-composite/setup-cache-go/action.yaml b/.forgejo/workflows-composite/setup-cache-go/action.yaml index 67372d9f36..1e0425fd0e 100644 --- a/.forgejo/workflows-composite/setup-cache-go/action.yaml +++ b/.forgejo/workflows-composite/setup-cache-go/action.yaml @@ -48,7 +48,7 @@ runs: - name: "Restore Go dependencies from cache or mark for later caching" id: cache-deps - uses: actions/cache@v4 + uses: https://data.forgejo.org/actions/cache@v4 with: key: setup-cache-go-deps-${{ runner.os }}-${{ inputs.username }}-${{ steps.go-version.outputs.go_version }}-${{ hashFiles('go.sum', 'go.mod') }} restore-keys: | diff --git a/.forgejo/workflows/build-release-integration.yml b/.forgejo/workflows/build-release-integration.yml index 6410915644..1af6d567dd 100644 --- a/.forgejo/workflows/build-release-integration.yml +++ b/.forgejo/workflows/build-release-integration.yml @@ -25,7 +25,7 @@ jobs: if: vars.ROLE == 'forgejo-coding' runs-on: lxc-bookworm steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - id: forgejo uses: https://data.forgejo.org/actions/setup-forgejo@v2.0.4 diff --git a/.forgejo/workflows/build-release.yml b/.forgejo/workflows/build-release.yml index 9d88cb43dd..0d7f94c5a6 100644 --- a/.forgejo/workflows/build-release.yml +++ b/.forgejo/workflows/build-release.yml @@ -33,7 +33,7 @@ jobs: # root is used for testing, allow it if: vars.ROLE == 'forgejo-integration' || github.repository_owner == 'root' steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 with: fetch-depth: 0 diff --git a/.forgejo/workflows/cascade-setup-end-to-end.yml b/.forgejo/workflows/cascade-setup-end-to-end.yml index 710cd27ba4..bcc7821f4f 100644 --- a/.forgejo/workflows/cascade-setup-end-to-end.yml +++ b/.forgejo/workflows/cascade-setup-end-to-end.yml @@ -37,11 +37,11 @@ jobs: container: image: data.forgejo.org/oci/node:20-bookworm steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 with: fetch-depth: '0' show-progress: 'false' - - uses: https://code.forgejo.org/actions/cascading-pr@v2.2.0 + - uses: https://data.forgejo.org/actions/cascading-pr@v2.2.0 with: origin-url: ${{ env.GITHUB_SERVER_URL }} origin-repo: ${{ github.repository }} diff --git a/.forgejo/workflows/publish-release.yml b/.forgejo/workflows/publish-release.yml index a3ff48c718..93ad54de1c 100644 --- a/.forgejo/workflows/publish-release.yml +++ b/.forgejo/workflows/publish-release.yml @@ -39,7 +39,7 @@ jobs: runs-on: lxc-bookworm if: vars.DOER != '' && vars.FORGEJO != '' && vars.TO_OWNER != '' && vars.FROM_OWNER != '' && secrets.TOKEN != '' steps: - - uses: actions/checkout@v4 + - uses: https://data.forgejo.org/actions/checkout@v4 - name: copy & sign uses: https://data.forgejo.org/forgejo/forgejo-build-publish/publish@v5.3.1 diff --git a/.forgejo/workflows/testing.yml b/.forgejo/workflows/testing.yml index eb3163d3ae..784bc45736 100644 --- a/.forgejo/workflows/testing.yml +++ b/.forgejo/workflows/testing.yml @@ -46,7 +46,7 @@ jobs: apt-get update -qq apt-get -q install -qq -y zstd - name: "Cache frontend build for playwright testing" - uses: actions/cache/save@v4 + uses: https://data.forgejo.org/actions/cache/save@v4 with: path: ${{github.workspace}}/public/assets key: frontend-build-${{ github.sha }} @@ -104,7 +104,7 @@ jobs: fetch-depth: 20 - uses: ./.forgejo/workflows-composite/setup-env - name: "Restore frontend build" - uses: actions/cache/restore@v4 + uses: https://data.forgejo.org/actions/cache/restore@v4 id: cache-frontend with: path: ${{github.workspace}}/public/assets diff --git a/Dockerfile b/Dockerfile index ae21a0821e..af9269a6ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ -FROM --platform=$BUILDPLATFORM code.forgejo.org/oci/xx AS xx +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/xx AS xx -FROM --platform=$BUILDPLATFORM code.forgejo.org/oci/golang:1.23-alpine3.20 as build-env +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/golang:1.23-alpine3.20 as build-env ARG GOPROXY ENV GOPROXY=${GOPROXY:-direct} @@ -51,7 +51,7 @@ RUN chmod 755 /tmp/local/usr/bin/entrypoint \ /go/src/code.gitea.io/gitea/environment-to-ini RUN chmod 644 /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete -FROM code.forgejo.org/oci/alpine:3.20 +FROM data.forgejo.org/oci/alpine:3.20 ARG RELEASE_VERSION LABEL maintainer="contact@forgejo.org" \ org.opencontainers.image.authors="Forgejo" \ diff --git a/Dockerfile.rootless b/Dockerfile.rootless index c5d6a13f35..82d15e8eac 100644 --- a/Dockerfile.rootless +++ b/Dockerfile.rootless @@ -1,6 +1,6 @@ -FROM --platform=$BUILDPLATFORM code.forgejo.org/oci/xx AS xx +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/xx AS xx -FROM --platform=$BUILDPLATFORM code.forgejo.org/oci/golang:1.23-alpine3.20 as build-env +FROM --platform=$BUILDPLATFORM data.forgejo.org/oci/golang:1.23-alpine3.20 as build-env ARG GOPROXY ENV GOPROXY=${GOPROXY:-direct} @@ -49,7 +49,7 @@ RUN chmod 755 /tmp/local/usr/local/bin/docker-entrypoint.sh \ /go/src/code.gitea.io/gitea/environment-to-ini RUN chmod 644 /go/src/code.gitea.io/gitea/contrib/autocompletion/bash_autocomplete -FROM code.forgejo.org/oci/alpine:3.20 +FROM data.forgejo.org/oci/alpine:3.20 LABEL maintainer="contact@forgejo.org" \ org.opencontainers.image.authors="Forgejo" \ org.opencontainers.image.url="https://forgejo.org" \ From 114d8975b55513c02ab06b13a7cc2bae63fec39a Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Wed, 29 Jan 2025 08:24:37 +0000 Subject: [PATCH 059/125] [v10.0/forgejo] fix: render issue titles consistently (#6717) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6715 - Render the issue titles in dashboard feed in consistent manner, by using the existing `RenderIssueTitle`. - Added integration tests (not exhaustive for all comment types, but exhaustive enough for the current code where some comment types are grouped together). - Resolves forgejo/forgejo#6705 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6717 Reviewed-by: Gusted Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- templates/user/dashboard/feeds.tmpl | 8 ++-- tests/integration/pull_icon_test.go | 15 +++---- tests/integration/pull_review_test.go | 10 +++-- tests/integration/user_dashboard_test.go | 51 ++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 16 deletions(-) diff --git a/templates/user/dashboard/feeds.tmpl b/templates/user/dashboard/feeds.tmpl index bd2a3800a2..85ae7266d9 100644 --- a/templates/user/dashboard/feeds.tmpl +++ b/templates/user/dashboard/feeds.tmpl @@ -103,11 +103,11 @@ {{ctx.Locale.Tr "action.compare_commits" $push.Len}} » {{end}} {{else if .GetOpType.InActions "create_issue"}} - {{index .GetIssueInfos 1 | RenderEmoji $.Context | RenderCodeBlock}} + {{RenderIssueTitle ctx (index .GetIssueInfos 1) (.Repo.ComposeMetas ctx)}} {{else if .GetOpType.InActions "create_pull_request"}} - {{index .GetIssueInfos 1 | RenderEmoji $.Context | RenderCodeBlock}} + {{RenderIssueTitle ctx (index .GetIssueInfos 1) (.Repo.ComposeMetas ctx)}} {{else if .GetOpType.InActions "comment_issue" "approve_pull_request" "reject_pull_request" "comment_pull"}} - {{(.GetIssueTitle ctx) | RenderEmoji $.Context | RenderCodeBlock}} + {{RenderIssueTitle ctx (.GetIssueTitle ctx) (.Repo.ComposeMetas ctx)}} {{$comment := index .GetIssueInfos 1}} {{if $comment}}
{{RenderMarkdownToHtml ctx $comment}}
@@ -115,7 +115,7 @@ {{else if .GetOpType.InActions "merge_pull_request"}}
{{index .GetIssueInfos 1}}
{{else if .GetOpType.InActions "close_issue" "reopen_issue" "close_pull_request" "reopen_pull_request"}} - {{(.GetIssueTitle ctx) | RenderEmoji $.Context | RenderCodeBlock}} + {{RenderIssueTitle ctx (.GetIssueTitle ctx) (.Repo.ComposeMetas ctx)}} {{else if .GetOpType.InActions "pull_review_dismissed"}}
{{ctx.Locale.Tr "action.review_dismissed_reason"}}
{{index .GetIssueInfos 2 | RenderEmoji $.Context}}
diff --git a/tests/integration/pull_icon_test.go b/tests/integration/pull_icon_test.go index 8fde547ce9..b678550c30 100644 --- a/tests/integration/pull_icon_test.go +++ b/tests/integration/pull_icon_test.go @@ -133,7 +133,7 @@ func testPullRequestListIcon(t *testing.T, doc *HTMLDoc, name, expectedColor, ex } func createOpenPullRequest(ctx context.Context, t *testing.T, user *user_model.User, repo *repo_model.Repository) *issues_model.PullRequest { - pull := createPullRequest(t, user, repo, "open") + pull := createPullRequest(t, user, repo, "branch-open", "open") assert.False(t, pull.Issue.IsClosed) assert.False(t, pull.HasMerged) @@ -143,7 +143,7 @@ func createOpenPullRequest(ctx context.Context, t *testing.T, user *user_model.U } func createOpenWipPullRequest(ctx context.Context, t *testing.T, user *user_model.User, repo *repo_model.Repository) *issues_model.PullRequest { - pull := createPullRequest(t, user, repo, "open-wip") + pull := createPullRequest(t, user, repo, "branch-open-wip", "open-wip") err := issue_service.ChangeTitle(ctx, pull.Issue, user, "WIP: "+pull.Issue.Title) require.NoError(t, err) @@ -156,7 +156,7 @@ func createOpenWipPullRequest(ctx context.Context, t *testing.T, user *user_mode } func createClosedPullRequest(ctx context.Context, t *testing.T, user *user_model.User, repo *repo_model.Repository) *issues_model.PullRequest { - pull := createPullRequest(t, user, repo, "closed") + pull := createPullRequest(t, user, repo, "branch-closed", "closed") err := issue_service.ChangeStatus(ctx, pull.Issue, user, "", true) require.NoError(t, err) @@ -169,7 +169,7 @@ func createClosedPullRequest(ctx context.Context, t *testing.T, user *user_model } func createClosedWipPullRequest(ctx context.Context, t *testing.T, user *user_model.User, repo *repo_model.Repository) *issues_model.PullRequest { - pull := createPullRequest(t, user, repo, "closed-wip") + pull := createPullRequest(t, user, repo, "branch-closed-wip", "closed-wip") err := issue_service.ChangeTitle(ctx, pull.Issue, user, "WIP: "+pull.Issue.Title) require.NoError(t, err) @@ -185,7 +185,7 @@ func createClosedWipPullRequest(ctx context.Context, t *testing.T, user *user_mo } func createMergedPullRequest(ctx context.Context, t *testing.T, user *user_model.User, repo *repo_model.Repository) *issues_model.PullRequest { - pull := createPullRequest(t, user, repo, "merged") + pull := createPullRequest(t, user, repo, "branch-merged", "merged") gitRepo, err := git.OpenRepository(ctx, repo.RepoPath()) defer gitRepo.Close() @@ -202,10 +202,7 @@ func createMergedPullRequest(ctx context.Context, t *testing.T, user *user_model return pull } -func createPullRequest(t *testing.T, user *user_model.User, repo *repo_model.Repository, name string) *issues_model.PullRequest { - branch := "branch-" + name - title := "Testing " + name - +func createPullRequest(t *testing.T, user *user_model.User, repo *repo_model.Repository, branch, title string) *issues_model.PullRequest { _, err := files_service.ChangeRepoFiles(git.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{ Files: []*files_service.ChangeRepoFile{ { diff --git a/tests/integration/pull_review_test.go b/tests/integration/pull_review_test.go index 1319db29bf..e1db171f16 100644 --- a/tests/integration/pull_review_test.go +++ b/tests/integration/pull_review_test.go @@ -518,7 +518,7 @@ func TestPullView_GivenApproveOrRejectReviewOnClosedPR(t *testing.T) { resp := testPullCreate(t, user1Session, "user1", "repo1", false, "master", "a-test-branch", "This is a pull title") elem := strings.Split(test.RedirectURL(resp), "/") assert.EqualValues(t, "pulls", elem[3]) - testIssueClose(t, user1Session, elem[1], elem[2], elem[4]) + testIssueClose(t, user1Session, elem[1], elem[2], elem[4], true) // Get the commit SHA pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ @@ -579,8 +579,12 @@ func testSubmitReview(t *testing.T, session *TestSession, csrf, owner, repo, pul return session.MakeRequest(t, req, expectedSubmitStatus) } -func testIssueClose(t *testing.T, session *TestSession, owner, repo, issueNumber string) *httptest.ResponseRecorder { - req := NewRequest(t, "GET", path.Join(owner, repo, "pulls", issueNumber)) +func testIssueClose(t *testing.T, session *TestSession, owner, repo, issueNumber string, isPull bool) *httptest.ResponseRecorder { + issueType := "issues" + if isPull { + issueType = "pulls" + } + req := NewRequest(t, "GET", path.Join(owner, repo, issueType, issueNumber)) resp := session.MakeRequest(t, req, http.StatusOK) htmlDoc := NewHTMLParser(t, resp.Body) diff --git a/tests/integration/user_dashboard_test.go b/tests/integration/user_dashboard_test.go index abc3e065d9..0ed5193c48 100644 --- a/tests/integration/user_dashboard_test.go +++ b/tests/integration/user_dashboard_test.go @@ -5,12 +5,21 @@ package integration import ( "net/http" + "net/url" + "strconv" "strings" "testing" + "code.gitea.io/gitea/models/db" + unit_model "code.gitea.io/gitea/models/unit" "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/translation" + issue_service "code.gitea.io/gitea/services/issue" + files_service "code.gitea.io/gitea/services/repository/files" + "code.gitea.io/gitea/tests" + "github.com/PuerkitoBio/goquery" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -28,3 +37,45 @@ func TestUserDashboardActionLinks(t *testing.T) { assert.EqualValues(t, locale.TrString("new_migrate.link"), strings.TrimSpace(links.Find("a[href='/repo/migrate']").Text())) assert.EqualValues(t, locale.TrString("new_org.link"), strings.TrimSpace(links.Find("a[href='/org/create']").Text())) } + +func TestDashboardTitleRendering(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + user4 := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 4}) + sess := loginUser(t, user4.Name) + + repo, _, f := tests.CreateDeclarativeRepo(t, user4, "", + []unit_model.Type{unit_model.TypePullRequests, unit_model.TypeIssues}, nil, + []*files_service.ChangeRepoFile{ + { + Operation: "create", + TreePath: "test.txt", + ContentReader: strings.NewReader("Just some text here"), + }, + }, + ) + defer f() + + issue := createIssue(t, user4, repo, "`:exclamation:` not rendered", "Hi there!") + pr := createPullRequest(t, user4, repo, "testing", "`:exclamation:` not rendered") + + _, err := issue_service.CreateIssueComment(db.DefaultContext, user4, repo, issue, "hi", nil) + require.NoError(t, err) + + _, err = issue_service.CreateIssueComment(db.DefaultContext, user4, repo, pr.Issue, "hi", nil) + require.NoError(t, err) + + testIssueClose(t, sess, repo.OwnerName, repo.Name, strconv.Itoa(int(issue.Index)), false) + testIssueClose(t, sess, repo.OwnerName, repo.Name, strconv.Itoa(int(pr.Issue.Index)), true) + + response := sess.MakeRequest(t, NewRequest(t, "GET", "/"), http.StatusOK) + htmlDoc := NewHTMLParser(t, response.Body) + + count := 0 + htmlDoc.doc.Find("#activity-feed .flex-item-main .title").Each(func(i int, s *goquery.Selection) { + count++ + assert.EqualValues(t, ":exclamation: not rendered", s.Text()) + }) + + assert.EqualValues(t, 6, count) + }) +} From c198cb6e65a8867e1fcacb9490e2677854c6ee07 Mon Sep 17 00:00:00 2001 From: forgejo-backport-action Date: Wed, 29 Jan 2025 08:28:25 +0000 Subject: [PATCH 060/125] [v10.0/forgejo] fix(i18n): add forgotten translatable string (#6718) **Backport:** https://codeberg.org/forgejo/forgejo/pulls/6701 - Regression of 75ce1e2ac12042cf00713353055ab1d40b53b798 Co-authored-by: Gusted Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/6718 Reviewed-by: Gusted Co-authored-by: forgejo-backport-action Co-committed-by: forgejo-backport-action --- options/locale/locale_en-US.ini | 1 + templates/repo/editor/commit_form.tmpl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index cf69f6ef16..4d1ebd62c6 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1428,6 +1428,7 @@ editor.user_no_push_to_branch = User cannot push to branch editor.require_signed_commit = Branch requires a signed commit editor.cherry_pick = Cherry-pick %s onto: editor.revert = Revert %s onto: +editor.commit_email = Commit email commits.desc = Browse source code change history. commits.commits = Commits diff --git a/templates/repo/editor/commit_form.tmpl b/templates/repo/editor/commit_form.tmpl index fc04289b70..c42eed69a5 100644 --- a/templates/repo/editor/commit_form.tmpl +++ b/templates/repo/editor/commit_form.tmpl @@ -67,7 +67,7 @@ {{end}}
- +