From 8083b5daf22796589a7ca34b40c7412a0d196ae6 Mon Sep 17 00:00:00 2001 From: Grzegorz Godlewski Date: Wed, 22 Jan 2025 12:55:58 +0100 Subject: [PATCH] Add git commit companion files rule See: #505 --- apps/ui/src/components/GitSettings.vue | 5 ++ package-lock.json | 16 ++++- package.json | 1 + .../google_folder/UserConfigService.ts | 5 +- src/containers/job/JobManagerContainer.ts | 66 +++++++++++++++++++ .../server/routes/ConfigController.ts | 4 ++ src/git/GitScanner.ts | 22 +++++++ test/git/SingleCommitTest.ts | 57 ++++++++++++++++ 8 files changed, 173 insertions(+), 3 deletions(-) create mode 100644 test/git/SingleCommitTest.ts diff --git a/apps/ui/src/components/GitSettings.vue b/apps/ui/src/components/GitSettings.vue index 3607d3de..b8f34b6c 100644 --- a/apps/ui/src/components/GitSettings.vue +++ b/apps/ui/src/components/GitSettings.vue @@ -36,6 +36,11 @@ + +
+ + +
diff --git a/package-lock.json b/package-lock.json index f2796a0b..66b9ede8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,18 +1,19 @@ { "name": "@mieweb/wikigdrive", - "version": "2.12.1", + "version": "2.14.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@mieweb/wikigdrive", - "version": "2.12.1", + "version": "2.14.1", "license": "ISC", "workspaces": [ "apps/ui", "website" ], "dependencies": { + "@nyariv/sandboxjs": "0.8.23", "@opentelemetry/api": "1.3.0", "@opentelemetry/context-zone": "1.8.0", "@opentelemetry/core": "1.28.0", @@ -1356,6 +1357,12 @@ "node": ">= 8" } }, + "node_modules/@nyariv/sandboxjs": { + "version": "0.8.23", + "resolved": "https://registry.npmjs.org/@nyariv/sandboxjs/-/sandboxjs-0.8.23.tgz", + "integrity": "sha512-OaD3i0czFTZzQFQdwgFVEqNyZVK19N6Jzmx/LjmDc+kiV751FQ2h3xscYkAeY4jfau2vZTZau+xR8fpkKrv/Ng==", + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.3.0.tgz", @@ -12782,6 +12789,11 @@ "fastq": "^1.6.0" } }, + "@nyariv/sandboxjs": { + "version": "0.8.23", + "resolved": "https://registry.npmjs.org/@nyariv/sandboxjs/-/sandboxjs-0.8.23.tgz", + "integrity": "sha512-OaD3i0czFTZzQFQdwgFVEqNyZVK19N6Jzmx/LjmDc+kiV751FQ2h3xscYkAeY4jfau2vZTZau+xR8fpkKrv/Ng==" + }, "@opentelemetry/api": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.3.0.tgz", diff --git a/package.json b/package.json index 435a708a..4485e275 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "website" ], "dependencies": { + "@nyariv/sandboxjs": "0.8.23", "@opentelemetry/core": "1.28.0", "@opentelemetry/api": "1.3.0", "@opentelemetry/context-zone": "1.8.0", diff --git a/src/containers/google_folder/UserConfigService.ts b/src/containers/google_folder/UserConfigService.ts index 6b445196..6ddda58f 100644 --- a/src/containers/google_folder/UserConfigService.ts +++ b/src/containers/google_folder/UserConfigService.ts @@ -43,6 +43,7 @@ export class UserConfig { fm_without_version?: boolean; actions_yaml?: string; rewrite_rules?: RewriteRule[]; + companion_files_rule?: string; } const DEFAULT_CONFIG: UserConfig = { @@ -90,7 +91,9 @@ export class UserConfigService { if (!this.config.rewrite_rules || this.config.rewrite_rules.length === 0) { this.config.rewrite_rules = DEFAULT_REWRITE_RULES; } - + if (!this.config.companion_files_rule) { + this.config.companion_files_rule = '(file.path == "content/navigation.md") || (file.path == "content/toc.md") || (commit.id && file.redirectTo == commit.id) || (commit.redirectTo == file.id && file.id)'; + } return this.config; } diff --git a/src/containers/job/JobManagerContainer.ts b/src/containers/job/JobManagerContainer.ts index a0b5c3a4..1a262978 100644 --- a/src/containers/job/JobManagerContainer.ts +++ b/src/containers/job/JobManagerContainer.ts @@ -3,6 +3,8 @@ import fs from 'node:fs'; import path from 'node:path'; import { randomUUID } from 'node:crypto'; +import Sandbox from '@nyariv/sandboxjs'; + import {Container, ContainerConfig, ContainerEngine} from '../../ContainerEngine.ts'; import {FileId} from '../../model/model.ts'; import {GoogleFolderContainer} from '../google_folder/GoogleFolderContainer.ts'; @@ -703,6 +705,70 @@ export class JobManagerContainer extends Container { const gitScanner = new GitScanner(logger, transformedFileSystem.getRealPath(), 'wikigdrive@wikigdrive.com'); await gitScanner.initialize(); + const googleFileSystem = await this.filesService.getSubFileService(driveId, ''); + const userConfigService = new UserConfigService(googleFileSystem); + const userConfig = await userConfigService.load(); + + const contentFileService = await getContentFileService(transformedFileSystem, userConfigService); + const markdownTreeProcessor = new MarkdownTreeProcessor(contentFileService); + await markdownTreeProcessor.load(); + + if (userConfig.companion_files_rule) { + gitScanner.setCompanionFileResolver(async (filePath: string) => { + if (!filePath.endsWith('.md')) { + return []; + } + + let subdir = (userConfigService.config.transform_subdir || '') + .replace(/^\//, '') + .replace(/\/$/, ''); + if (subdir.length > 0) { + subdir += '/'; + } + + filePath = filePath + .replace(/^\//, '') + .substring(subdir.length); + + const tuple = await markdownTreeProcessor.findByPath('/' + filePath); + + const treeItem = tuple[0]; + if (!treeItem) { + return []; + } + + const retVal: Set = new Set(); + + const sandbox = new Sandbox.default(); + const exec = sandbox.compile('return ' + (userConfig.companion_files_rule || 'false')); + + await markdownTreeProcessor.walkTree((treeNode) => { + const commit = { + path: subdir + treeItem.path.replace(/^\//, ''), + id: treeItem.id, + fileName: treeItem.fileName, + mimeType: treeItem.mimeType, + redirectTo: treeItem.redirectTo + }; + const file = { + path: subdir + treeNode.path.replace(/^\//, ''), + id: treeNode.id, + fileName: treeNode.fileName, + mimeType: treeNode.mimeType, + redirectTo: treeNode.redirectTo + }; + + const result = exec({ commit, file }).run(); + + if (result) { + retVal.add(file.path); + } + return false; + }); + return Array.from(retVal); + }); + } + await gitScanner.commit(message, filePaths, user); await this.schedule(driveId, { diff --git a/src/containers/server/routes/ConfigController.ts b/src/containers/server/routes/ConfigController.ts index d201dad9..60790d9a 100644 --- a/src/containers/server/routes/ConfigController.ts +++ b/src/containers/server/routes/ConfigController.ts @@ -13,6 +13,7 @@ export interface ConfigBody { config_toml?: string; transform_subdir?: string; rewrite_rules_yaml?: string; + companion_files_rule?: string; hugo_theme: HugoTheme; auto_sync: boolean; use_google_markdowns: boolean; @@ -98,6 +99,9 @@ export class ConfigController extends Controller { if (body.config?.rewrite_rules_yaml) { userConfigService.config.rewrite_rules = yaml.load(body.config?.rewrite_rules_yaml); } + if (body.config?.companion_files_rule) { + userConfigService.config.companion_files_rule = body.config?.companion_files_rule; + } let modified = false; if ('string' === typeof body.config?.transform_subdir) { let trimmed = body.config?.transform_subdir.trim(); diff --git a/src/git/GitScanner.ts b/src/git/GitScanner.ts index 51aeb890..e9a5d7bc 100644 --- a/src/git/GitScanner.ts +++ b/src/git/GitScanner.ts @@ -45,6 +45,7 @@ interface ExecOpts { export class GitScanner { private logger: Logger; + private companionFileResolver: (filePath: string) => Promise = async () => ([]); constructor(logger: Logger, public readonly rootPath: string, private email: string) { this.logger = logger.child({ filename: __filename }); @@ -201,10 +202,26 @@ export class GitScanner { return retValArr; } + async resolveCompanionFiles(filePaths: string[]): Promise { + const retVal = []; + for (const filePath of filePaths) { + retVal.push(filePath); + try { + retVal.push(...(await this.companionFileResolver(filePath) || [])); + } catch (err) { + this.logger.warn('Error evaluating companion files: ' + err.message); + break; + } + } + return retVal; + } + async commit(message: string, selectedFiles: string[], committer: Commiter): Promise { selectedFiles = selectedFiles.map(fileName => fileName.startsWith('/') ? fileName.substring(1) : fileName) .filter(fileName => !!fileName); + selectedFiles = await this.resolveCompanionFiles(selectedFiles); + const addedFiles: string[] = []; const removedFiles: string[] = []; @@ -964,4 +981,9 @@ export class GitScanner { async removeCached(filePath: string) { await this.exec(`git rm --cached ${filePath}`); } + + setCompanionFileResolver(resolver: (filePath: string) => Promise) { + this.companionFileResolver = resolver; + } + } diff --git a/test/git/SingleCommitTest.ts b/test/git/SingleCommitTest.ts new file mode 100644 index 00000000..7f6acd99 --- /dev/null +++ b/test/git/SingleCommitTest.ts @@ -0,0 +1,57 @@ +import {createTmpDir} from '../utils.ts'; +import {GitScanner} from '../../src/git/GitScanner.ts'; +import fs from 'node:fs'; +import path from 'node:path'; + +import test from '../tester.ts'; +import winston from 'winston'; +import {instrumentLogger} from '../../src/utils/logger/logger.ts'; +import Sandbox from '@nyariv/sandboxjs'; + +const COMMITER1 = { + name: 'John', email: 'john@example.tld' +}; + +const logger = winston.createLogger({ + level: 'debug', + defaultMeta: {}, + transports: [ + new winston.transports.Console() + ] +}); +instrumentLogger(logger); + +test('test single commit', async (t) => { + t.timeout(5000); + + const localRepoDir: string = createTmpDir(); + + try { + const scannerLocal = new GitScanner(logger, localRepoDir, COMMITER1.email); + await scannerLocal.initialize(); + + fs.writeFileSync(path.join(scannerLocal.rootPath, 'test1.md'), 'test'); + fs.writeFileSync(path.join(scannerLocal.rootPath, 'navigation.md'), 'nav'); + fs.writeFileSync(path.join(scannerLocal.rootPath, 'toc.md'), 'toc'); + + { + const changes = await scannerLocal.changes(); + t.is(changes.length, 4); + } + + // console.log('Sandbox', Sandbox); + const sandbox = new Sandbox.default(); + const exec = sandbox.compileExpression('return (filePath.endsWith(".md") && filePath == "test1.md" && ["navigation.md", "toc.md"]) || []'); + + scannerLocal.setCompanionFileResolver((filePath: string) => { + return exec({ filePath }).run(); + }); + + await scannerLocal.commit('initial commit', ['.gitignore', 'test1.md'], COMMITER1); + + const changes = await scannerLocal.changes(); + t.is(changes.length, 0); + } finally { + fs.rmSync(localRepoDir, { recursive: true, force: true }); + } +});