feat(editor): improve obsidian import (#15304)
fix #15290 #### PR Dependency Tree * **PR #15304** 👈 This tree was auto-generated by [Charcoal](https://github.com/danerwilliams/charcoal) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Improved Obsidian vault imports with more reliable attachment and embedded image matching. * Supports nested vault structures and configured attachment folders. * Preserves imported folder hierarchy and organizes imported content more accurately. * **Bug Fixes** * Fixed asset resolution for attachments with nested paths or duplicate filenames. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
This commit is contained in:
@@ -826,6 +826,60 @@ Hello world
|
|||||||
.sort((a, b) => (a ?? '').localeCompare(b ?? '')),
|
.sort((a, b) => (a ?? '').localeCompare(b ?? '')),
|
||||||
entry: snapshotDocByTitle(collection, 'entry', titleById),
|
entry: snapshotDocByTitle(collection, 'entry', titleById),
|
||||||
}).toMatchSnapshot();
|
}).toMatchSnapshot();
|
||||||
|
|
||||||
|
const nestedCollection = new TestWorkspace();
|
||||||
|
nestedCollection.storeExtensions = testStoreExtensions;
|
||||||
|
nestedCollection.meta.initialize();
|
||||||
|
const nestedPlan = await ObsidianTransformer.planObsidianVault({
|
||||||
|
collection: nestedCollection,
|
||||||
|
schema,
|
||||||
|
importedFiles: [
|
||||||
|
withRelativePath(
|
||||||
|
new File(['![[logo.png]]'], 'entry.md', {
|
||||||
|
type: 'text/markdown',
|
||||||
|
}),
|
||||||
|
'vault/notes/entry.md'
|
||||||
|
),
|
||||||
|
withRelativePath(
|
||||||
|
new File(
|
||||||
|
[JSON.stringify({ attachmentFolderPath: 'attachments' })],
|
||||||
|
'app.json',
|
||||||
|
{ type: 'application/json' }
|
||||||
|
),
|
||||||
|
'vault/.obsidian/app.json'
|
||||||
|
),
|
||||||
|
withRelativePath(
|
||||||
|
new File([new Uint8Array([137, 80, 78, 71])], 'logo.png', {
|
||||||
|
type: 'image/png',
|
||||||
|
}),
|
||||||
|
'vault/attachments/logo.png'
|
||||||
|
),
|
||||||
|
withRelativePath(
|
||||||
|
new File([new Uint8Array([137, 80, 78, 72])], 'logo.png', {
|
||||||
|
type: 'image/png',
|
||||||
|
}),
|
||||||
|
'vault/other/logo.png'
|
||||||
|
),
|
||||||
|
],
|
||||||
|
extensions: testStoreExtensions,
|
||||||
|
});
|
||||||
|
const configuredLogo = nestedPlan.batch.blobs.find(
|
||||||
|
blob => blob.sourcePath === 'vault/attachments/logo.png'
|
||||||
|
);
|
||||||
|
expect(configuredLogo).toBeTruthy();
|
||||||
|
expect(JSON.stringify(nestedPlan.batch.docs)).toContain(
|
||||||
|
configuredLogo!.blobId
|
||||||
|
);
|
||||||
|
expect(nestedPlan.batch.blobs).toHaveLength(2);
|
||||||
|
expect(nestedPlan.batch.folders).toEqual(
|
||||||
|
expect.arrayContaining([
|
||||||
|
expect.objectContaining({ path: 'notes', name: 'notes' }),
|
||||||
|
expect.objectContaining({
|
||||||
|
parentPath: 'notes',
|
||||||
|
pageId: nestedPlan.docIds[0],
|
||||||
|
}),
|
||||||
|
])
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('imports notion html zip golden baseline', async () => {
|
test('imports notion html zip golden baseline', async () => {
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import {
|
|||||||
blobsFromAssets,
|
blobsFromAssets,
|
||||||
type ImportBatch,
|
type ImportBatch,
|
||||||
type ImportDoc,
|
type ImportDoc,
|
||||||
|
type ImportFolder,
|
||||||
} from './import-batch.js';
|
} from './import-batch.js';
|
||||||
import {
|
import {
|
||||||
bindImportedAssetsToJob,
|
bindImportedAssetsToJob,
|
||||||
@@ -397,6 +398,70 @@ type ObsidianAttachmentEmbed = {
|
|||||||
fileType: string;
|
fileType: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
type ObsidianAssetLookup = {
|
||||||
|
resolve: (
|
||||||
|
targetPath: string,
|
||||||
|
currentFilePath: string
|
||||||
|
) => { blobId: string; path: string } | null;
|
||||||
|
};
|
||||||
|
|
||||||
|
function createAssetLookup(
|
||||||
|
pathBlobIdMap: ReadonlyMap<string, string>,
|
||||||
|
vaultRoot: string,
|
||||||
|
attachmentFolderPath?: string
|
||||||
|
): ObsidianAssetLookup {
|
||||||
|
const exact = new Map<string, { blobId: string; path: string }>();
|
||||||
|
const byBasename = new Map<
|
||||||
|
string,
|
||||||
|
{ blobId: string; path: string } | typeof AMBIGUOUS_PAGE_LOOKUP
|
||||||
|
>();
|
||||||
|
|
||||||
|
for (const [path, blobId] of pathBlobIdMap) {
|
||||||
|
const entry = { blobId, path };
|
||||||
|
exact.set(normalizeLookupKey(path), entry);
|
||||||
|
const name = normalizeLookupKey(basename(path));
|
||||||
|
const existing = byBasename.get(name);
|
||||||
|
if (!existing) {
|
||||||
|
byBasename.set(name, entry);
|
||||||
|
} else if (
|
||||||
|
existing !== AMBIGUOUS_PAGE_LOOKUP &&
|
||||||
|
existing.blobId !== blobId
|
||||||
|
) {
|
||||||
|
byBasename.set(name, AMBIGUOUS_PAGE_LOOKUP);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
resolve(targetPath, currentFilePath) {
|
||||||
|
const normalizedTarget = normalizeFilePathReference(targetPath);
|
||||||
|
const rootPath = vaultRoot
|
||||||
|
? `${vaultRoot}/${normalizedTarget}`
|
||||||
|
: normalizedTarget;
|
||||||
|
const candidates = [
|
||||||
|
getImageFullPath(currentFilePath, normalizedTarget),
|
||||||
|
rootPath,
|
||||||
|
];
|
||||||
|
if (attachmentFolderPath) {
|
||||||
|
candidates.push(
|
||||||
|
vaultRoot
|
||||||
|
? `${vaultRoot}/${attachmentFolderPath}/${basename(normalizedTarget)}`
|
||||||
|
: `${attachmentFolderPath}/${basename(normalizedTarget)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const candidate of candidates) {
|
||||||
|
const resolved = exact.get(normalizeLookupKey(candidate));
|
||||||
|
if (resolved) return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
const match = byBasename.get(
|
||||||
|
normalizeLookupKey(basename(normalizedTarget))
|
||||||
|
);
|
||||||
|
return match && match !== AMBIGUOUS_PAGE_LOOKUP ? match : null;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function createObsidianAttach(embed: ObsidianAttachmentEmbed): string {
|
function createObsidianAttach(embed: ObsidianAttachmentEmbed): string {
|
||||||
return `<!-- ${OBSIDIAN_ATTACHMENT_EMBED_TAG} ${encodeURIComponent(
|
return `<!-- ${OBSIDIAN_ATTACHMENT_EMBED_TAG} ${encodeURIComponent(
|
||||||
JSON.stringify(embed)
|
JSON.stringify(embed)
|
||||||
@@ -501,7 +566,7 @@ function preprocessObsidianEmbeds(
|
|||||||
markdown: string,
|
markdown: string,
|
||||||
filePath: string,
|
filePath: string,
|
||||||
pageLookupMap: ReadonlyMap<string, string>,
|
pageLookupMap: ReadonlyMap<string, string>,
|
||||||
pathBlobIdMap: ReadonlyMap<string, string>
|
assetLookup: ObsidianAssetLookup
|
||||||
): string {
|
): string {
|
||||||
return replaceWikiLinks(markdown, true, ({ raw, rawTarget, rawAlias }) => {
|
return replaceWikiLinks(markdown, true, ({ raw, rawTarget, rawAlias }) => {
|
||||||
const targetPageId = resolvePageIdFromLookup(
|
const targetPageId = resolvePageIdFromLookup(
|
||||||
@@ -516,8 +581,11 @@ function preprocessObsidianEmbeds(
|
|||||||
const { path } = parseObsidianTarget(rawTarget);
|
const { path } = parseObsidianTarget(rawTarget);
|
||||||
if (!path) return raw;
|
if (!path) return raw;
|
||||||
|
|
||||||
const assetPath = getImageFullPath(filePath, path);
|
const resolvedAsset = assetLookup.resolve(path, filePath);
|
||||||
const encodedPath = encodeMarkdownPath(assetPath);
|
const assetPath = resolvedAsset?.path ?? getImageFullPath(filePath, path);
|
||||||
|
const encodedPath = encodeMarkdownPath(
|
||||||
|
resolvedAsset ? `/${assetPath}` : path
|
||||||
|
);
|
||||||
|
|
||||||
if (isImageAssetPath(path)) {
|
if (isImageAssetPath(path)) {
|
||||||
const alt = getEmbedLabel(rawAlias, path, false);
|
const alt = getEmbedLabel(rawAlias, path, false);
|
||||||
@@ -525,7 +593,7 @@ function preprocessObsidianEmbeds(
|
|||||||
}
|
}
|
||||||
|
|
||||||
const label = getEmbedLabel(rawAlias, path, true);
|
const label = getEmbedLabel(rawAlias, path, true);
|
||||||
const blobId = pathBlobIdMap.get(assetPath);
|
const blobId = resolvedAsset?.blobId;
|
||||||
if (!blobId) return `[${escapeMarkdownLabel(label)}](${encodedPath})`;
|
if (!blobId) return `[${escapeMarkdownLabel(label)}](${encodedPath})`;
|
||||||
|
|
||||||
const extension = path.split('.').at(-1)?.toLowerCase() ?? '';
|
const extension = path.split('.').at(-1)?.toLowerCase() ?? '';
|
||||||
@@ -541,7 +609,7 @@ function preprocessObsidianMarkdown(
|
|||||||
markdown: string,
|
markdown: string,
|
||||||
filePath: string,
|
filePath: string,
|
||||||
pageLookupMap: ReadonlyMap<string, string>,
|
pageLookupMap: ReadonlyMap<string, string>,
|
||||||
pathBlobIdMap: ReadonlyMap<string, string>
|
assetLookup: ObsidianAssetLookup
|
||||||
): string {
|
): string {
|
||||||
const { content: contentWithoutFootnotes, footnotes: extractedFootnotes } =
|
const { content: contentWithoutFootnotes, footnotes: extractedFootnotes } =
|
||||||
extractObsidianFootnotes(markdown);
|
extractObsidianFootnotes(markdown);
|
||||||
@@ -549,7 +617,7 @@ function preprocessObsidianMarkdown(
|
|||||||
contentWithoutFootnotes,
|
contentWithoutFootnotes,
|
||||||
filePath,
|
filePath,
|
||||||
pageLookupMap,
|
pageLookupMap,
|
||||||
pathBlobIdMap
|
assetLookup
|
||||||
);
|
);
|
||||||
const normalizedMarkdown = preprocessTitleHeader(
|
const normalizedMarkdown = preprocessTitleHeader(
|
||||||
preprocessObsidianCallouts(content)
|
preprocessObsidianCallouts(content)
|
||||||
@@ -696,6 +764,76 @@ export type PlanObsidianVaultResult = ImportObsidianVaultResult & {
|
|||||||
batch: ImportBatch;
|
batch: ImportBatch;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function getVaultRoot(paths: string[]): string {
|
||||||
|
const first = paths[0]?.split('/').find(Boolean);
|
||||||
|
return first && paths.every(path => path.startsWith(`${first}/`))
|
||||||
|
? first
|
||||||
|
: '';
|
||||||
|
}
|
||||||
|
|
||||||
|
function isObsidianConfigPath(path: string): boolean {
|
||||||
|
return normalizeFilePathReference(path)
|
||||||
|
.split('/')
|
||||||
|
.some(segment => segment === '.obsidian');
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAttachmentFolderPath(importedFiles: File[]) {
|
||||||
|
const appConfig = importedFiles.find(file => {
|
||||||
|
const path = normalizeFilePathReference(
|
||||||
|
file.webkitRelativePath || file.name
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
path.endsWith('/.obsidian/app.json') || path === '.obsidian/app.json'
|
||||||
|
);
|
||||||
|
});
|
||||||
|
if (!appConfig) return undefined;
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parsed = JSON.parse(await appConfig.text()) as {
|
||||||
|
attachmentFolderPath?: unknown;
|
||||||
|
};
|
||||||
|
return typeof parsed.attachmentFolderPath === 'string' &&
|
||||||
|
parsed.attachmentFolderPath.trim()
|
||||||
|
? normalizeFilePathReference(parsed.attachmentFolderPath.trim())
|
||||||
|
: undefined;
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildObsidianFolders(
|
||||||
|
markdownFiles: MarkdownFileImportEntry[],
|
||||||
|
vaultRoot: string
|
||||||
|
): ImportFolder[] | undefined {
|
||||||
|
const folders = new Map<string, ImportFolder>();
|
||||||
|
|
||||||
|
for (const file of markdownFiles) {
|
||||||
|
const normalizedPath = normalizeFilePathReference(file.fullPath);
|
||||||
|
const vaultPath =
|
||||||
|
vaultRoot && normalizedPath.startsWith(`${vaultRoot}/`)
|
||||||
|
? normalizedPath.slice(vaultRoot.length + 1)
|
||||||
|
: normalizedPath;
|
||||||
|
const parts = vaultPath.split('/').filter(Boolean);
|
||||||
|
parts.pop();
|
||||||
|
if (parts.length === 0) continue;
|
||||||
|
|
||||||
|
let parentPath: string | undefined;
|
||||||
|
for (const name of parts) {
|
||||||
|
const path = parentPath ? `${parentPath}/${name}` : name;
|
||||||
|
folders.set(path, { path, name, parentPath });
|
||||||
|
parentPath = path;
|
||||||
|
}
|
||||||
|
folders.set(`${parentPath}/__doc__${file.pageId}`, {
|
||||||
|
path: `${parentPath}/__doc__${file.pageId}`,
|
||||||
|
name: `__doc__${file.pageId}`,
|
||||||
|
parentPath,
|
||||||
|
pageId: file.pageId,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return folders.size ? Array.from(folders.values()) : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
export async function planObsidianVault({
|
export async function planObsidianVault({
|
||||||
collection,
|
collection,
|
||||||
schema,
|
schema,
|
||||||
@@ -715,10 +853,17 @@ export async function planObsidianVault({
|
|||||||
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
|
const pendingPathBlobIdMap: PathBlobIdMap = new Map();
|
||||||
const markdownBlobs: MarkdownFileImportEntry[] = [];
|
const markdownBlobs: MarkdownFileImportEntry[] = [];
|
||||||
const pageLookupMap = new Map<string, string>();
|
const pageLookupMap = new Map<string, string>();
|
||||||
|
const importedPaths = importedFiles.map(
|
||||||
|
file => file.webkitRelativePath || file.name
|
||||||
|
);
|
||||||
|
const vaultRoot = getVaultRoot(importedPaths);
|
||||||
|
const attachmentFolderPath = await getAttachmentFolderPath(importedFiles);
|
||||||
|
|
||||||
for (const file of importedFiles) {
|
for (const file of importedFiles) {
|
||||||
const filePath = file.webkitRelativePath || file.name;
|
const filePath = file.webkitRelativePath || file.name;
|
||||||
if (isSystemImportPath(filePath)) continue;
|
if (isSystemImportPath(filePath) || isObsidianConfigPath(filePath)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
if (file.name.endsWith('.md')) {
|
if (file.name.endsWith('.md')) {
|
||||||
const fileNameWithoutExt = file.name.replace(/\.[^/.]+$/, '');
|
const fileNameWithoutExt = file.name.replace(/\.[^/.]+$/, '');
|
||||||
@@ -765,6 +910,12 @@ export async function planObsidianVault({
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const assetLookup = createAssetLookup(
|
||||||
|
pendingPathBlobIdMap,
|
||||||
|
vaultRoot,
|
||||||
|
attachmentFolderPath
|
||||||
|
);
|
||||||
|
|
||||||
for (const existingDocMeta of collection.meta.docMetas) {
|
for (const existingDocMeta of collection.meta.docMetas) {
|
||||||
if (existingDocMeta.title) {
|
if (existingDocMeta.title) {
|
||||||
registerPageLookup(
|
registerPageLookup(
|
||||||
@@ -802,17 +953,13 @@ export async function planObsidianVault({
|
|||||||
job.adapterConfigs.set('obsidian:pageEmoji:' + id, emoji);
|
job.adapterConfigs.set('obsidian:pageEmoji:' + id, emoji);
|
||||||
}
|
}
|
||||||
|
|
||||||
const pathBlobIdMap = bindImportedAssetsToJob(
|
bindImportedAssetsToJob(job, pendingAssets, pendingPathBlobIdMap);
|
||||||
job,
|
|
||||||
pendingAssets,
|
|
||||||
pendingPathBlobIdMap
|
|
||||||
);
|
|
||||||
|
|
||||||
const preprocessedMarkdown = preprocessObsidianMarkdown(
|
const preprocessedMarkdown = preprocessObsidianMarkdown(
|
||||||
content,
|
content,
|
||||||
fullPath,
|
fullPath,
|
||||||
pageLookupMap,
|
pageLookupMap,
|
||||||
pathBlobIdMap
|
assetLookup
|
||||||
);
|
);
|
||||||
const mdAdapter = new MarkdownAdapter(job, provider);
|
const mdAdapter = new MarkdownAdapter(job, provider);
|
||||||
const snapshot = await mdAdapter.toDocSnapshot({
|
const snapshot = await mdAdapter.toDocSnapshot({
|
||||||
@@ -838,6 +985,7 @@ export async function planObsidianVault({
|
|||||||
batch: {
|
batch: {
|
||||||
docs,
|
docs,
|
||||||
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
|
blobs: await blobsFromAssets(pendingAssets, pendingPathBlobIdMap),
|
||||||
|
folders: buildObsidianFolders(markdownBlobs, vaultRoot),
|
||||||
icons: Array.from(docEmojis, ([docId, emoji]) => ({
|
icons: Array.from(docEmojis, ([docId, emoji]) => ({
|
||||||
docId,
|
docId,
|
||||||
icon: { type: 'emoji', unicode: emoji },
|
icon: { type: 'emoji', unicode: emoji },
|
||||||
|
|||||||
@@ -92,6 +92,7 @@ export class ImportService extends Service {
|
|||||||
async importObsidianVault(files: File[], context?: ImportRunContext) {
|
async importObsidianVault(files: File[], context?: ImportRunContext) {
|
||||||
const collection = this.workspaceService.workspace.docCollection;
|
const collection = this.workspaceService.workspace.docCollection;
|
||||||
const commitService = this.createCommitService({
|
const commitService = this.createCommitService({
|
||||||
|
organize: true,
|
||||||
explorerIcon: true,
|
explorerIcon: true,
|
||||||
});
|
});
|
||||||
if (!BUILD_CONFIG.isElectron) {
|
if (!BUILD_CONFIG.isElectron) {
|
||||||
|
|||||||
Reference in New Issue
Block a user