diff --git a/main.ts b/main.ts index ad575c1..6dff13a 100644 --- a/main.ts +++ b/main.ts @@ -1,14 +1,17 @@ import { Plugin } from "obsidian"; -import { PluginSettings, DEFAULT_SETTINGS } from "./src/types"; +import { PluginSettings, DEFAULT_SETTINGS, SyncProgressSnapshot } from "./src/types"; import { RemarkableClient } from "./src/RemarkableClient"; import { SyncEngine } from "./src/SyncEngine"; import { RemarkableSyncSettingTab } from "./src/SettingsTab"; +import { SyncProgressModal } from "./src/SyncProgressModal"; import { EpubView, EPUB_VIEW_TYPE } from "./src/EpubView"; export default class RemarkableSyncPlugin extends Plugin { settings!: PluginSettings; private client: RemarkableClient | null = null; private syncEngine: SyncEngine | null = null; + private statusBarItemEl!: HTMLElement; + private syncProgressModal: SyncProgressModal | null = null; async onload(): Promise { await this.loadPluginSettings(); @@ -25,9 +28,18 @@ export default class RemarkableSyncPlugin extends Plugin { // Status bar const statusBarItem = this.addStatusBarItem(); + this.statusBarItemEl = statusBarItem; statusBarItem.addClass("mod-clickable"); - statusBarItem.setText("Slate"); - statusBarItem.addEventListener("click", () => { void this.runSync(); }); + statusBarItem.addEventListener("click", () => { + if (this.syncEngine?.syncing) { + this.showSyncProgress(); + return; + } + + void this.runSync(); + }); + this.updateStatusBar(); + this.registerInterval(window.setInterval(() => this.updateStatusBar(), 1000)); // Commands this.addCommand({ @@ -78,9 +90,54 @@ export default class RemarkableSyncPlugin extends Plugin { if (!this.syncEngine) { return; } + + if (this.syncEngine.syncing) { + this.showSyncProgress(); + return; + } + + this.showSyncProgress(); await this.syncEngine.sync(); } + getSyncProgress(): SyncProgressSnapshot | null { + return this.syncEngine?.getProgressSnapshot() ?? null; + } + + private showSyncProgress(): void { + if (!this.syncProgressModal) { + this.syncProgressModal = new SyncProgressModal(this.app, this, () => { + this.syncProgressModal = null; + }); + } + + this.syncProgressModal.open(); + } + + private updateStatusBar(): void { + if (!this.statusBarItemEl) { + return; + } + + const progress = this.getSyncProgress(); + if (!this.syncEngine?.syncing || !progress) { + this.statusBarItemEl.setText("Slate"); + return; + } + + if (progress.phase === "Listing cloud items") { + this.statusBarItemEl.setText(`Slate ${progress.inspectedItemCount}/${progress.cloudItemCount}`); + return; + } + + if (progress.documentCount > 0) { + this.statusBarItemEl.setText(`Slate ${progress.processedDocumentCount}/${progress.documentCount}`); + return; + } + + this.statusBarItemEl.setText(`Slate ${progress.phase}`); + } + // ── Settings Persistence ──────────────────────────────────────────────── async loadPluginSettings(): Promise { diff --git a/response.txt b/response.txt deleted file mode 100644 index 792b6d9..0000000 --- a/response.txt +++ /dev/null @@ -1,10 +0,0 @@ - - - -500 Server Error - - -

Error: Server Error

-

The server encountered an error and could not complete your request.

Please try again in 30 seconds.

-

- diff --git a/response2.txt b/response2.txt deleted file mode 100644 index 792b6d9..0000000 --- a/response2.txt +++ /dev/null @@ -1,10 +0,0 @@ - - - -500 Server Error - - -

Error: Server Error

-

The server encountered an error and could not complete your request.

Please try again in 30 seconds.

-

- diff --git a/src/RemarkableClient.ts b/src/RemarkableClient.ts index 15d82ac..70f1e49 100644 --- a/src/RemarkableClient.ts +++ b/src/RemarkableClient.ts @@ -2,8 +2,6 @@ import { requestUrl, RequestUrlResponse } from "obsidian"; import { RawEntry, EntriesFile, - ItemMetadata, - RemarkableItem, RootHashResponse, } from "./types"; import { @@ -13,6 +11,9 @@ import { } from "./constants"; const TAG = "[RemarkableSync]"; +const MAX_RATE_LIMIT_RETRIES = 5; +const MAX_SERVER_RETRIES = 3; +const BASE_DELAY_MS = 1000; async function rmRequest(opts: { url: string; @@ -23,13 +24,65 @@ async function rmRequest(opts: { }): Promise { const { url, method = "GET", headers, body, contentType } = opts; - try { - return await requestUrl({ url, method, headers, body, contentType }); - } catch (err: unknown) { - const statusCode = (err as { status?: number })?.status; - const shortUrl = url.split("?")[0].slice(0, 80); - console.error(`${TAG} ${method} ${shortUrl} → ${statusCode ?? "network_error"}`, err); - throw new RmApiError(statusCode ? String(statusCode) : "network_error", statusCode ?? 0); + for (let attempt = 0; ; attempt++) { + try { + return await requestUrl({ url, method, headers, body, contentType }); + } catch (err: unknown) { + const statusCode = (err as { status?: number })?.status; + const shortUrl = url.split("?")[0].slice(0, 80); + const retryLimit = getRetryLimit(statusCode); + + if (retryLimit > 0 && attempt < retryLimit) { + const delay = BASE_DELAY_MS * Math.pow(2, attempt); + console.warn( + `${TAG} ${method} ${shortUrl} → ${statusCode}, retry ${attempt + 1}/${retryLimit} in ${delay}ms`, + ); + await sleep(delay); + continue; + } + + console.error(`${TAG} ${method} ${shortUrl} → ${statusCode ?? "network_error"}`, err); + throw new RmApiError(statusCode ? String(statusCode) : "network_error", statusCode ?? 0); + } + } +} + +function getRetryLimit(statusCode?: number): number { + if (statusCode === 429) { + return MAX_RATE_LIMIT_RETRIES; + } + + if (statusCode === 500 || statusCode === 502 || statusCode === 503) { + return MAX_SERVER_RETRIES; + } + + return 0; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => window.setTimeout(resolve, ms)); +} + +function isHtmlResponse(text: string): boolean { + const normalized = text.trimStart().toLowerCase(); + return normalized.startsWith(" Promise, hash: string): Promise { + for (let attempt = 0; ; attempt++) { + const response = await requestFn(); + if (!isHtmlResponse(response.text)) { + return response.text; + } + + if (attempt < MAX_SERVER_RETRIES) { + const delay = BASE_DELAY_MS * Math.pow(2, attempt); + console.warn(`${TAG} GET file/${hash.slice(0, 12)}... returned HTML, retry ${attempt + 1}/${MAX_SERVER_RETRIES} in ${delay}ms`); + await sleep(delay); + continue; + } + + throw new RmApiError(`html_error_response:${hash}`, 500); } } @@ -114,13 +167,18 @@ export class RemarkableClient { } async getEntries(hash: string): Promise { - const response = await this.authedRequest(`${RM_RAW_HOST}/sync/v3/files/${hash}`); - return parseEntriesText(response.text); + const text = await getTextWithHtmlRetry( + () => this.authedRequest(`${RM_RAW_HOST}/sync/v3/files/${hash}`), + hash, + ); + return parseEntriesText(text); } async getTextByHash(hash: string): Promise { - const response = await this.authedRequest(`${RM_RAW_HOST}/sync/v3/files/${hash}`); - return response.text; + return await getTextWithHtmlRetry( + () => this.authedRequest(`${RM_RAW_HOST}/sync/v3/files/${hash}`), + hash, + ); } async getBinaryByHash(hash: string): Promise { @@ -128,55 +186,6 @@ export class RemarkableClient { return response.arrayBuffer; } - async listItems(): Promise { - if (!this.userToken) { - await this.refreshToken(); - } - - const root = await this.getRootHash(); - const rootEntries = await this.getEntries(root.hash); - - const items: RemarkableItem[] = []; - - for (const entry of rootEntries.entries) { - try { - const itemEntries = await this.getEntries(entry.hash); - const fileEntries = itemEntries.entries; - - const metaEntry = fileEntries.find(e => e.id.endsWith(".metadata")); - if (!metaEntry) { - console.warn(`${TAG} No metadata for entry ${entry.id}, skipping`); - continue; - } - - const metaText = await this.getTextByHash(metaEntry.hash); - const metadata = JSON.parse(metaText) as ItemMetadata; - - if (metadata.deleted) continue; - - if (metadata.type !== "DocumentType" && metadata.type !== "CollectionType") { - continue; - } - - items.push({ - id: entry.id, - hash: entry.hash, - visibleName: metadata.visibleName, - lastModified: metadata.lastModified, - parent: metadata.parent, - pinned: metadata.pinned, - type: metadata.type, - fileEntries, - }); - } catch (err) { - console.warn(`${TAG} Failed to read entry ${entry.id}:`, err); - } - } - - - return items; - } - get isRegistered(): boolean { return !!this.deviceToken; } get isAuthenticated(): boolean { return !!this.userToken; } diff --git a/src/SettingsTab.ts b/src/SettingsTab.ts index c5b60aa..363c39d 100644 --- a/src/SettingsTab.ts +++ b/src/SettingsTab.ts @@ -1,6 +1,7 @@ -import { App, Notice, PluginSettingTab, Setting } from "obsidian"; +import { App, Notice, PluginSettingTab, Setting, TFile } from "obsidian"; import type RemarkableSyncPlugin from "../main"; import { generateDeviceId, RemarkableClient } from "./RemarkableClient"; +import { SyncReport } from "./types"; export class RemarkableSyncSettingTab extends PluginSettingTab { constructor(app: App, private plugin: RemarkableSyncPlugin) { @@ -21,7 +22,7 @@ export class RemarkableSyncSettingTab extends PluginSettingTab { if (this.plugin.settings.deviceToken) { statusEl.createEl("p", { - text: "Connected to remarkable cloud", + text: "Connected to remarkable cloud", cls: "remarkable-sync-connected", }); statusEl.createEl("p", { @@ -46,14 +47,14 @@ export class RemarkableSyncSettingTab extends PluginSettingTab { btn .setButtonText("Disconnect") .setWarning() - .onClick(async () => { - this.plugin.settings.deviceToken = ""; - this.plugin.settings.deviceId = ""; - this.plugin.settings.syncState = {}; - await this.plugin.savePluginSettings(); + .onClick(async () => { + this.plugin.settings.deviceToken = ""; + this.plugin.settings.deviceId = ""; + this.plugin.settings.syncState = {}; + await this.plugin.savePluginSettings(); new Notice("Disconnected from remarkable cloud."); - this.display(); // Refresh - }) + this.display(); // Refresh + }) ); } else { const descFrag = document.createDocumentFragment(); @@ -77,8 +78,8 @@ export class RemarkableSyncSettingTab extends PluginSettingTab { }); text.inputEl.addEventListener("keydown", (e) => { if (e.key === "Enter") { - void registerDevice(this.plugin, codeValue, this); - } + void registerDevice(this.plugin, codeValue, this); + } }); }) .addButton((btn) => @@ -168,8 +169,85 @@ export class RemarkableSyncSettingTab extends PluginSettingTab { this.display(); }) ); + + new Setting(containerEl).setName("Last sync results").setHeading(); + renderLastSyncResults(containerEl, this.plugin, this.app); + } + } +} + +function renderLastSyncResults( + containerEl: HTMLElement, + plugin: RemarkableSyncPlugin, + app: App, +): void { + const report: SyncReport | undefined = plugin.settings.lastSyncReport; + if (!report) { + containerEl.createEl("p", { + text: "No sync report available yet.", + cls: "setting-item-description", + }); + return; + } + + const summaryEl = containerEl.createDiv({ cls: "setting-item-description" }); + summaryEl.createEl("p", { + text: `Completed: ${new Date(report.endTime).toLocaleString()}`, + }); + summaryEl.createEl("p", { + text: `Duration: ${formatDuration(report.durationMs)}`, + }); + summaryEl.createEl("p", { + text: `Cloud items: ${report.cloudItemCount} | Listed: ${report.listedCount} | Synced: ${report.syncedCount} | Unchanged: ${report.skippedCount} | Failed: ${report.failedCount}`, + }); + + if (report.fatalError) { + summaryEl.createEl("p", { + text: `Fatal error: ${report.fatalError}`, + cls: "remarkable-sync-disconnected", + }); + } + + if (report.failedItems.length > 0) { + summaryEl.createEl("p", { text: "Failed items:" }); + const failedListEl = summaryEl.createEl("ul"); + for (const item of report.failedItems) { + failedListEl.createEl("li", { + text: `${item.name}: ${item.error}`, + }); } } + + new Setting(containerEl) + .setName("Sync log") + .setDesc(report.logPath ?? getSyncLogPath(plugin.settings.syncFolder)) + .addButton((btn) => + btn + .setButtonText("Open") + .onClick(async () => { + const logPath = report.logPath ?? getSyncLogPath(plugin.settings.syncFolder); + const file = app.vault.getAbstractFileByPath(logPath); + if (!(file instanceof TFile)) { + new Notice("Sync log not found."); + return; + } + + await app.workspace.getLeaf(true).openFile(file); + }) + ); +} + +function getSyncLogPath(syncFolder: string): string { + const base = syncFolder || "remarkable"; + return `${base}/.sync-log.md`; +} + +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + return `${(durationMs / 1000).toFixed(1)}s`; } async function registerDevice( diff --git a/src/SyncEngine.ts b/src/SyncEngine.ts index b275bad..7c43d93 100644 --- a/src/SyncEngine.ts +++ b/src/SyncEngine.ts @@ -3,17 +3,41 @@ import { RemarkableClient } from "./RemarkableClient"; import { parseRmFile } from "./RmParser"; import { generatePdf, overlayAnnotations } from "./PdfGenerator"; import { generateMarkdown } from "./MarkdownGenerator"; +import { SyncLogger } from "./SyncLogger"; import { + RawEntry, + ItemMetadata, RemarkableItem, DocumentContent, RmPage, ParsedDocument, PluginSettings, + SyncFailure, + SyncProgressSnapshot, + SyncRecord, + SyncReport, + SyncSkip, } from "./types"; +type SyncDocumentResult = + | { status: "synced" } + | { status: "skipped"; reason: string; kind: SyncSkip["kind"] }; + +interface LocalDocumentRecord { + lastModified: string; + vaultPath: string; +} + +interface QueuedDocument { + doc: RemarkableItem; + existingLocalDocument: boolean; +} + +const MAX_PROGRESS_ITEMS = 8; export class SyncEngine { private isSyncing = false; + private progress: SyncProgressSnapshot | null = null; constructor( private client: RemarkableClient, @@ -26,9 +50,21 @@ export class SyncEngine { return this.isSyncing; } + getProgressSnapshot(): SyncProgressSnapshot | null { + if (!this.progress) { + return null; + } + + return { + ...this.progress, + recentCompleted: this.progress.recentCompleted.map((item) => ({ ...item })), + recentSkipped: this.progress.recentSkipped.map((item) => ({ ...item })), + recentFailures: this.progress.recentFailures.map((item) => ({ ...item })), + }; + } + async sync(): Promise { if (this.isSyncing) { - new Notice("Slate: already syncing..."); return; } @@ -38,59 +74,133 @@ export class SyncEngine { } this.isSyncing = true; + this.startProgress(); + const syncLogger = new SyncLogger(); + syncLogger.setLogPath(this.getSyncLogPath()); try { + await this.ensureFolderExists(this.getSyncRootPath()); + const localDocumentIndex = await this.buildLocalDocumentIndex(); + + this.setProgressPhase("Authenticating"); new Notice("Slate: authenticating..."); await this.client.refreshToken(); + this.setProgressPhase("Listing cloud items"); new Notice("Slate: fetching document list..."); - const items = await this.client.listItems(); + const root = await this.client.getRootHash(); + const rootEntries = await this.client.getEntries(root.hash); + syncLogger.setCloudItemCount(rootEntries.entries.length); + this.setCloudItemCount(rootEntries.entries.length); - const documents = items.filter(i => i.type === "DocumentType"); - const allItems = items; + const allItems: RemarkableItem[] = []; + const documentsToSync: QueuedDocument[] = []; - let syncedCount = 0; - let errorCount = 0; + for (const entry of rootEntries.entries) { + const cached = this.settings.syncState[entry.id]; + this.hydrateCachedVaultPath(cached, localDocumentIndex.get(entry.id)); + const cachedItem = this.getCachedItem(entry, cached); + this.setProgressPhase("Listing cloud items", cachedItem?.visibleName ?? cached?.visibleName ?? entry.id); - for (const doc of documents) { - const existing = this.settings.syncState[doc.id]; - if (existing && existing.hash === doc.hash) { - const safeName = sanitizeFilename(doc.visibleName); - const mdPath = `${existing.vaultPath}/${safeName}.md`; - const localExists = !!this.vault.getAbstractFileByPath(mdPath); - if (localExists) continue; + try { + if (cachedItem?.type === "CollectionType") { + allItems.push(cachedItem); + syncLogger.logListed(cachedItem.id, cachedItem.visibleName); + this.recordListedFromCache(cachedItem.type); + continue; + } + + if (cachedItem?.type === "DocumentType" && this.hasLocalMarkdown(cachedItem, cached)) { + allItems.push(cachedItem); + syncLogger.logListed(cachedItem.id, cachedItem.visibleName); + syncLogger.logSkipped(cachedItem.id, cachedItem.visibleName); + this.recordListedFromCache(cachedItem.type); + this.recordDocumentQueued(); + this.recordSkipped( + cachedItem.id, + cachedItem.visibleName, + "unchanged (already synced)", + "cached_unchanged", + ); + continue; + } + + const item = await this.fetchItem(entry); + if (!item) { + continue; + } + + allItems.push(item); + syncLogger.logListed(item.id, item.visibleName); + this.recordListedFromCloud(); + this.cacheItem(item); + + if (item.type === "DocumentType") { + this.recordDocumentQueued(); + const localDocument = localDocumentIndex.get(item.id); + const existingLocalDocument = this.hasExistingLocalDocument(item, cached, localDocument); + if (this.canReuseLocalDocument(item, localDocument)) { + this.settings.syncState[item.id].vaultPath = localDocument.vaultPath; + syncLogger.logSkipped(item.id, item.visibleName); + this.recordSkipped( + item.id, + item.visibleName, + "unchanged (reused local markdown)", + "reused_local_markdown", + ); + continue; + } + documentsToSync.push({ doc: item, existingLocalDocument }); + } + } catch (err) { + const name = cached?.visibleName ?? entry.id; + console.warn(`[RemarkableSync] Failed to read entry ${entry.id}:`, err); + syncLogger.logFailed(entry.id, name, this.getErrorMessage(err)); + this.recordFailure(entry.id, name, this.getErrorMessage(err), false); + } finally { + this.recordInspectedItem(); + } } + this.setProgressPhase("Syncing documents"); + for (const queuedDocument of documentsToSync) { + const { doc, existingLocalDocument } = queuedDocument; + this.setProgressPhase("Syncing documents", doc.visibleName); try { - await this.syncDocument(doc, allItems); - syncedCount++; + const result = await this.syncDocument(doc, allItems); + if (result.status === "synced") { + syncLogger.logSynced(doc.id, doc.visibleName); + this.recordSynced( + doc.visibleName, + existingLocalDocument ? "redownloaded" : "new_download", + ); + } else { + syncLogger.logSkipped(doc.id, doc.visibleName); + this.recordSkipped(doc.id, doc.visibleName, result.reason, result.kind); + } } catch (err) { console.error(`Failed to sync "${doc.visibleName}":`, err); - errorCount++; + syncLogger.logFailed(doc.id, doc.visibleName, this.getErrorMessage(err)); + this.recordFailure(doc.id, doc.visibleName, this.getErrorMessage(err)); } } - const cloudIds = new Set(documents.map(d => d.id)); + const cloudIds = new Set(rootEntries.entries.map((entry) => entry.id)); for (const id of Object.keys(this.settings.syncState)) { if (!cloudIds.has(id)) { delete this.settings.syncState[id]; } } - this.settings.lastSyncTimestamp = Date.now(); - await this.saveSettings(); - - if (syncedCount === 0 && errorCount === 0) { - new Notice("Slate: everything is up to date."); - } else { - const msg = []; - if (syncedCount > 0) msg.push(`${syncedCount} synced`); - if (errorCount > 0) msg.push(`${errorCount} failed`); - new Notice(`Slate: ${msg.join(", ")}.`); - } + const report = await this.finalizeSync(syncLogger); + this.finishProgress(report); + new Notice(syncLogger.formatNotice(report)); } catch (err) { + syncLogger.logSessionFailure(this.getErrorMessage(err)); console.error("Slate error:", err); - new Notice(`Slate failed: ${(err as Error).message}`); + const report = await this.finalizeSync(syncLogger); + this.finishProgress(report); + new Notice(syncLogger.formatNotice(report)); } finally { this.isSyncing = false; } @@ -99,9 +209,7 @@ export class SyncEngine { private async syncDocument( doc: RemarkableItem, allItems: RemarkableItem[], - ): Promise { - - + ): Promise { const contentEntry = doc.fileEntries.find(e => e.id.endsWith(".content")); let content: DocumentContent = { fileType: "", @@ -125,7 +233,7 @@ export class SyncEngine { } if (this.settings.excludePdfs && content.fileType === "pdf") { - return; + return { status: "skipped", reason: "excluded PDF", kind: "excluded_pdf" }; } let basePdf: ArrayBuffer | null = null; @@ -182,7 +290,7 @@ export class SyncEngine { } if (!pdfBytes && !baseEpub && rawRmFiles.size === 0) { - return; + return { status: "skipped", reason: "no supported content", kind: "no_supported_content" }; } if (pdfBytes) { @@ -214,14 +322,91 @@ export class SyncEngine { const pdfRelPath = pdfBytes ? `attachments/${safeName}.pdf` : ""; const mdContent = generateMarkdown(parsed, pdfRelPath, epubRelPath); - await this.writeFile(mdPath, new TextEncoder().encode(mdContent).buffer); + await this.writeFile(mdPath, new TextEncoder().encode(mdContent).buffer as ArrayBuffer); this.settings.syncState[doc.id] = { hash: doc.hash, lastModified: doc.lastModified, vaultPath: vaultFolderPath, + visibleName: doc.visibleName, + parent: doc.parent, + type: doc.type, }; await this.saveSettings(); + + return { status: "synced" }; + } + + private async fetchItem(entry: RawEntry): Promise { + const itemEntries = await this.client.getEntries(entry.hash); + const fileEntries = itemEntries.entries; + + const metaEntry = fileEntries.find((fileEntry) => fileEntry.id.endsWith(".metadata")); + if (!metaEntry) { + console.warn(`[RemarkableSync] No metadata for entry ${entry.id}, skipping`); + return null; + } + + const metaText = await this.client.getTextByHash(metaEntry.hash); + const metadata = JSON.parse(metaText) as ItemMetadata; + + if (metadata.deleted) { + return null; + } + + if (metadata.type !== "DocumentType" && metadata.type !== "CollectionType") { + return null; + } + + return { + id: entry.id, + hash: entry.hash, + visibleName: metadata.visibleName, + lastModified: metadata.lastModified, + parent: metadata.parent, + pinned: metadata.pinned, + type: metadata.type, + fileEntries, + }; + } + + private getCachedItem(entry: RawEntry, cached?: SyncRecord): RemarkableItem | null { + if (!cached || cached.hash !== entry.hash || !cached.visibleName || !cached.type) { + return null; + } + + return { + id: entry.id, + hash: entry.hash, + visibleName: cached.visibleName, + lastModified: cached.lastModified, + parent: cached.parent ?? "", + pinned: false, + type: cached.type, + fileEntries: [], + }; + } + + private cacheItem(item: RemarkableItem): void { + const existing = this.settings.syncState[item.id]; + this.settings.syncState[item.id] = { + hash: item.hash, + lastModified: item.lastModified, + vaultPath: existing?.vaultPath ?? "", + visibleName: item.visibleName, + parent: item.parent, + type: item.type, + }; + } + + private hasLocalMarkdown(item: RemarkableItem, cached?: SyncRecord): boolean { + if (!cached?.vaultPath) { + return false; + } + + const safeName = sanitizeFilename(item.visibleName); + const mdPath = `${cached.vaultPath}/${safeName}.md`; + return !!this.vault.getAbstractFileByPath(mdPath); } private getPageOrder(content: DocumentContent, doc: RemarkableItem): string[] { @@ -291,6 +476,301 @@ export class SyncEngine { await this.vault.createBinary(path, data); } } + + private getSyncRootPath(): string { + return this.settings.syncFolder || "remarkable"; + } + + private getSyncLogPath(): string { + return `${this.getSyncRootPath()}/.sync-log.md`; + } + + private async buildLocalDocumentIndex(): Promise> { + if (!this.shouldBuildLocalDocumentIndex()) { + return new Map(); + } + + const index = new Map(); + const prefix = `${this.getSyncRootPath()}/`; + + for (const file of this.vault.getMarkdownFiles()) { + if (!file.path.startsWith(prefix)) { + continue; + } + + const content = await this.vault.cachedRead(file); + const remarkableId = extractFrontmatterValue(content, "remarkable_id"); + const lastModified = extractFrontmatterValue(content, "last_modified"); + + if (!remarkableId || !lastModified) { + continue; + } + + index.set(remarkableId, { + lastModified, + vaultPath: getParentPath(file.path), + }); + } + + return index; + } + + private shouldBuildLocalDocumentIndex(): boolean { + if (this.settings.lastSyncTimestamp === 0) { + return true; + } + + for (const record of Object.values(this.settings.syncState)) { + if (record.type === "DocumentType" && !record.vaultPath) { + return true; + } + } + + return false; + } + + private hydrateCachedVaultPath( + cached: SyncRecord | undefined, + localDocument: LocalDocumentRecord | undefined, + ): void { + if (!cached || cached.vaultPath || !localDocument) { + return; + } + + if (cached.lastModified === localDocument.lastModified) { + cached.vaultPath = localDocument.vaultPath; + } + } + + private canReuseLocalDocument( + item: RemarkableItem, + localDocument: LocalDocumentRecord | undefined, + ): localDocument is LocalDocumentRecord { + return !!localDocument && item.lastModified === localDocument.lastModified; + } + + private hasExistingLocalDocument( + item: RemarkableItem, + cached: SyncRecord | undefined, + localDocument: LocalDocumentRecord | undefined, + ): boolean { + if (localDocument) { + return true; + } + + return this.hasLocalMarkdown(item, cached); + } + + private startProgress(): void { + this.progress = { + phase: "Starting", + startedAt: Date.now(), + cloudItemCount: 0, + inspectedItemCount: 0, + documentCount: 0, + processedDocumentCount: 0, + listedCount: 0, + listedFromCacheCount: 0, + listedFromCloudCount: 0, + cachedCollectionCount: 0, + cachedDocumentCount: 0, + syncedCount: 0, + newDownloadCount: 0, + redownloadCount: 0, + skippedCount: 0, + cachedSkipCount: 0, + reusedLocalSkipCount: 0, + excludedPdfSkipCount: 0, + unsupportedContentSkipCount: 0, + otherSkipCount: 0, + failedCount: 0, + recentCompleted: [], + recentSkipped: [], + recentFailures: [], + logPath: this.getSyncLogPath(), + }; + } + + private setProgressPhase(phase: string, currentItem?: string): void { + if (!this.progress) { + return; + } + + this.progress.phase = phase; + this.progress.currentItem = currentItem; + } + + private setCloudItemCount(count: number): void { + if (this.progress) { + this.progress.cloudItemCount = count; + } + } + + private recordInspectedItem(): void { + if (this.progress) { + this.progress.inspectedItemCount += 1; + } + } + + private recordListedFromCache(type: RemarkableItem["type"]): void { + if (!this.progress) { + return; + } + + this.progress.listedCount += 1; + this.progress.listedFromCacheCount += 1; + if (type === "CollectionType") { + this.progress.cachedCollectionCount += 1; + } else { + this.progress.cachedDocumentCount += 1; + } + } + + private recordListedFromCloud(): void { + if (!this.progress) { + return; + } + + this.progress.listedCount += 1; + this.progress.listedFromCloudCount += 1; + } + + private recordDocumentQueued(): void { + if (this.progress) { + this.progress.documentCount += 1; + } + } + + private recordSynced(name: string, kind: "new_download" | "redownloaded"): void { + if (!this.progress) { + return; + } + + this.progress.processedDocumentCount += 1; + this.progress.syncedCount += 1; + if (kind === "new_download") { + this.progress.newDownloadCount += 1; + } else { + this.progress.redownloadCount += 1; + } + + this.pushRecentCompleted(name, kind); + } + + private recordSkipped(id: string, name: string, reason: string, kind: SyncSkip["kind"]): void { + if (!this.progress) { + return; + } + + this.progress.processedDocumentCount += 1; + this.progress.skippedCount += 1; + switch (kind) { + case "cached_unchanged": + this.progress.cachedSkipCount += 1; + break; + case "reused_local_markdown": + this.progress.reusedLocalSkipCount += 1; + break; + case "excluded_pdf": + this.progress.excludedPdfSkipCount += 1; + break; + case "no_supported_content": + this.progress.unsupportedContentSkipCount += 1; + break; + default: + this.progress.otherSkipCount += 1; + } + + this.pushRecentSkipped({ id, name, reason, kind }); + } + + private recordFailure(id: string, name: string, error: string, countsAsDocument = true): void { + if (!this.progress) { + return; + } + + if (countsAsDocument) { + this.progress.processedDocumentCount += 1; + } + + this.progress.failedCount += 1; + this.pushRecentFailure({ id, name, error }); + } + + private pushRecentCompleted(name: string, kind: "new_download" | "redownloaded"): void { + if (!this.progress) { + return; + } + + this.progress.recentCompleted = [{ name, kind }, ...this.progress.recentCompleted].slice(0, MAX_PROGRESS_ITEMS); + } + + private pushRecentSkipped(skip: SyncSkip): void { + if (!this.progress) { + return; + } + + this.progress.recentSkipped = [skip, ...this.progress.recentSkipped].slice(0, MAX_PROGRESS_ITEMS); + } + + private pushRecentFailure(failure: SyncFailure): void { + if (!this.progress) { + return; + } + + this.progress.recentFailures = [failure, ...this.progress.recentFailures].slice(0, MAX_PROGRESS_ITEMS); + } + + private finishProgress(report: SyncReport): void { + if (!this.progress) { + return; + } + + this.progress.phase = report.fatalError ? "Failed" : "Completed"; + this.progress.currentItem = undefined; + this.progress.listedCount = report.listedCount; + this.progress.syncedCount = report.syncedCount; + this.progress.skippedCount = report.skippedCount; + this.progress.failedCount = report.failedCount; + this.progress.fatalError = report.fatalError; + this.progress.logPath = report.logPath ?? this.progress.logPath; + } + + private async finalizeSync(syncLogger: SyncLogger): Promise { + this.settings.lastSyncTimestamp = Date.now(); + const report = syncLogger.getReport(); + this.settings.lastSyncReport = report; + + try { + await this.writeSyncLog(syncLogger, report); + } catch (err) { + console.error("Slate log write error:", err); + } + + await this.saveSettings(); + return report; + } + + private async writeSyncLog(syncLogger: SyncLogger, report: SyncReport): Promise { + const logPath = this.getSyncLogPath(); + const existing = this.vault.getAbstractFileByPath(logPath); + + if (existing instanceof TFile) { + const currentContent = await this.vault.read(existing); + await this.vault.modify(existing, syncLogger.updateLogContent(currentContent, report)); + return; + } + + await this.vault.create(logPath, syncLogger.createLogContent(report)); + } + + private getErrorMessage(err: unknown): string { + if (err instanceof Error && err.message) { + return err.message; + } + + return "Unknown error"; + } } function sanitizeFilename(name: string): string { @@ -299,3 +779,22 @@ function sanitizeFilename(name: string): string { .replace(/\s+/g, " ") .trim(); } + +function extractFrontmatterValue(content: string, key: string): string | null { + const frontmatterMatch = content.match(/^---\n([\s\S]*?)\n---/); + if (!frontmatterMatch) { + return null; + } + + const valueMatch = frontmatterMatch[1].match(new RegExp(`^${key}:\\s*"([^"]*)"$`, "m")); + return valueMatch?.[1] ?? null; +} + +function getParentPath(path: string): string { + const lastSlash = path.lastIndexOf("/"); + if (lastSlash === -1) { + return ""; + } + + return path.slice(0, lastSlash); +} diff --git a/src/SyncLogger.ts b/src/SyncLogger.ts new file mode 100644 index 0000000..1c4a85f --- /dev/null +++ b/src/SyncLogger.ts @@ -0,0 +1,166 @@ +import { SyncFailure, SyncReport } from "./types"; + +type SyncAction = "listed" | "synced" | "skipped" | "failed"; + +interface SyncEvent { + id: string; + name: string; + action: SyncAction; + error?: string; +} + +const LOG_HEADER = "# Slate Sync Log\n\nNewest entries appear first.\n"; +const MAX_LOG_ENTRIES = 50; +const MAX_NOTICE_FAILURES = 3; + +export class SyncLogger { + private readonly startTime = Date.now(); + private events: SyncEvent[] = []; + private cloudItemCount = 0; + private fatalError: string | null = null; + private logPath = ""; + + setCloudItemCount(count: number): void { + this.cloudItemCount = count; + } + + setLogPath(path: string): void { + this.logPath = path; + } + + logListed(id: string, name: string): void { + this.events.push({ id, name, action: "listed" }); + } + + logSynced(id: string, name: string): void { + this.events.push({ id, name, action: "synced" }); + } + + logSkipped(id: string, name: string): void { + this.events.push({ id, name, action: "skipped" }); + } + + logFailed(id: string, name: string, error: string): void { + this.events.push({ id, name, action: "failed", error }); + } + + logSessionFailure(error: string): void { + this.fatalError = error; + } + + getReport(): SyncReport { + const endTime = Date.now(); + const failedItems = this.events + .filter((event): event is SyncEvent & { error: string } => event.action === "failed" && !!event.error) + .map((event): SyncFailure => ({ + id: event.id, + name: event.name, + error: event.error, + })); + + return { + startTime: this.startTime, + endTime, + durationMs: endTime - this.startTime, + cloudItemCount: this.cloudItemCount, + listedCount: this.events.filter((event) => event.action === "listed").length, + syncedCount: this.events.filter((event) => event.action === "synced").length, + skippedCount: this.events.filter((event) => event.action === "skipped").length, + failedCount: failedItems.length, + failedItems, + fatalError: this.fatalError ?? undefined, + logPath: this.logPath || undefined, + }; + } + + formatNotice(report: SyncReport): string { + if (report.fatalError) { + const logPath = report.logPath ?? ".sync-log.md"; + return `Slate sync failed: ${report.fatalError}. Check ${logPath} for details.`; + } + + const parts = [ + `${report.cloudItemCount} found`, + `${report.syncedCount} synced`, + `${report.skippedCount} unchanged`, + ]; + + if (report.failedCount > 0) { + const failedNames = report.failedItems + .slice(0, MAX_NOTICE_FAILURES) + .map((item) => item.name) + .join(", "); + const extraFailures = report.failedCount - MAX_NOTICE_FAILURES; + const suffix = extraFailures > 0 ? `, +${extraFailures} more` : ""; + parts.push(`${report.failedCount} failed (${failedNames}${suffix})`); + } + + return `Slate: ${parts.join(", ")}.`; + } + + formatLogEntry(report: SyncReport): string { + const lines = [ + `## ${new Date(report.endTime).toLocaleString()}`, + "", + `- **Duration:** ${formatDuration(report.durationMs)}`, + `- **Cloud items:** ${report.cloudItemCount}`, + `- **Listed:** ${report.listedCount}`, + `- **Synced:** ${report.syncedCount}`, + `- **Unchanged:** ${report.skippedCount}`, + `- **Failed:** ${report.failedCount}`, + ]; + + if (report.fatalError) { + lines.push(`- **Fatal error:** ${escapeLogText(report.fatalError)}`); + } + + if (report.failedItems.length > 0) { + lines.push("- **Failed items:**"); + for (const item of report.failedItems) { + lines.push(` - \"${escapeLogText(item.name)}\" - ${escapeLogText(item.error)}`); + } + } + + return `${lines.join("\n")}\n`; + } + + updateLogContent(existingContent: string, report: SyncReport): string { + const sections = splitLogSections(existingContent); + const nextSections = [this.formatLogEntry(report), ...sections].slice(0, MAX_LOG_ENTRIES); + return `${LOG_HEADER}${nextSections.join("\n")}`.trimEnd() + "\n"; + } + + createLogContent(report: SyncReport): string { + return `${LOG_HEADER}${this.formatLogEntry(report)}`; + } +} + +function splitLogSections(content: string): string[] { + const trimmed = content.trim(); + if (!trimmed) { + return []; + } + + const withoutHeader = trimmed.replace(/^# Slate Sync Log\n\nNewest entries appear first\.\n?/, "").trim(); + if (!withoutHeader) { + return []; + } + + return withoutHeader + .split(/\n(?=## )/) + .map((section) => section.trim()) + .filter(Boolean) + .map((section) => `${section}\n`); +} + +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + return `${(durationMs / 1000).toFixed(1)}s`; +} + +function escapeLogText(value: string): string { + return value.replace(/\n+/g, " ").replace(/[<>]/g, "").trim(); +} diff --git a/src/SyncProgressModal.ts b/src/SyncProgressModal.ts new file mode 100644 index 0000000..7066c3e --- /dev/null +++ b/src/SyncProgressModal.ts @@ -0,0 +1,110 @@ +import { App, Modal } from "obsidian"; +import type RemarkableSyncPlugin from "../main"; + +export class SyncProgressModal extends Modal { + private refreshTimer: number | null = null; + + constructor( + app: App, + private plugin: RemarkableSyncPlugin, + private onModalClose?: () => void, + ) { + super(app); + } + + onOpen(): void { + this.setTitle("Slate Sync Progress"); + this.render(); + if (this.refreshTimer !== null) { + window.clearInterval(this.refreshTimer); + } + this.refreshTimer = window.setInterval(() => this.render(), 1000); + } + + onClose(): void { + if (this.refreshTimer !== null) { + window.clearInterval(this.refreshTimer); + this.refreshTimer = null; + } + + this.contentEl.empty(); + this.onModalClose?.(); + } + + private render(): void { + const snapshot = this.plugin.getSyncProgress(); + const { contentEl } = this; + contentEl.empty(); + + if (!snapshot) { + contentEl.createEl("p", { text: "No sync in progress." }); + return; + } + + contentEl.createEl("p", { text: `Phase: ${snapshot.phase}` }); + contentEl.createEl("p", { text: `Elapsed: ${formatDuration(Date.now() - snapshot.startedAt)}` }); + + if (snapshot.currentItem) { + contentEl.createEl("p", { text: `Current item: ${snapshot.currentItem}` }); + } + + contentEl.createEl("p", { + text: `Cloud items: ${snapshot.inspectedItemCount}/${snapshot.cloudItemCount} inspected | ${snapshot.listedCount} listed`, + }); + contentEl.createEl("p", { + text: `Documents: ${snapshot.processedDocumentCount}/${snapshot.documentCount} processed | ${snapshot.syncedCount} completed | ${snapshot.skippedCount} skipped | ${snapshot.failedCount} errors`, + }); + contentEl.createEl("p", { + text: `Listed from cache: ${snapshot.listedFromCacheCount} (docs ${snapshot.cachedDocumentCount}, collections ${snapshot.cachedCollectionCount}) | fetched from cloud: ${snapshot.listedFromCloudCount}`, + }); + contentEl.createEl("p", { + text: `Completed: ${snapshot.syncedCount} total | new downloads ${snapshot.newDownloadCount} | re-downloaded ${snapshot.redownloadCount}`, + }); + contentEl.createEl("p", { + text: `Skipped: ${snapshot.skippedCount} total | cached unchanged ${snapshot.cachedSkipCount} | reused local markdown ${snapshot.reusedLocalSkipCount} | excluded PDF ${snapshot.excludedPdfSkipCount} | no supported content ${snapshot.unsupportedContentSkipCount}${snapshot.otherSkipCount > 0 ? ` | other ${snapshot.otherSkipCount}` : ""}`, + }); + + if (snapshot.fatalError) { + contentEl.createEl("p", { text: `Fatal error: ${snapshot.fatalError}` }); + } + + renderDetailedList( + contentEl, + "Recent completed", + snapshot.recentCompleted.map((item) => `${item.name}: ${item.kind === "new_download" ? "new download" : "re-downloaded"}`), + ); + renderDetailedList( + contentEl, + "Recent skipped", + snapshot.recentSkipped.map((item) => `${item.name}: ${item.reason}`), + ); + renderDetailedList( + contentEl, + "Recent errors", + snapshot.recentFailures.map((item) => `${item.name}: ${item.error}`), + ); + + if (snapshot.logPath) { + contentEl.createEl("p", { text: `Log file: ${snapshot.logPath}` }); + } + } +} +function renderDetailedList(containerEl: HTMLElement, heading: string, items: string[]): void { + if (items.length === 0) { + return; + } + + containerEl.createEl("h4", { text: heading }); + const listEl = containerEl.createEl("ul"); + for (const item of items) { + listEl.createEl("li", { text: item }); + } +} + +function formatDuration(durationMs: number): string { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + + return `${(durationMs / 1000).toFixed(1)}s`; +} diff --git a/src/types.ts b/src/types.ts index e044c14..2f52f3c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -161,6 +161,76 @@ export interface SyncRecord { hash: string; lastModified: string; vaultPath: string; + visibleName?: string; + parent?: string; + type?: "DocumentType" | "CollectionType"; +} + +export interface SyncFailure { + id: string; + name: string; + error: string; +} + +export interface SyncSkip { + id: string; + name: string; + reason: string; + kind: + | "cached_unchanged" + | "reused_local_markdown" + | "excluded_pdf" + | "no_supported_content" + | "other"; +} + +export interface SyncCompleted { + name: string; + kind: "new_download" | "redownloaded"; +} + +export interface SyncReport { + startTime: number; + endTime: number; + durationMs: number; + cloudItemCount: number; + listedCount: number; + syncedCount: number; + skippedCount: number; + failedCount: number; + failedItems: SyncFailure[]; + fatalError?: string; + logPath?: string; +} + +export interface SyncProgressSnapshot { + phase: string; + startedAt: number; + currentItem?: string; + cloudItemCount: number; + inspectedItemCount: number; + documentCount: number; + processedDocumentCount: number; + listedCount: number; + listedFromCacheCount: number; + listedFromCloudCount: number; + cachedCollectionCount: number; + cachedDocumentCount: number; + syncedCount: number; + newDownloadCount: number; + redownloadCount: number; + skippedCount: number; + cachedSkipCount: number; + reusedLocalSkipCount: number; + excludedPdfSkipCount: number; + unsupportedContentSkipCount: number; + otherSkipCount: number; + failedCount: number; + recentCompleted: SyncCompleted[]; + recentSkipped: SyncSkip[]; + recentFailures: SyncFailure[]; + fatalError?: string; + logPath?: string; } // ── Plugin Settings ──────────────────────────────────────────────────────── @@ -171,6 +241,7 @@ export interface PluginSettings { syncOnStartup: boolean; excludePdfs: boolean; lastSyncTimestamp: number; + lastSyncReport?: SyncReport; syncState: Record; deviceId: string; }