From 44aedcbee61f8cd03245ad33ead665a933f460fa Mon Sep 17 00:00:00 2001 From: tigeren Date: Thu, 28 Aug 2025 18:56:59 +0000 Subject: [PATCH] feat: implement infinite scrolling and replace virtualized media grid - Replaced the existing VirtualizedMediaGrid component with a new InfiniteVirtualGrid component to support infinite scrolling for bookmarks, photos, and videos. - Enhanced media item fetching with batch loading and search functionality, improving user experience and performance. - Removed unused state variables and functions related to photo indexing and file size formatting, streamlining the codebase. --- CLAUDE.md | 60 +-- src/app/bookmarks/page.tsx | 15 +- src/app/photos/page.tsx | 30 +- src/app/videos/page.tsx | 4 +- src/components/infinite-virtual-grid.tsx | 569 +++++++++++++++++++++++ 5 files changed, 579 insertions(+), 99 deletions(-) create mode 100644 src/components/infinite-virtual-grid.tsx diff --git a/CLAUDE.md b/CLAUDE.md index 934a5b0..da65d66 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,60 +28,6 @@ UI: 8. the video card can be clicked, once clicked, a poped up video player will be displayed. it can be closed, fast forward, expand to full screen, etc. 9. can bookmark/un-bookmark the video, can star the video -Performance Optimization Plan - -Phase 1: Critical API Pagination (Immediate Fix) -Implement database pagination across all list APIs -Add limit/offset parameters to /api/videos, /api/photos, /api/bookmarks -Add server-side filtering and sorting -Implement cursor-based pagination for better performance -Add database indexes for pagination queries -Create compound indexes for (type, created_at) queries -Add path-based indexes for folder-viewer queries -Optimize bookmark/star count queries - -Phase 2: Frontend Memory Optimization -Implement virtual scrolling for large lists -Use react-window or react-virtualized for video/photo grids -Add infinite scroll with intersection observer -Implement client-side caching with LRU eviction -Add progressive loading strategies -Lazy load thumbnails as they come into view -Implement skeleton loaders during data fetching -Add debounced search with server-side filtering - -Phase 3: File System Scanning Optimization -Parallel processing implementation -Use worker threads for thumbnail generation -Implement batch processing for database inserts -Add progress reporting with WebSocket/SSE -Smart scanning strategies -Implement incremental scanning (only new/changed files) -Add file watching for real-time updates -Use streaming file discovery instead of loading all paths - -Phase 4: Database Performance -Connection pooling and optimization -Implement better-sqlite3 connection pooling -Add prepared statement caching -Implement batch operations for inserts/updates -Advanced indexing strategy -Full-text search indexes for title/path searching -Composite indexes for common query patterns -Materialized views for aggregated data (ratings, bookmarks) - -Phase 5: Caching & CDN Strategy -Multi-level caching -Redis for API response caching -Browser caching with ETags -CDN for thumbnail delivery -Thumbnail optimization -WebP format with fallbacks -Multiple thumbnail sizes for different viewports -Lazy generation with background processing -Implementation Priority -P0 (Critical) : API pagination + frontend virtual scrolling -P1 (High) : Database indexes + connection optimization -P2 (Medium) : File scanning improvements -P3 (Low) : Advanced caching + CDN -This plan addresses all identified bottlenecks systematically, starting with the most critical issues that would cause immediate system failure with large datasets. \ No newline at end of file +Development Rules: +1. Everytime after making all the changes, run 'pnpm build' to verify the changes are compiling correct. +2. Once added debug logs, don't delete it until told so. diff --git a/src/app/bookmarks/page.tsx b/src/app/bookmarks/page.tsx index 828f4ec..6bba703 100644 --- a/src/app/bookmarks/page.tsx +++ b/src/app/bookmarks/page.tsx @@ -1,7 +1,7 @@ 'use client'; import { useState } from 'react'; -import VirtualizedMediaGrid from '@/components/virtualized-media-grid'; +import InfiniteVirtualGrid from '@/components/infinite-virtual-grid'; import VideoViewer from '@/components/video-viewer'; import PhotoViewer from '@/components/photo-viewer'; @@ -19,7 +19,6 @@ interface MediaItem { export default function BookmarksPage() { const [selectedItem, setSelectedItem] = useState(null); - const [currentPhotoIndex, setCurrentPhotoIndex] = useState(0); const [isViewerOpen, setIsViewerOpen] = useState(false); const [isVideoPlayerOpen, setIsVideoPlayerOpen] = useState(false); @@ -71,17 +70,9 @@ export default function BookmarksPage() { } }; - const formatFileSize = (bytes: number) => { - if (bytes === 0) return '0 Bytes'; - const k = 1024; - const sizes = ['Bytes', 'KB', 'MB', 'GB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; - }; - return ( <> - (null); - const [currentPhotoIndex, setCurrentPhotoIndex] = useState(0); const [isViewerOpen, setIsViewerOpen] = useState(false); - const handlePhotoClick = (photo: Photo, index: number = 0) => { + const handlePhotoClick = (photo: Photo) => { setSelectedPhoto(photo); - setCurrentPhotoIndex(index); setIsViewerOpen(true); }; @@ -32,16 +30,6 @@ export default function PhotosPage() { setSelectedPhoto(null); }; - const handleNextPhoto = () => { - // This would need to be implemented with the virtualized grid - // For now, we'll keep the current simple behavior - }; - - const handlePrevPhoto = () => { - // This would need to be implemented with the virtualized grid - // For now, we'll keep the current simple behavior - }; - const handleBookmark = async (photoId: number) => { try { await fetch(`/api/bookmarks/${photoId}`, { method: 'POST' }); @@ -70,17 +58,9 @@ export default function PhotosPage() { } }; - const formatFileSize = (bytes: number) => { - if (bytes === 0) return '0 Bytes'; - const k = 1024; - const sizes = ['Bytes', 'KB', 'MB', 'GB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); - return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; - }; - return ( <> - { return ( <> - void; + onBookmark: (id: number) => Promise; + onUnbookmark: (id: number) => Promise; + onRate: (id: number, rating: number) => Promise; +} + +const ITEM_HEIGHT = 300; +const ITEMS_PER_BATCH = 50; +const BUFFER_SIZE = 200; + +export default function InfiniteVirtualGrid({ + type, + onItemClick, + onBookmark, + onUnbookmark, + onRate +}: InfiniteVirtualGridProps) { + const [totalItems, setTotalItems] = useState(0); + const [searchTerm, setSearchTerm] = useState(''); + const [containerWidth, setContainerWidth] = useState(0); + const [isLoadingInitial, setIsLoadingInitial] = useState(true); + + const loadingRef = useRef>(new Set()); + const dataCacheRef = useRef>(new Map()); + const containerRef = useRef(null); + const gridRef = useRef(null); + + const formatFileSize = useCallback((bytes: number) => { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const sizes = ['Bytes', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; + }, []); + + const formatFilePath = useCallback((path: string) => { + if (!path) return ''; + + const lastSlashIndex = path.lastIndexOf('/'); + const lastBackslashIndex = path.lastIndexOf('\\'); + const lastSeparatorIndex = Math.max(lastSlashIndex, lastBackslashIndex); + + if (lastSeparatorIndex === -1) { + return path; + } + + const directory = path.substring(0, lastSeparatorIndex); + const filename = path.substring(lastSeparatorIndex + 1); + + if (directory.length <= 30) { + return `${directory}/${filename}`; + } + + const maxDirLength = 25; + const startLength = Math.floor(maxDirLength / 2); + const endLength = maxDirLength - startLength - 3; + + const truncatedDir = directory.length > maxDirLength + ? `${directory.substring(0, startLength)}...${directory.substring(directory.length - endLength)}` + : directory; + + return `${truncatedDir}/${filename}`; + }, []); + + const getColumnCount = useCallback(() => { + if (containerWidth === 0) return 6; + if (containerWidth < 640) return 2; + if (containerWidth < 768) return 3; + if (containerWidth < 1024) return 4; + if (containerWidth < 1280) return 5; + if (containerWidth < 1536) return 6; + return 7; + }, [containerWidth]); + + const getColumnWidth = useCallback(() => { + const cols = getColumnCount(); + const availableWidth = containerWidth - 32; + const gapWidth = (cols - 1) * 16; + return Math.floor((availableWidth - gapWidth) / cols); + }, [containerWidth, getColumnCount]); + + const rowCount = useMemo(() => { + return Math.ceil(totalItems / getColumnCount()); + }, [totalItems, getColumnCount]); + + const fetchItems = useCallback(async (startIndex: number, endIndex: number) => { + const batchStart = Math.floor(startIndex / ITEMS_PER_BATCH) * ITEMS_PER_BATCH; + const batchKey = Math.floor(startIndex / ITEMS_PER_BATCH); + + if (loadingRef.current.has(batchKey) || dataCacheRef.current.has(batchKey)) { + return; + } + + loadingRef.current.add(batchKey); + + try { + const limit = Math.min(ITEMS_PER_BATCH, totalItems - batchStart); + if (limit <= 0) return; + + const params = new URLSearchParams({ + limit: limit.toString(), + offset: batchStart.toString(), + ...(searchTerm && { search: searchTerm }) + }); + + const endpoint = type === 'bookmark' ? 'bookmarks' : `${type}s`; + const response = await fetch(`/api/${endpoint}?${params}`); + const data = await response.json(); + + const itemsKey = type === 'bookmark' ? 'bookmarks' : `${type}s`; + const items = data[itemsKey] || []; + + dataCacheRef.current.set(batchKey, items); + + } catch (error) { + console.error(`Error fetching ${type}s:`, error); + } finally { + loadingRef.current.delete(batchKey); + } + }, [type, searchTerm, totalItems]); + + const fetchTotalCount = useCallback(async () => { + try { + const endpoint = type === 'bookmark' ? 'bookmarks' : `${type}s`; + + const params = new URLSearchParams({ + limit: '50', + offset: '0', + ...(searchTerm && { search: searchTerm }) + }); + + const response = await fetch(`/api/${endpoint}?${params}`); + const data = await response.json(); + + const itemsKey = type === 'bookmark' ? 'bookmarks' : `${type}s`; + const items = data[itemsKey] || []; + + if (items.length > 0) { + dataCacheRef.current.set(0, items); + } + + setTotalItems(data.pagination.total || 0); + setIsLoadingInitial(false); + } catch (error) { + console.error(`Error fetching total count for ${type}:`, error); + setTotalItems(0); + setIsLoadingInitial(false); + } + }, [type, searchTerm]); + + useEffect(() => { + dataCacheRef.current.clear(); + loadingRef.current.clear(); + setTotalItems(0); + setIsLoadingInitial(true); + fetchTotalCount(); + }, [type, searchTerm, fetchTotalCount]); + + useEffect(() => { + const updateWidth = () => { + if (containerRef.current) { + const width = containerRef.current.offsetWidth; + setContainerWidth(width); + } + }; + + // Use ResizeObserver for more reliable width detection + const resizeObserver = new ResizeObserver(entries => { + if (entries[0]) { + setContainerWidth(entries[0].contentRect.width); + } + }); + + if (containerRef.current) { + resizeObserver.observe(containerRef.current); + updateWidth(); // Initial measurement + } + + // Window resize fallback + const handleResize = () => { + updateWidth(); + }; + window.addEventListener('resize', handleResize); + + // Fallback to manual measurement if ResizeObserver not available + const fallbackUpdate = () => { + setTimeout(updateWidth, 100); + }; + fallbackUpdate(); + + return () => { + resizeObserver.disconnect(); + window.removeEventListener('resize', handleResize); + }; + }, []); + + // Aggressive width detection fallback + useEffect(() => { + const aggressiveWidthCheck = () => { + // Try to find any suitable container + const containers = [ + containerRef.current, + document.querySelector('.flex-1') as HTMLElement, + document.querySelector('.max-w-7xl') as HTMLElement, + document.querySelector('.w-full') as HTMLElement + ].filter(Boolean); + + for (const container of containers) { + if (container) { + const width = (container as HTMLElement).offsetWidth || (container as HTMLElement).clientWidth; + if (width > 0) { + setContainerWidth(width); + return; + } + } + } + + // As a last resort, use window width minus some padding + const fallbackWidth = Math.max(window.innerWidth - 64, 320); + setContainerWidth(fallbackWidth); + }; + + // Try multiple times with increasing delays + aggressiveWidthCheck(); + const timeouts = [ + setTimeout(aggressiveWidthCheck, 50), + setTimeout(aggressiveWidthCheck, 100), + setTimeout(aggressiveWidthCheck, 500) + ]; + + return () => { + timeouts.forEach(timeout => clearTimeout(timeout)); + }; + }, []); + + const getItemData = useCallback((index: number): MediaItem | null => { + if (index >= totalItems) return null; + + const batchKey = Math.floor(index / ITEMS_PER_BATCH); + const batchItems = dataCacheRef.current.get(batchKey); + + if (!batchItems) { + return null; + } + + const itemIndex = index % ITEMS_PER_BATCH; + return batchItems[itemIndex] || null; + }, [totalItems]); + + const handleItemsRendered = useCallback((params: any) => { + const { visibleRowStartIndex, visibleRowStopIndex } = params; + + console.log('onItemsRendered triggered:', { visibleRowStartIndex, visibleRowStopIndex, totalItems, columnCount: getColumnCount() }); + + if (typeof visibleRowStartIndex !== 'number' || typeof visibleRowStopIndex !== 'number') { + console.log('Invalid indices received'); + return; + } + + if (totalItems === 0) { + console.log('No total items'); + return; + } + + const columnCount = getColumnCount(); + + // Convert row indices to absolute item indices + const startIndex = visibleRowStartIndex * columnCount; + const endIndex = Math.min((visibleRowStopIndex + 1) * columnCount - 1, totalItems - 1); + + console.log('Calculated indices:', { startIndex, endIndex, columnCount }); + + // Check what batches we need to load + const startBatch = Math.floor(startIndex / ITEMS_PER_BATCH); + const endBatch = Math.floor(endIndex / ITEMS_PER_BATCH); + + console.log('Batch range needed:', { startBatch, endBatch }); + + // Load missing batches + let loadedAny = false; + for (let batch = startBatch; batch <= endBatch; batch++) { + const batchStart = batch * ITEMS_PER_BATCH; + const batchEnd = Math.min((batch + 1) * ITEMS_PER_BATCH, totalItems); + + if (!dataCacheRef.current.has(batch) && !loadingRef.current.has(batch)) { + console.log(`Loading batch ${batch}: ${batchStart}-${batchEnd}`); + fetchItems(batchStart, batchEnd); + loadedAny = true; + } + } + + // Also check if we need to load more for infinite scroll + if (!loadedAny) { + const lastLoadedBatch = Math.max(...Array.from(dataCacheRef.current.keys()), -1); + const nextBatch = lastLoadedBatch + 1; + const nextBatchStart = nextBatch * ITEMS_PER_BATCH; + + if (nextBatchStart < totalItems && endIndex + BUFFER_SIZE >= nextBatchStart) { + const batchEnd = Math.min(nextBatchStart + ITEMS_PER_BATCH, totalItems); + console.log(`Loading next batch for infinite scroll ${nextBatch}: ${nextBatchStart}-${batchEnd}`); + fetchItems(nextBatchStart, batchEnd); + } + } + }, [totalItems, fetchItems, getColumnCount]); + + const Cell = ({ columnIndex, rowIndex, style }: any) => { + const columnCount = getColumnCount(); + const index = rowIndex * columnCount + columnIndex; + + if (index >= totalItems) return null; + + const item = getItemData(index); + + if (!item) { + return ( +
+ +
+ +
+
+ + +
+ ); + } + + return ( +
+ onItemClick(item)} + > +
+ {item.title} { + (e.target as HTMLImageElement).src = type === 'video' ? "/placeholder-video.svg" : "/placeholder-photo.svg"; + }} + /> +
+ +
+
+ {type === 'video' ? + : + + } +
+
+ +
+
+ {type === 'video' ? + : + + } +
+
+
+ + +

+ {item.title || item.path.split('/').pop()} +

+ + {(item.avg_rating > 0 || item.star_count > 0) && ( +
+ +
+ )} + +
+
+ + {formatFileSize(item.size)} +
+
+

+ {formatFilePath(item.path)} +

+ + +
+ ); + }; + + const getAvailableHeight = useCallback(() => { + if (typeof window === 'undefined') return 600; + const headerHeight = 180; + const bottomPadding = 20; + return Math.max(window.innerHeight - headerHeight - bottomPadding, 400); + }, []); + + if (isLoadingInitial) { + return ( +
+
+
+
+ {type === 'video' ? + : + + } +
+

Loading {type}s...

+
+
+
+ ); + } + + if (totalItems === 0) { + return ( +
+
+
+
+
+ {type === 'video' ? + : + type === 'photo' ? + : + + } +
+
+

+ {type === 'bookmark' ? 'Bookmarked Items' : `${type}s`} +

+

+ 0 {type === 'bookmark' ? 'item' : type}{type === 'bookmark' || type === 'photo' ? 's' : ''} in your library +

+
+
+ +
+ + setSearchTerm(e.target.value)} + className="pl-10 bg-background border-border" + /> +
+
+ +
+
+
+ +
+

+ No {type === 'bookmark' ? 'bookmarked items' : `${type}s`} found +

+

+ {searchTerm ? 'Try adjusting your search terms' : + type === 'bookmark' ? 'Start bookmarking videos and photos to see them here' : + 'Add media libraries and scan for content to get started'} +

+
+
+
+
+ ); + } + + return ( +
+
+
+
+
+ {type === 'video' ? + : + type === 'photo' ? + : + + } +
+
+

+ {type === 'bookmark' ? 'Bookmarked Items' : `${type}s`} +

+

+ {totalItems.toLocaleString()} {type === 'bookmark' ? 'item' : type}{totalItems === 1 ? '' : 's'} in your library +

+
+
+ +
+ + setSearchTerm(e.target.value)} + className="pl-10 bg-background border-border" + /> +
+
+ +
+ {containerWidth > 0 && ( + + {Cell} + + )} + {containerWidth === 0 && ( +
+
+
+ )} +
+
+
+ ); +} \ No newline at end of file