Compare commits
2 Commits
345fd05a2b
...
fbdeba5088
| Author | SHA1 | Date |
|---|---|---|
|
|
fbdeba5088 | |
|
|
dea3a6bd6a |
|
|
@ -6,14 +6,21 @@ WORKDIR /app
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
build-essential \
|
build-essential \
|
||||||
libreoffice \
|
libreoffice \
|
||||||
|
wget \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
# Copy requirements first to leverage Docker cache
|
# Copy requirements first to leverage Docker cache
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
|
RUN pip install huggingface_hub
|
||||||
|
RUN wget https://github.com/opendatalab/MinerU/raw/master/scripts/download_models_hf.py -O download_models_hf.py
|
||||||
|
RUN python download_models_hf.py
|
||||||
|
|
||||||
|
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
RUN pip install -U magic-pdf[full]
|
RUN pip install -U magic-pdf[full]
|
||||||
|
|
||||||
|
|
||||||
# Copy the rest of the application
|
# Copy the rest of the application
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,11 @@ import os
|
||||||
from ...core.config import settings
|
from ...core.config import settings
|
||||||
from ...core.database import get_db
|
from ...core.database import get_db
|
||||||
from ...models.file import File as FileModel, FileStatus
|
from ...models.file import File as FileModel, FileStatus
|
||||||
from ...services.file_service import process_file
|
from ...services.file_service import process_file, delete_file
|
||||||
from ...schemas.file import FileResponse as FileResponseSchema, FileList
|
from ...schemas.file import FileResponse as FileResponseSchema, FileList
|
||||||
import asyncio
|
import asyncio
|
||||||
from fastapi import WebSocketDisconnect
|
from fastapi import WebSocketDisconnect
|
||||||
|
import uuid
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
@ -27,14 +28,20 @@ async def upload_file(
|
||||||
detail=f"File type not allowed. Allowed types: {', '.join(settings.ALLOWED_EXTENSIONS)}"
|
detail=f"File type not allowed. Allowed types: {', '.join(settings.ALLOWED_EXTENSIONS)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Save file
|
# Generate unique file ID
|
||||||
file_path = settings.UPLOAD_FOLDER / file.filename
|
file_id = str(uuid.uuid4())
|
||||||
|
file_extension = os.path.splitext(file.filename)[1]
|
||||||
|
unique_filename = f"{file_id}{file_extension}"
|
||||||
|
|
||||||
|
# Save file with unique name
|
||||||
|
file_path = settings.UPLOAD_FOLDER / unique_filename
|
||||||
with open(file_path, "wb") as buffer:
|
with open(file_path, "wb") as buffer:
|
||||||
content = await file.read()
|
content = await file.read()
|
||||||
buffer.write(content)
|
buffer.write(content)
|
||||||
|
|
||||||
# Create database entry
|
# Create database entry
|
||||||
db_file = FileModel(
|
db_file = FileModel(
|
||||||
|
id=file_id,
|
||||||
filename=file.filename,
|
filename=file.filename,
|
||||||
original_path=str(file_path),
|
original_path=str(file_path),
|
||||||
status=FileStatus.NOT_STARTED
|
status=FileStatus.NOT_STARTED
|
||||||
|
|
@ -108,4 +115,24 @@ async def websocket_endpoint(websocket: WebSocket, file_id: str, db: Session = D
|
||||||
|
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
except WebSocketDisconnect:
|
except WebSocketDisconnect:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@router.delete("/files/{file_id}")
|
||||||
|
async def delete_file_endpoint(
|
||||||
|
file_id: str,
|
||||||
|
db: Session = Depends(get_db)
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Delete a file and its associated records.
|
||||||
|
This will remove:
|
||||||
|
1. The database record
|
||||||
|
2. The original uploaded file
|
||||||
|
3. The processed markdown file (if it exists)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
delete_file(file_id)
|
||||||
|
return {"message": "File deleted successfully"}
|
||||||
|
except HTTPException as e:
|
||||||
|
raise e
|
||||||
|
except Exception as e:
|
||||||
|
raise HTTPException(status_code=500, detail=str(e))
|
||||||
|
|
@ -7,6 +7,7 @@ import sys
|
||||||
import os
|
import os
|
||||||
from ..core.services.document_service import DocumentService
|
from ..core.services.document_service import DocumentService
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
|
||||||
celery = Celery(
|
celery = Celery(
|
||||||
|
|
@ -15,6 +16,39 @@ celery = Celery(
|
||||||
backend=settings.CELERY_RESULT_BACKEND
|
backend=settings.CELERY_RESULT_BACKEND
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def delete_file(file_id: str):
|
||||||
|
"""
|
||||||
|
Delete a file and its associated records.
|
||||||
|
This will:
|
||||||
|
1. Delete the database record
|
||||||
|
2. Delete the original uploaded file
|
||||||
|
3. Delete the processed markdown file (if it exists)
|
||||||
|
"""
|
||||||
|
db = SessionLocal()
|
||||||
|
try:
|
||||||
|
# Get the file record
|
||||||
|
file = db.query(File).filter(File.id == file_id).first()
|
||||||
|
if not file:
|
||||||
|
raise HTTPException(status_code=404, detail="File not found")
|
||||||
|
|
||||||
|
# Delete the original file if it exists
|
||||||
|
if file.original_path and os.path.exists(file.original_path):
|
||||||
|
os.remove(file.original_path)
|
||||||
|
|
||||||
|
# Delete the processed file if it exists
|
||||||
|
if file.processed_path and os.path.exists(file.processed_path):
|
||||||
|
os.remove(file.processed_path)
|
||||||
|
|
||||||
|
# Delete the database record
|
||||||
|
db.delete(file)
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
db.rollback()
|
||||||
|
raise HTTPException(status_code=500, detail=f"Error deleting file: {str(e)}")
|
||||||
|
finally:
|
||||||
|
db.close()
|
||||||
|
|
||||||
@celery.task
|
@celery.task
|
||||||
def process_file(file_id: str):
|
def process_file(file_id: str):
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
|
|
@ -31,9 +65,8 @@ def process_file(file_id: str):
|
||||||
# Process the file using your existing masking system
|
# Process the file using your existing masking system
|
||||||
process_service = DocumentService()
|
process_service = DocumentService()
|
||||||
|
|
||||||
# Determine output path
|
# Determine output path using file_id with .md extension
|
||||||
input_path = Path(file.original_path)
|
output_filename = f"{file_id}.md"
|
||||||
output_filename = f"processed_{input_path.name}"
|
|
||||||
output_path = str(settings.PROCESSED_FOLDER / output_filename)
|
output_path = str(settings.PROCESSED_FOLDER / output_filename)
|
||||||
|
|
||||||
# Process document with both input and output paths
|
# Process document with both input and output paths
|
||||||
|
|
|
||||||
|
|
@ -11,8 +11,13 @@ import {
|
||||||
Checkbox,
|
Checkbox,
|
||||||
Button,
|
Button,
|
||||||
Chip,
|
Chip,
|
||||||
|
Dialog,
|
||||||
|
DialogTitle,
|
||||||
|
DialogContent,
|
||||||
|
DialogActions,
|
||||||
|
Typography,
|
||||||
} from '@mui/material';
|
} from '@mui/material';
|
||||||
import { Download as DownloadIcon } from '@mui/icons-material';
|
import { Download as DownloadIcon, Delete as DeleteIcon } from '@mui/icons-material';
|
||||||
import { File, FileStatus } from '../types/file';
|
import { File, FileStatus } from '../types/file';
|
||||||
import { api } from '../services/api';
|
import { api } from '../services/api';
|
||||||
|
|
||||||
|
|
@ -23,6 +28,8 @@ interface FileListProps {
|
||||||
|
|
||||||
const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
|
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
|
||||||
|
const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
|
||||||
|
const [fileToDelete, setFileToDelete] = useState<string | null>(null);
|
||||||
|
|
||||||
const handleSelectFile = (fileId: string) => {
|
const handleSelectFile = (fileId: string) => {
|
||||||
setSelectedFiles((prev) =>
|
setSelectedFiles((prev) =>
|
||||||
|
|
@ -60,6 +67,29 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const handleDeleteClick = (fileId: string) => {
|
||||||
|
setFileToDelete(fileId);
|
||||||
|
setDeleteDialogOpen(true);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDeleteConfirm = async () => {
|
||||||
|
if (fileToDelete) {
|
||||||
|
try {
|
||||||
|
await api.deleteFile(fileToDelete);
|
||||||
|
onFileStatusChange();
|
||||||
|
} catch (error) {
|
||||||
|
console.error('Error deleting file:', error);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setDeleteDialogOpen(false);
|
||||||
|
setFileToDelete(null);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDeleteCancel = () => {
|
||||||
|
setDeleteDialogOpen(false);
|
||||||
|
setFileToDelete(null);
|
||||||
|
};
|
||||||
|
|
||||||
const getStatusColor = (status: FileStatus) => {
|
const getStatusColor = (status: FileStatus) => {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case FileStatus.SUCCESS:
|
case FileStatus.SUCCESS:
|
||||||
|
|
@ -81,6 +111,7 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
color="primary"
|
color="primary"
|
||||||
onClick={handleDownloadSelected}
|
onClick={handleDownloadSelected}
|
||||||
disabled={selectedFiles.length === 0}
|
disabled={selectedFiles.length === 0}
|
||||||
|
sx={{ mr: 1 }}
|
||||||
>
|
>
|
||||||
Download Selected
|
Download Selected
|
||||||
</Button>
|
</Button>
|
||||||
|
|
@ -123,10 +154,19 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
{new Date(file.created_at).toLocaleString()}
|
{new Date(file.created_at).toLocaleString()}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell>
|
<TableCell>
|
||||||
|
<IconButton
|
||||||
|
onClick={() => handleDeleteClick(file.id)}
|
||||||
|
size="small"
|
||||||
|
color="error"
|
||||||
|
sx={{ mr: 1 }}
|
||||||
|
>
|
||||||
|
<DeleteIcon />
|
||||||
|
</IconButton>
|
||||||
{file.status === FileStatus.SUCCESS && (
|
{file.status === FileStatus.SUCCESS && (
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={() => handleDownload(file.id)}
|
onClick={() => handleDownload(file.id)}
|
||||||
size="small"
|
size="small"
|
||||||
|
color="primary"
|
||||||
>
|
>
|
||||||
<DownloadIcon />
|
<DownloadIcon />
|
||||||
</IconButton>
|
</IconButton>
|
||||||
|
|
@ -137,6 +177,24 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
</TableBody>
|
</TableBody>
|
||||||
</Table>
|
</Table>
|
||||||
</TableContainer>
|
</TableContainer>
|
||||||
|
|
||||||
|
<Dialog
|
||||||
|
open={deleteDialogOpen}
|
||||||
|
onClose={handleDeleteCancel}
|
||||||
|
>
|
||||||
|
<DialogTitle>Confirm Delete</DialogTitle>
|
||||||
|
<DialogContent>
|
||||||
|
<Typography>
|
||||||
|
Are you sure you want to delete this file? This action cannot be undone.
|
||||||
|
</Typography>
|
||||||
|
</DialogContent>
|
||||||
|
<DialogActions>
|
||||||
|
<Button onClick={handleDeleteCancel}>Cancel</Button>
|
||||||
|
<Button onClick={handleDeleteConfirm} color="error" variant="contained">
|
||||||
|
Delete
|
||||||
|
</Button>
|
||||||
|
</DialogActions>
|
||||||
|
</Dialog>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -31,4 +31,8 @@ export const api = {
|
||||||
});
|
});
|
||||||
return response.data;
|
return response.data;
|
||||||
},
|
},
|
||||||
|
|
||||||
|
deleteFile: async (fileId: string): Promise<void> => {
|
||||||
|
await axios.delete(`${API_BASE_URL}/files/files/${fileId}`);
|
||||||
|
},
|
||||||
};
|
};
|
||||||
Loading…
Reference in New Issue