Compare commits
No commits in common. "fbdeba508869cd615ecd14e40f364265f184de21" and "345fd05a2b7881c108b6dc774686bbc89b4412ce" have entirely different histories.
fbdeba5088
...
345fd05a2b
|
|
@ -6,21 +6,14 @@ WORKDIR /app
|
||||||
RUN apt-get update && apt-get install -y \
|
RUN apt-get update && apt-get install -y \
|
||||||
build-essential \
|
build-essential \
|
||||||
libreoffice \
|
libreoffice \
|
||||||
wget \
|
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
|
||||||
# Copy requirements first to leverage Docker cache
|
# Copy requirements first to leverage Docker cache
|
||||||
COPY requirements.txt .
|
COPY requirements.txt .
|
||||||
RUN pip install huggingface_hub
|
|
||||||
RUN wget https://github.com/opendatalab/MinerU/raw/master/scripts/download_models_hf.py -O download_models_hf.py
|
|
||||||
RUN python download_models_hf.py
|
|
||||||
|
|
||||||
|
|
||||||
RUN pip install --no-cache-dir -r requirements.txt
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
RUN pip install -U magic-pdf[full]
|
RUN pip install -U magic-pdf[full]
|
||||||
|
|
||||||
|
|
||||||
# Copy the rest of the application
|
# Copy the rest of the application
|
||||||
COPY . .
|
COPY . .
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -6,11 +6,10 @@ import os
|
||||||
from ...core.config import settings
|
from ...core.config import settings
|
||||||
from ...core.database import get_db
|
from ...core.database import get_db
|
||||||
from ...models.file import File as FileModel, FileStatus
|
from ...models.file import File as FileModel, FileStatus
|
||||||
from ...services.file_service import process_file, delete_file
|
from ...services.file_service import process_file
|
||||||
from ...schemas.file import FileResponse as FileResponseSchema, FileList
|
from ...schemas.file import FileResponse as FileResponseSchema, FileList
|
||||||
import asyncio
|
import asyncio
|
||||||
from fastapi import WebSocketDisconnect
|
from fastapi import WebSocketDisconnect
|
||||||
import uuid
|
|
||||||
|
|
||||||
router = APIRouter()
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
@ -28,20 +27,14 @@ async def upload_file(
|
||||||
detail=f"File type not allowed. Allowed types: {', '.join(settings.ALLOWED_EXTENSIONS)}"
|
detail=f"File type not allowed. Allowed types: {', '.join(settings.ALLOWED_EXTENSIONS)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Generate unique file ID
|
# Save file
|
||||||
file_id = str(uuid.uuid4())
|
file_path = settings.UPLOAD_FOLDER / file.filename
|
||||||
file_extension = os.path.splitext(file.filename)[1]
|
|
||||||
unique_filename = f"{file_id}{file_extension}"
|
|
||||||
|
|
||||||
# Save file with unique name
|
|
||||||
file_path = settings.UPLOAD_FOLDER / unique_filename
|
|
||||||
with open(file_path, "wb") as buffer:
|
with open(file_path, "wb") as buffer:
|
||||||
content = await file.read()
|
content = await file.read()
|
||||||
buffer.write(content)
|
buffer.write(content)
|
||||||
|
|
||||||
# Create database entry
|
# Create database entry
|
||||||
db_file = FileModel(
|
db_file = FileModel(
|
||||||
id=file_id,
|
|
||||||
filename=file.filename,
|
filename=file.filename,
|
||||||
original_path=str(file_path),
|
original_path=str(file_path),
|
||||||
status=FileStatus.NOT_STARTED
|
status=FileStatus.NOT_STARTED
|
||||||
|
|
@ -115,24 +108,4 @@ async def websocket_endpoint(websocket: WebSocket, file_id: str, db: Session = D
|
||||||
|
|
||||||
await asyncio.sleep(1)
|
await asyncio.sleep(1)
|
||||||
except WebSocketDisconnect:
|
except WebSocketDisconnect:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@router.delete("/files/{file_id}")
|
|
||||||
async def delete_file_endpoint(
|
|
||||||
file_id: str,
|
|
||||||
db: Session = Depends(get_db)
|
|
||||||
):
|
|
||||||
"""
|
|
||||||
Delete a file and its associated records.
|
|
||||||
This will remove:
|
|
||||||
1. The database record
|
|
||||||
2. The original uploaded file
|
|
||||||
3. The processed markdown file (if it exists)
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
delete_file(file_id)
|
|
||||||
return {"message": "File deleted successfully"}
|
|
||||||
except HTTPException as e:
|
|
||||||
raise e
|
|
||||||
except Exception as e:
|
|
||||||
raise HTTPException(status_code=500, detail=str(e))
|
|
||||||
|
|
@ -7,7 +7,6 @@ import sys
|
||||||
import os
|
import os
|
||||||
from ..core.services.document_service import DocumentService
|
from ..core.services.document_service import DocumentService
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from fastapi import HTTPException
|
|
||||||
|
|
||||||
|
|
||||||
celery = Celery(
|
celery = Celery(
|
||||||
|
|
@ -16,39 +15,6 @@ celery = Celery(
|
||||||
backend=settings.CELERY_RESULT_BACKEND
|
backend=settings.CELERY_RESULT_BACKEND
|
||||||
)
|
)
|
||||||
|
|
||||||
def delete_file(file_id: str):
|
|
||||||
"""
|
|
||||||
Delete a file and its associated records.
|
|
||||||
This will:
|
|
||||||
1. Delete the database record
|
|
||||||
2. Delete the original uploaded file
|
|
||||||
3. Delete the processed markdown file (if it exists)
|
|
||||||
"""
|
|
||||||
db = SessionLocal()
|
|
||||||
try:
|
|
||||||
# Get the file record
|
|
||||||
file = db.query(File).filter(File.id == file_id).first()
|
|
||||||
if not file:
|
|
||||||
raise HTTPException(status_code=404, detail="File not found")
|
|
||||||
|
|
||||||
# Delete the original file if it exists
|
|
||||||
if file.original_path and os.path.exists(file.original_path):
|
|
||||||
os.remove(file.original_path)
|
|
||||||
|
|
||||||
# Delete the processed file if it exists
|
|
||||||
if file.processed_path and os.path.exists(file.processed_path):
|
|
||||||
os.remove(file.processed_path)
|
|
||||||
|
|
||||||
# Delete the database record
|
|
||||||
db.delete(file)
|
|
||||||
db.commit()
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
db.rollback()
|
|
||||||
raise HTTPException(status_code=500, detail=f"Error deleting file: {str(e)}")
|
|
||||||
finally:
|
|
||||||
db.close()
|
|
||||||
|
|
||||||
@celery.task
|
@celery.task
|
||||||
def process_file(file_id: str):
|
def process_file(file_id: str):
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
|
|
@ -65,8 +31,9 @@ def process_file(file_id: str):
|
||||||
# Process the file using your existing masking system
|
# Process the file using your existing masking system
|
||||||
process_service = DocumentService()
|
process_service = DocumentService()
|
||||||
|
|
||||||
# Determine output path using file_id with .md extension
|
# Determine output path
|
||||||
output_filename = f"{file_id}.md"
|
input_path = Path(file.original_path)
|
||||||
|
output_filename = f"processed_{input_path.name}"
|
||||||
output_path = str(settings.PROCESSED_FOLDER / output_filename)
|
output_path = str(settings.PROCESSED_FOLDER / output_filename)
|
||||||
|
|
||||||
# Process document with both input and output paths
|
# Process document with both input and output paths
|
||||||
|
|
|
||||||
|
|
@ -11,13 +11,8 @@ import {
|
||||||
Checkbox,
|
Checkbox,
|
||||||
Button,
|
Button,
|
||||||
Chip,
|
Chip,
|
||||||
Dialog,
|
|
||||||
DialogTitle,
|
|
||||||
DialogContent,
|
|
||||||
DialogActions,
|
|
||||||
Typography,
|
|
||||||
} from '@mui/material';
|
} from '@mui/material';
|
||||||
import { Download as DownloadIcon, Delete as DeleteIcon } from '@mui/icons-material';
|
import { Download as DownloadIcon } from '@mui/icons-material';
|
||||||
import { File, FileStatus } from '../types/file';
|
import { File, FileStatus } from '../types/file';
|
||||||
import { api } from '../services/api';
|
import { api } from '../services/api';
|
||||||
|
|
||||||
|
|
@ -28,8 +23,6 @@ interface FileListProps {
|
||||||
|
|
||||||
const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
|
const [selectedFiles, setSelectedFiles] = useState<string[]>([]);
|
||||||
const [deleteDialogOpen, setDeleteDialogOpen] = useState(false);
|
|
||||||
const [fileToDelete, setFileToDelete] = useState<string | null>(null);
|
|
||||||
|
|
||||||
const handleSelectFile = (fileId: string) => {
|
const handleSelectFile = (fileId: string) => {
|
||||||
setSelectedFiles((prev) =>
|
setSelectedFiles((prev) =>
|
||||||
|
|
@ -67,29 +60,6 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDeleteClick = (fileId: string) => {
|
|
||||||
setFileToDelete(fileId);
|
|
||||||
setDeleteDialogOpen(true);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleDeleteConfirm = async () => {
|
|
||||||
if (fileToDelete) {
|
|
||||||
try {
|
|
||||||
await api.deleteFile(fileToDelete);
|
|
||||||
onFileStatusChange();
|
|
||||||
} catch (error) {
|
|
||||||
console.error('Error deleting file:', error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
setDeleteDialogOpen(false);
|
|
||||||
setFileToDelete(null);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleDeleteCancel = () => {
|
|
||||||
setDeleteDialogOpen(false);
|
|
||||||
setFileToDelete(null);
|
|
||||||
};
|
|
||||||
|
|
||||||
const getStatusColor = (status: FileStatus) => {
|
const getStatusColor = (status: FileStatus) => {
|
||||||
switch (status) {
|
switch (status) {
|
||||||
case FileStatus.SUCCESS:
|
case FileStatus.SUCCESS:
|
||||||
|
|
@ -111,7 +81,6 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
color="primary"
|
color="primary"
|
||||||
onClick={handleDownloadSelected}
|
onClick={handleDownloadSelected}
|
||||||
disabled={selectedFiles.length === 0}
|
disabled={selectedFiles.length === 0}
|
||||||
sx={{ mr: 1 }}
|
|
||||||
>
|
>
|
||||||
Download Selected
|
Download Selected
|
||||||
</Button>
|
</Button>
|
||||||
|
|
@ -154,19 +123,10 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
{new Date(file.created_at).toLocaleString()}
|
{new Date(file.created_at).toLocaleString()}
|
||||||
</TableCell>
|
</TableCell>
|
||||||
<TableCell>
|
<TableCell>
|
||||||
<IconButton
|
|
||||||
onClick={() => handleDeleteClick(file.id)}
|
|
||||||
size="small"
|
|
||||||
color="error"
|
|
||||||
sx={{ mr: 1 }}
|
|
||||||
>
|
|
||||||
<DeleteIcon />
|
|
||||||
</IconButton>
|
|
||||||
{file.status === FileStatus.SUCCESS && (
|
{file.status === FileStatus.SUCCESS && (
|
||||||
<IconButton
|
<IconButton
|
||||||
onClick={() => handleDownload(file.id)}
|
onClick={() => handleDownload(file.id)}
|
||||||
size="small"
|
size="small"
|
||||||
color="primary"
|
|
||||||
>
|
>
|
||||||
<DownloadIcon />
|
<DownloadIcon />
|
||||||
</IconButton>
|
</IconButton>
|
||||||
|
|
@ -177,24 +137,6 @@ const FileList: React.FC<FileListProps> = ({ files, onFileStatusChange }) => {
|
||||||
</TableBody>
|
</TableBody>
|
||||||
</Table>
|
</Table>
|
||||||
</TableContainer>
|
</TableContainer>
|
||||||
|
|
||||||
<Dialog
|
|
||||||
open={deleteDialogOpen}
|
|
||||||
onClose={handleDeleteCancel}
|
|
||||||
>
|
|
||||||
<DialogTitle>Confirm Delete</DialogTitle>
|
|
||||||
<DialogContent>
|
|
||||||
<Typography>
|
|
||||||
Are you sure you want to delete this file? This action cannot be undone.
|
|
||||||
</Typography>
|
|
||||||
</DialogContent>
|
|
||||||
<DialogActions>
|
|
||||||
<Button onClick={handleDeleteCancel}>Cancel</Button>
|
|
||||||
<Button onClick={handleDeleteConfirm} color="error" variant="contained">
|
|
||||||
Delete
|
|
||||||
</Button>
|
|
||||||
</DialogActions>
|
|
||||||
</Dialog>
|
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -31,8 +31,4 @@ export const api = {
|
||||||
});
|
});
|
||||||
return response.data;
|
return response.data;
|
||||||
},
|
},
|
||||||
|
|
||||||
deleteFile: async (fileId: string): Promise<void> => {
|
|
||||||
await axios.delete(`${API_BASE_URL}/files/files/${fileId}`);
|
|
||||||
},
|
|
||||||
};
|
};
|
||||||
Loading…
Reference in New Issue