diff --git a/download_models.py b/download_models.py new file mode 100644 index 0000000..626473d --- /dev/null +++ b/download_models.py @@ -0,0 +1,67 @@ +import json +import shutil +import os + +import requests +from modelscope import snapshot_download + + +def download_json(url): + # 下载JSON文件 + response = requests.get(url) + response.raise_for_status() # 检查请求是否成功 + return response.json() + + +def download_and_modify_json(url, local_filename, modifications): + if os.path.exists(local_filename): + data = json.load(open(local_filename)) + config_version = data.get('config_version', '0.0.0') + if config_version < '1.2.0': + data = download_json(url) + else: + data = download_json(url) + + # 修改内容 + for key, value in modifications.items(): + data[key] = value + + # 保存修改后的内容 + with open(local_filename, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=4) + + +if __name__ == '__main__': + mineru_patterns = [ + # "models/Layout/LayoutLMv3/*", + "models/Layout/YOLO/*", + "models/MFD/YOLO/*", + "models/MFR/unimernet_hf_small_2503/*", + "models/OCR/paddleocr_torch/*", + # "models/TabRec/TableMaster/*", + # "models/TabRec/StructEqTable/*", + ] + model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns) + layoutreader_model_dir = snapshot_download('ppaanngggg/layoutreader') + model_dir = model_dir + '/models' + print(f'model_dir is: {model_dir}') + print(f'layoutreader_model_dir is: {layoutreader_model_dir}') + + # paddleocr_model_dir = model_dir + '/OCR/paddleocr' + # user_paddleocr_dir = os.path.expanduser('~/.paddleocr') + # if os.path.exists(user_paddleocr_dir): + # shutil.rmtree(user_paddleocr_dir) + # shutil.copytree(paddleocr_model_dir, user_paddleocr_dir) + + json_url = 'https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/magic-pdf.template.json' + config_file_name = 'magic-pdf.json' + home_dir = os.path.expanduser('~') + config_file = os.path.join(home_dir, config_file_name) + + json_mods = { + 'models-dir': model_dir, + 'layoutreader-model-dir': layoutreader_model_dir, + } + + download_and_modify_json(json_url, config_file, json_mods) + print(f'The configuration file has been configured successfully, the path is: {config_file}') diff --git a/requirements.txt b/requirements.txt index c5280ad..a003c83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,10 @@ pydantic-settings>=2.0.0 python-dotenv==1.0.0 watchdog==2.1.6 -requests==2.26.0 +requests==2.28.1 # Document processing python-docx>=0.8.11 PyPDF2>=3.0.0 pandas>=2.0.0 +magic-pdf[full] \ No newline at end of file