|
|
@ -3,6 +3,7 @@ |
|
|
import json |
|
|
import json |
|
|
import os |
|
|
import os |
|
|
import queue |
|
|
import queue |
|
|
|
|
|
import shutil |
|
|
import sys |
|
|
import sys |
|
|
import threading |
|
|
import threading |
|
|
import time |
|
|
import time |
|
|
@ -420,7 +421,10 @@ class CollectorApp: |
|
|
continue |
|
|
continue |
|
|
|
|
|
|
|
|
if self._upload_file(row, file_path): |
|
|
if self._upload_file(row, file_path): |
|
|
self.upload_state[state_key] = signature |
|
|
|
|
|
|
|
|
if self._backup_and_remove_file(row, file_path): |
|
|
|
|
|
self.upload_state[state_key] = signature |
|
|
|
|
|
else: |
|
|
|
|
|
self.upload_state.pop(state_key, None) |
|
|
|
|
|
|
|
|
def _file_signature(self, file_path): |
|
|
def _file_signature(self, file_path): |
|
|
try: |
|
|
try: |
|
|
@ -430,6 +434,34 @@ class CollectorApp: |
|
|
self.log("读取文件状态失败: %s, 原因: %s" % (file_path, e)) |
|
|
self.log("读取文件状态失败: %s, 原因: %s" % (file_path, e)) |
|
|
return None |
|
|
return None |
|
|
|
|
|
|
|
|
|
|
|
def _get_backup_dir(self, source_path): |
|
|
|
|
|
source = Path(source_path) |
|
|
|
|
|
return source.parent / (source.name + "_bak") |
|
|
|
|
|
|
|
|
|
|
|
def _backup_and_remove_file(self, row, file_path): |
|
|
|
|
|
source_path = row["source_path"] |
|
|
|
|
|
row_index = row["row_index"] |
|
|
|
|
|
backup_dir = self._get_backup_dir(source_path) |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
backup_dir.mkdir(parents=True, exist_ok=True) |
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
self.log("第%d行创建备份目录失败: %s, 原因: %s" % (row_index, backup_dir, e)) |
|
|
|
|
|
return False |
|
|
|
|
|
|
|
|
|
|
|
target = backup_dir / file_path.name |
|
|
|
|
|
if target.exists(): |
|
|
|
|
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") |
|
|
|
|
|
target = backup_dir / ("%s_%s%s" % (file_path.stem, timestamp, file_path.suffix)) |
|
|
|
|
|
|
|
|
|
|
|
try: |
|
|
|
|
|
shutil.move(str(file_path), str(target)) |
|
|
|
|
|
self.log("第%d行源文件已转移至备份: %s -> %s" % (row_index, file_path.name, target)) |
|
|
|
|
|
return True |
|
|
|
|
|
except Exception as e: |
|
|
|
|
|
self.log("第%d行源文件备份转移失败: %s, 原因: %s" % (row_index, file_path.name, e)) |
|
|
|
|
|
return False |
|
|
|
|
|
|
|
|
def _upload_file(self, row, file_path): |
|
|
def _upload_file(self, row, file_path): |
|
|
url = self._base_url() + "/collector/client/upload" |
|
|
url = self._base_url() + "/collector/client/upload" |
|
|
data = { |
|
|
data = { |
|
|
|