Eduardo_Scripniciuc/st-ten-1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev.r5portal upload system wip 2

Browse Source
This commit is contained in:
edo-neo 2024-10-22 16:37:55 +02:00
parent 5464f9a9c4
commit e1289065c8
1 changed files with 17 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/components/archive_synchronizer.py
View File

@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
self.machine_id = self.config.machine_id self.machine_id = self.config.machine_id
if "--dev-portal" in sys.argv: if "--dev-portal" in sys.argv:
self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/" self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
self.base_media_url = f"https://dev.r5portal.it/media/uploads" self.base_media_url = f"https://dev.r5portal.it"
else: else:
self.archive_endpoint = self.config[self.name]["archive_endpoint"] self.archive_endpoint = self.config[self.name]["archive_endpoint"]
#self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"] #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
@ -161,19 +161,21 @@ class ArchiveSynchronizer(Component):
:param machine_id: The machine ID used to construct the folder path. :param machine_id: The machine ID used to construct the folder path.
:param subfolder: The subfolder within the machine-specific path to fetch files from. :param subfolder: The subfolder within the machine-specific path to fetch files from.
:return: A list of downloaded file paths, or False if an error occurs. :return: A list of downloaded file paths or False if an error occurs.
""" """
base_url = f"{self.base_media_url}/{subfolder}/{machine_id}" base_url = f"{self.base_media_url}/list-files/"
responses = [] responses = []
try: try:
if not self.simulate: if not self.simulate:
with requests.Session() as s: with requests.Session() as s:
s.mount("", HTTPAdapter(max_retries=Retry(total=0))) s.mount("", HTTPAdapter(max_retries=Retry(total=0)))
self.log.info(f"Attempting to fetch directory listing from: {base_url}") # Construct the request to the new API endpoint
params = {"machine_id": machine_id, "subfolder": subfolder}
self.log.info(f"Attempting to fetch file list from: {base_url} with params: {params}")
# Make the HTTP GET request to the directory URL # Make the HTTP GET request to fetch the file URLs
response = s.get(base_url, timeout=5, verify=False) response = s.get(base_url, params=params, timeout=5, verify=False)
# Log response details # Log response details
self.log.info(f"HTTP Status Code: {response.status_code}") self.log.info(f"HTTP Status Code: {response.status_code}")
@ -185,18 +187,19 @@ class ArchiveSynchronizer(Component):
f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.") f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.")
return False return False
elif response.status_code == 403: elif response.status_code == 403:
self.log.warning(f"Access forbidden for directory listing: {base_url}") self.log.warning(f"Access forbidden for file list: {base_url}")
return False return False
elif response.status_code != 200: elif response.status_code != 200:
self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}") self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}")
return False return False
# Manually parse the response content to get all file URLs # Parse the JSON response to get the list of file URLs
content = response.content.decode('iso-8859-1') # or appropriate charset file_data = response.json()
file_urls = [] if "file_urls" not in file_data:
matches = re.findall(r'href="([^"]+\.(?:png|csv|svg))"', content, re.IGNORECASE) self.log.error(f"Invalid response structure: {file_data}")
for href in matches: return False
file_urls.append(f"{base_url}{href}")
file_urls = file_data["file_urls"]
# Download each file # Download each file
for file_url in file_urls: for file_url in file_urls:
@ -210,7 +213,7 @@ class ArchiveSynchronizer(Component):
f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}") f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}")
continue continue
local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url)) local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url)) #further adjust
os.makedirs(os.path.dirname(local_file_path), exist_ok=True) os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
with open(local_file_path, "wb") as f: with open(local_file_path, "wb") as f: