Eduardo_Scripniciuc/st-ten-1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev.r5portal upload system wip

Browse Source
This commit is contained in:
edo-neo 2024-10-22 12:11:56 +02:00
parent a54935fcad
commit 5464f9a9c4
1 changed files with 27 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/components/archive_synchronizer.py
View File

@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
self.machine_id = self.config.machine_id self.machine_id = self.config.machine_id
if "--dev-portal" in sys.argv: if "--dev-portal" in sys.argv:
self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/" self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
self.base_media_url = f"https://dev.r5portal.it/media/uploads/" self.base_media_url = f"https://dev.r5portal.it/media/uploads"
else: else:
self.archive_endpoint = self.config[self.name]["archive_endpoint"] self.archive_endpoint = self.config[self.name]["archive_endpoint"]
#self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"] #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
@ -155,7 +155,6 @@ class ArchiveSynchronizer(Component):
self.log.info(f"id: {record.id}: stored remotely") self.log.info(f"id: {record.id}: stored remotely")
return True return True
def remote_fetch(self, machine_id, subfolder="warning_images"): def remote_fetch(self, machine_id, subfolder="warning_images"):
""" """
Fetch and download all files from the server's folder. Fetch and download all files from the server's folder.
@ -171,24 +170,35 @@ class ArchiveSynchronizer(Component):
with requests.Session() as s: with requests.Session() as s:
s.mount("", HTTPAdapter(max_retries=Retry(total=0))) s.mount("", HTTPAdapter(max_retries=Retry(total=0)))
# Get directory listing self.log.info(f"Attempting to fetch directory listing from: {base_url}")
# Make the HTTP GET request to the directory URL
response = s.get(base_url, timeout=5, verify=False) response = s.get(base_url, timeout=5, verify=False)
if response.status_code == 403: # Log response details
self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}")
# Handle HTTP errors appropriately
if response.status_code == 404:
self.log.warning(
f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.")
return False
elif response.status_code == 403:
self.log.warning(f"Access forbidden for directory listing: {base_url}") self.log.warning(f"Access forbidden for directory listing: {base_url}")
return False return False
elif response.status_code != 200: elif response.status_code != 200:
raise AssertionError(f"bad status response for directory listing: {base_url}") self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}")
return False
# Parse the response content to get all file URLs. # Manually parse the response content to get all file URLs
soup = BeautifulSoup(response.content, 'html.parser') content = response.content.decode('iso-8859-1') # or appropriate charset
file_urls = [] file_urls = []
for link in soup.find_all('a'): matches = re.findall(r'href="([^"]+\.(?:png|csv|svg))"', content, re.IGNORECASE)
href = link.get('href') for href in matches:
if href and re.search(r'\.(png|csv|svg)$', href, file_urls.append(f"{base_url}{href}")
re.IGNORECASE): # Adjust file extensions as needed
file_urls.append(f"{self.base_media_url}/{machine_id}/{href}")
# Download each file
for file_url in file_urls: for file_url in file_urls:
file_response = s.get(file_url, timeout=5, verify=False) file_response = s.get(file_url, timeout=5, verify=False)
@ -196,9 +206,11 @@ class ArchiveSynchronizer(Component):
self.log.warning(f"Access forbidden for file URL: {file_url}") self.log.warning(f"Access forbidden for file URL: {file_url}")
continue continue
elif file_response.status_code != 200: elif file_response.status_code != 200:
raise AssertionError(f"bad status response for {file_url}") self.log.error(
f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}")
continue
local_file_path = os.path.join("downloaded_files", os.path.basename(file_url)) local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url))
os.makedirs(os.path.dirname(local_file_path), exist_ok=True) os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
with open(local_file_path, "wb") as f: with open(local_file_path, "wb") as f: