diff --git a/src/components/archive_synchronizer.py b/src/components/archive_synchronizer.py index 9177ac2..d7dc30f 100644 --- a/src/components/archive_synchronizer.py +++ b/src/components/archive_synchronizer.py @@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component): self.machine_id = self.config.machine_id if "--dev-portal" in sys.argv: self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/" - self.base_media_url = f"https://dev.r5portal.it/media/uploads/" + self.base_media_url = f"https://dev.r5portal.it/media/uploads" else: self.archive_endpoint = self.config[self.name]["archive_endpoint"] #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"] @@ -155,7 +155,6 @@ class ArchiveSynchronizer(Component): self.log.info(f"id: {record.id}: stored remotely") return True - def remote_fetch(self, machine_id, subfolder="warning_images"): """ Fetch and download all files from the server's folder. @@ -171,24 +170,35 @@ class ArchiveSynchronizer(Component): with requests.Session() as s: s.mount("", HTTPAdapter(max_retries=Retry(total=0))) - # Get directory listing + self.log.info(f"Attempting to fetch directory listing from: {base_url}") + + # Make the HTTP GET request to the directory URL response = s.get(base_url, timeout=5, verify=False) - if response.status_code == 403: + # Log response details + self.log.info(f"HTTP Status Code: {response.status_code}") + self.log.info(f"Response Headers: {response.headers}") + + # Handle HTTP errors appropriately + if response.status_code == 404: + self.log.warning( + f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.") + return False + elif response.status_code == 403: self.log.warning(f"Access forbidden for directory listing: {base_url}") return False elif response.status_code != 200: - raise AssertionError(f"bad status response for directory listing: {base_url}") + self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}") + return False - # Parse the response content to get all file URLs. - soup = BeautifulSoup(response.content, 'html.parser') + # Manually parse the response content to get all file URLs + content = response.content.decode('iso-8859-1') # or appropriate charset file_urls = [] - for link in soup.find_all('a'): - href = link.get('href') - if href and re.search(r'\.(png|csv|svg)$', href, - re.IGNORECASE): # Adjust file extensions as needed - file_urls.append(f"{self.base_media_url}/{machine_id}/{href}") + matches = re.findall(r'href="([^"]+\.(?:png|csv|svg))"', content, re.IGNORECASE) + for href in matches: + file_urls.append(f"{base_url}{href}") + # Download each file for file_url in file_urls: file_response = s.get(file_url, timeout=5, verify=False) @@ -196,9 +206,11 @@ class ArchiveSynchronizer(Component): self.log.warning(f"Access forbidden for file URL: {file_url}") continue elif file_response.status_code != 200: - raise AssertionError(f"bad status response for {file_url}") + self.log.error( + f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}") + continue - local_file_path = os.path.join("downloaded_files", os.path.basename(file_url)) + local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url)) os.makedirs(os.path.dirname(local_file_path), exist_ok=True) with open(local_file_path, "wb") as f: @@ -215,4 +227,4 @@ class ArchiveSynchronizer(Component): except Exception as e: self.log.error(f"An unexpected error occurred: {str(e)}") return False - return responses \ No newline at end of file + return responses