diff --git a/src/components/archive_synchronizer.py b/src/components/archive_synchronizer.py index 05326d4..0ab74c6 100644 --- a/src/components/archive_synchronizer.py +++ b/src/components/archive_synchronizer.py @@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component): self.machine_id = self.config.machine_id if "--dev-portal" in sys.argv: self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/" - self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv" + #self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv" else: self.archive_endpoint = self.config[self.name]["archive_endpoint"] #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"] @@ -57,7 +57,7 @@ class ArchiveSynchronizer(Component): self.gcs_bucket = None if "--dev-portal" in sys.argv: self.update_machine_status() - self.remote_fetch(self.url) + self.remote_fetch(self.machine_status) super()._get() def update_machine_status(self): @@ -155,59 +155,107 @@ class ArchiveSynchronizer(Component): self.log.info(f"id: {record.id}: stored remotely") return True - def remote_fetch(self, machine_status): + def remote_fetch(self, machine_status="logged-in"): """ - Fetch and download a specific file from the server. + Fetch and download files from the server. - :param specific_file_url: The full URL of the file to download. - :return: The downloaded file path or False if an error occurs. + :param machine_status: Status of the machine (e.g., 'logged-in'). + :return: A list of downloaded file paths or a dictionary with errors if any occur. """ + # URLs endpoint from your Django server + urls_endpoint = "http://dev.r5portal.it/get-file-urls/" + if machine_status == "logged-in": try: - specific_file_url = "https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv" + # Fetching the list of specific file URLs from the Django server if not self.simulate: with requests.Session() as s: - self.log.info(f"Attempting to fetch file from: {specific_file_url}") + self.log.info(f"Fetching list of file URLs from: {urls_endpoint}") - # Make the HTTP GET request to fetch the specific file URL - response = s.get(specific_file_url, timeout=5, verify=False) + # Make the HTTP GET request to fetch the list of URLs + response = s.get(urls_endpoint, timeout=5, verify=False) # Log response details self.log.info(f"HTTP Status Code: {response.status_code}") self.log.info(f"Response Headers: {response.headers}") - - # Handle HTTP errors appropriately + self.log.info(f"Response Content: {response.content}") + # Handle HTTP errors for the list of URLs fetch if response.status_code == 404: - self.log.warning( - f"File not found: {specific_file_url}. Please check the URL path and ensure the server has the expected file.") - return False + self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.") + return {"error": "URLs endpoint not found"} elif response.status_code == 403: - self.log.warning(f"Access forbidden for file: {specific_file_url}") - return False + self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}") + return {"error": "Access forbidden"} elif response.status_code != 200: self.log.error( - f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}") - return False + f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}") + return {"error": "Unexpected HTTP response status"} - # Save the file to /tmp/ - project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp" - local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url)) - os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + # Parse the list of URLs (assuming the response contains a JSON array of URLs) + try: + file_urls = response.json() + self.log.info(f"Fetched file URLs: {file_urls}") + except ValueError as e: + self.log.error(f"Error parsing JSON response: {str(e)}") + self.log.error(f"Response Content: {response.content}") + return {"error": "Failed to parse JSON response"} + downloaded_files = [] + errors = {} - with open(local_file_path, "wb") as f: - f.write(response.content) + # Iterate over the list of URLs to download each file + for specific_file_url in file_urls: + try: + self.log.info(f"Attempting to fetch file from: {specific_file_url}") - self.log.info(f"File downloaded successfully: {local_file_path}") - return local_file_path + # Make the HTTP GET request to fetch the file URL + response = s.get(specific_file_url, timeout=5, verify=False) + + # Log response details + self.log.info(f"HTTP Status Code: {response.status_code}") + self.log.info(f"Response Headers: {response.headers}") + + # Handle HTTP errors appropriately for each file + if response.status_code == 404: + self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.") + errors[specific_file_url] = "File not found" + continue + elif response.status_code == 403: + self.log.warning(f"Access forbidden for file: {specific_file_url}") + errors[specific_file_url] = "Access forbidden" + continue + elif response.status_code != 200: + self.log.error( + f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}") + errors[specific_file_url] = f"Unexpected status {response.status_code}" + continue + + # Save the file to /tmp/ + project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp" + local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url)) + os.makedirs(os.path.dirname(local_file_path), exist_ok=True) + + with open(local_file_path, "wb") as f: + f.write(response.content) + + self.log.info(f"File downloaded successfully: {local_file_path}") + downloaded_files.append(local_file_path) + + except (requests.ConnectionError, requests.Timeout) as e: + self.log.warning( + f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}") + errors[specific_file_url] = "URL unreachable" + except Exception as e: + self.log.error( + f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}") + errors[specific_file_url] = "Unexpected error" + + # Return the list of downloaded files and any errors + return { + "downloaded_files": downloaded_files, + "errors": errors + } - except AssertionError as e: - self.log.warning(f"Failed to download file: {str(e)}") - return False - except (requests.ConnectionError, requests.Timeout) as e: - self.log.warning(f"Failed to download file, the URL might be unreachable: {str(e)}") - return False except Exception as e: - self.log.error(f"An unexpected error occurred: {str(e)}") - return False - + self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}") + return {"error": "Unexpected error"}