From 118c0367b9d9d03a05a63855a371dd79b982579d Mon Sep 17 00:00:00 2001 From: edo-neo Date: Tue, 29 Oct 2024 17:28:35 +0100 Subject: [PATCH] Refactor remote_fetch and add check_actions_to_do Simplify the remote_fetch method by removing machine_status parameter and updating error handling. Introduce check_actions_to_do method to determine necessary actions for the machine, including invoking remote_fetch as a required action. --- src/components/archive_synchronizer.py | 212 +++++++++++++------------ 1 file changed, 113 insertions(+), 99 deletions(-) diff --git a/src/components/archive_synchronizer.py b/src/components/archive_synchronizer.py index 13ea82d..8fdc591 100644 --- a/src/components/archive_synchronizer.py +++ b/src/components/archive_synchronizer.py @@ -11,6 +11,8 @@ import traceback import requests from google.api_core.exceptions import Forbidden from google.cloud import storage +from requests import JSONDecodeError + from lib.db import Archive, db from PyQt5.QtCore import QThread from requests.adapters import HTTPAdapter, Retry @@ -59,7 +61,7 @@ class ArchiveSynchronizer(Component): if "--dev-portal" in sys.argv: self.update_machine_status() - self.remote_fetch(self.machine_status,self.machine_id) + self.check_actions_to_do(self.machine_id) super()._get() def update_machine_status(self): @@ -152,123 +154,135 @@ class ArchiveSynchronizer(Component): self.log.info(f"id: {record.id}: stored remotely") return True - def remote_fetch(self, machine_status=None, machine_id=None): + def remote_fetch(self, machine_id=None): """ Fetch and download files from the server. - :param machine_status: Status of the machine (e.g., 'logged-in'). :param machine_id: ID of the machine. :return: A list of downloaded file paths or a dictionary with errors if any occur. """ - # URLs endpoint from your Django server with machine_id as a query parameter if machine_id is None: raise ValueError("machine_id cannot be None") urls_endpoint = f"http://dev.r5portal.it/get-file-urls/?machine_id={machine_id}" - if machine_status == "logged-in": - try: - # Fetching the list of specific file URLs from the Django server - if not self.simulate: - with requests.Session() as s: - self.log.info(f"Fetching list of file URLs from: {urls_endpoint}") + try: + if not self.simulate: + with requests.Session() as s: + self.log.info(f"Fetching list of file URLs from: {urls_endpoint}") - # Make the HTTP GET request to fetch the list of URLs - response = s.get(urls_endpoint, timeout=5, verify=False) + # Make the HTTP GET request to fetch the list of URLs + response = s.get(urls_endpoint, timeout=5, verify=False) - # Log response details - self.log.info(f"HTTP Status Code: {response.status_code}") - self.log.info(f"Response Headers: {response.headers}") - self.log.info(f"Response Content: {response.content}") + # Log response details + self.log.info(f"HTTP Status Code: {response.status_code}") + self.log.info(f"Response Headers: {response.headers}") + self.log.info(f"Response Content: {response.content}") - # Handle HTTP errors for the list of URLs fetch - if response.status_code == 404: - self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.") - return {"error": "URLs endpoint not found"} - elif response.status_code == 403: - self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}") - return {"error": "Access forbidden"} - elif response.status_code != 200: - self.log.error( - f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}") - return {"error": "Unexpected HTTP response status"} + # Handle HTTP errors for the list of URLs fetch + if response.status_code == 404: + self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.") + return {"error": "URLs endpoint not found"} + elif response.status_code == 403 or response.status_code == 401: + self.log.warning(f"Access forbidden or not logged in for URLs endpoint: {urls_endpoint}") + return {"error": "Access forbidden or not logged in"} + elif response.status_code != 200: + self.log.error( + f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}") + return {"error": "Unexpected HTTP response status"} - # Parse the list of URLs (assuming the response contains a JSON array of URLs) + # Parse the list of URLs (assuming the response contains a JSON array of URLs) + try: + file_urls = response.json() + self.log.info(f"Fetched file URLs: {file_urls}") + except ValueError as e: + self.log.error(f"Error parsing JSON response: {str(e)}") + self.log.error(f"Response Content: {response.content}") + return {"error": "Failed to parse JSON response"} + + downloaded_files = [] + errors = {} + + # Setup the base download path + base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1") + + # Iterate over the list of URLs to download each file + for specific_file_url in file_urls: try: - file_urls = response.json() - self.log.info(f"Fetched file URLs: {file_urls}") - except ValueError as e: - self.log.error(f"Error parsing JSON response: {str(e)}") - self.log.error(f"Response Content: {response.content}") - return {"error": "Failed to parse JSON response"} + self.log.info(f"Attempting to fetch file from: {specific_file_url}") - downloaded_files = [] - errors = {} + # Make the HTTP GET request to fetch the file URL + response = s.get(specific_file_url, timeout=5, verify=False) - # Setup the base download path - base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1") + # Log response details + self.log.info(f"HTTP Status Code: {response.status_code}") + self.log.info(f"Response Headers: {response.headers}") - # Iterate over the list of URLs to download each file - for specific_file_url in file_urls: - try: - self.log.info(f"Attempting to fetch file from: {specific_file_url}") - - # Make the HTTP GET request to fetch the file URL - response = s.get(specific_file_url, timeout=5, verify=False) - - # Log response details - self.log.info(f"HTTP Status Code: {response.status_code}") - self.log.info(f"Response Headers: {response.headers}") - - # Handle HTTP errors appropriately for each file - if response.status_code == 404: - self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.") - errors[specific_file_url] = "File not found" - continue - elif response.status_code == 403: - self.log.warning(f"Access forbidden for file: {specific_file_url}") - errors[specific_file_url] = "Access forbidden" - continue - elif response.status_code != 200: - self.log.error( - f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}") - errors[specific_file_url] = f"Unexpected status {response.status_code}" - continue - - # Determine the subdirectory based on the file URL or other criteria - if "warning_images" in specific_file_url: - sub_dir = os.path.join("config", "warning_images", machine_id) - elif "ricette" in specific_file_url: - sub_dir = os.path.join("config", "csv_import", "auto_csv_import") - else: - sub_dir = "tmp" # Save others under the temporary folder - - # Ensure the directory exists - download_path = os.path.join(base_download_path, sub_dir) - os.makedirs(download_path, exist_ok=True) - - # Save the file to the determined directory - local_file_path = os.path.join(download_path, os.path.basename(specific_file_url)) - with open(local_file_path, "wb") as f: - f.write(response.content) - - self.log.info(f"File downloaded successfully: {local_file_path}") - downloaded_files.append(local_file_path) - - except (requests.ConnectionError, requests.Timeout) as e: - self.log.warning( - f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}") - errors[specific_file_url] = "URL unreachable" - except Exception as e: + # Handle HTTP errors appropriately for each file + if response.status_code == 404: + self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.") + errors[specific_file_url] = "File not found" + continue + elif response.status_code == 403: + self.log.warning(f"Access forbidden for file: {specific_file_url}") + errors[specific_file_url] = "Access forbidden" + continue + elif response.status_code != 200: self.log.error( - f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}") - errors[specific_file_url] = "Unexpected error" + f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}") + errors[specific_file_url] = f"Unexpected status {response.status_code}" + continue - return { - "downloaded_files": downloaded_files, - "errors": errors - } + # Determine the subdirectory based on the file URL or other criteria + if "warning_images" in specific_file_url: + sub_dir = os.path.join("config", "warning_images", machine_id) + elif "ricette" in specific_file_url: + sub_dir = os.path.join("config", "csv_import", "auto_csv_import") + else: + sub_dir = "tmp" # Save others under the temporary folder - except Exception as e: - self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}") - return {"error": "Unexpected error"} + # Ensure the directory exists + download_path = os.path.join(base_download_path, sub_dir) + os.makedirs(download_path, exist_ok=True) + # Save the file to the determined directory + local_file_path = os.path.join(download_path, os.path.basename(specific_file_url)) + with open(local_file_path, "wb") as f: + f.write(response.content) + + self.log.info(f"File downloaded successfully: {local_file_path}") + downloaded_files.append(local_file_path) + + except (requests.ConnectionError, requests.Timeout) as e: + self.log.warning( + f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}") + errors[specific_file_url] = "URL unreachable" + except Exception as e: + self.log.error( + f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}") + errors[specific_file_url] = "Unexpected error" + + return { + "downloaded_files": downloaded_files, + "errors": errors + } + + except Exception as e: + self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}") + return {"error": "Unexpected error"} + + def check_actions_to_do(self, machine_id: str) : + update_url = f"http://dev.r5portal.it/device/?machine-id={machine_id}" + response = requests.get(update_url) + + if response.status_code == 200: + try: + #json_response = response.json() + #action_to_do = json_response.get("ACTIONS_TO_DO", "NO_ACTION") + #if action_to_do == "DOWNLOAD_FILES": + self.remote_fetch(machine_id) + #else: + # self.log.info(f"No action required: {json_response}") + except JSONDecodeError as e: + self.log.error(f"JSON decode error: {str(e)} - Content: {response.content}") + else: + self.log.error(f"Failed to update device info: {response.status_code} - {response.content}")