Eduardo_Scripniciuc/st-ten-1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor remote_fetch and add check_actions_to_do

Browse Source 
Simplify the remote_fetch method by removing machine_status parameter and updating error handling. Introduce check_actions_to_do method to determine necessary actions for the machine, including invoking remote_fetch as a required action.
This commit is contained in:
edo-neo 2024-10-29 17:28:35 +01:00
parent 3f4f0aab82
commit 118c0367b9
1 changed files with 113 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
src/components/archive_synchronizer.py
View File

@ -11,6 +11,8 @@ import traceback
import requests import requests
from google.api_core.exceptions import Forbidden from google.api_core.exceptions import Forbidden
from google.cloud import storage from google.cloud import storage
from requests import JSONDecodeError
from lib.db import Archive, db from lib.db import Archive, db
from PyQt5.QtCore import QThread from PyQt5.QtCore import QThread
from requests.adapters import HTTPAdapter, Retry from requests.adapters import HTTPAdapter, Retry
@ -59,7 +61,7 @@ class ArchiveSynchronizer(Component):
if "--dev-portal" in sys.argv: if "--dev-portal" in sys.argv:
self.update_machine_status() self.update_machine_status()
self.remote_fetch(self.machine_status,self.machine_id) self.check_actions_to_do(self.machine_id)
super()._get() super()._get()
def update_machine_status(self): def update_machine_status(self):
@ -152,123 +154,135 @@ class ArchiveSynchronizer(Component):
self.log.info(f"id: {record.id}: stored remotely") self.log.info(f"id: {record.id}: stored remotely")
return True return True
def remote_fetch(self, machine_status=None, machine_id=None): def remote_fetch(self, machine_id=None):
""" """
Fetch and download files from the server. Fetch and download files from the server.
:param machine_status: Status of the machine (e.g., 'logged-in').
:param machine_id: ID of the machine. :param machine_id: ID of the machine.
:return: A list of downloaded file paths or a dictionary with errors if any occur. :return: A list of downloaded file paths or a dictionary with errors if any occur.
""" """
# URLs endpoint from your Django server with machine_id as a query parameter
if machine_id is None: if machine_id is None:
raise ValueError("machine_id cannot be None") raise ValueError("machine_id cannot be None")
urls_endpoint = f"http://dev.r5portal.it/get-file-urls/?machine_id={machine_id}" urls_endpoint = f"http://dev.r5portal.it/get-file-urls/?machine_id={machine_id}"
if machine_status == "logged-in": try:
try: if not self.simulate:
# Fetching the list of specific file URLs from the Django server with requests.Session() as s:
if not self.simulate: self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
with requests.Session() as s:
self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
# Make the HTTP GET request to fetch the list of URLs # Make the HTTP GET request to fetch the list of URLs
response = s.get(urls_endpoint, timeout=5, verify=False) response = s.get(urls_endpoint, timeout=5, verify=False)
# Log response details # Log response details
self.log.info(f"HTTP Status Code: {response.status_code}") self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}") self.log.info(f"Response Headers: {response.headers}")
self.log.info(f"Response Content: {response.content}") self.log.info(f"Response Content: {response.content}")
# Handle HTTP errors for the list of URLs fetch # Handle HTTP errors for the list of URLs fetch
if response.status_code == 404: if response.status_code == 404:
self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.") self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
return {"error": "URLs endpoint not found"} return {"error": "URLs endpoint not found"}
elif response.status_code == 403: elif response.status_code == 403 or response.status_code == 401:
self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}") self.log.warning(f"Access forbidden or not logged in for URLs endpoint: {urls_endpoint}")
return {"error": "Access forbidden"} return {"error": "Access forbidden or not logged in"}
elif response.status_code != 200: elif response.status_code != 200:
self.log.error( self.log.error(
f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}") f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
return {"error": "Unexpected HTTP response status"} return {"error": "Unexpected HTTP response status"}
# Parse the list of URLs (assuming the response contains a JSON array of URLs) # Parse the list of URLs (assuming the response contains a JSON array of URLs)
try:
file_urls = response.json()
self.log.info(f"Fetched file URLs: {file_urls}")
except ValueError as e:
self.log.error(f"Error parsing JSON response: {str(e)}")
self.log.error(f"Response Content: {response.content}")
return {"error": "Failed to parse JSON response"}
downloaded_files = []
errors = {}
# Setup the base download path
base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1")
# Iterate over the list of URLs to download each file
for specific_file_url in file_urls:
try: try:
file_urls = response.json() self.log.info(f"Attempting to fetch file from: {specific_file_url}")
self.log.info(f"Fetched file URLs: {file_urls}")
except ValueError as e:
self.log.error(f"Error parsing JSON response: {str(e)}")
self.log.error(f"Response Content: {response.content}")
return {"error": "Failed to parse JSON response"}
downloaded_files = [] # Make the HTTP GET request to fetch the file URL
errors = {} response = s.get(specific_file_url, timeout=5, verify=False)
# Setup the base download path # Log response details
base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1") self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}")
# Iterate over the list of URLs to download each file # Handle HTTP errors appropriately for each file
for specific_file_url in file_urls: if response.status_code == 404:
try: self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
self.log.info(f"Attempting to fetch file from: {specific_file_url}") errors[specific_file_url] = "File not found"
continue
# Make the HTTP GET request to fetch the file URL elif response.status_code == 403:
response = s.get(specific_file_url, timeout=5, verify=False) self.log.warning(f"Access forbidden for file: {specific_file_url}")
errors[specific_file_url] = "Access forbidden"
# Log response details continue
self.log.info(f"HTTP Status Code: {response.status_code}") elif response.status_code != 200:
self.log.info(f"Response Headers: {response.headers}")
# Handle HTTP errors appropriately for each file
if response.status_code == 404:
self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
errors[specific_file_url] = "File not found"
continue
elif response.status_code == 403:
self.log.warning(f"Access forbidden for file: {specific_file_url}")
errors[specific_file_url] = "Access forbidden"
continue
elif response.status_code != 200:
self.log.error(
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
errors[specific_file_url] = f"Unexpected status {response.status_code}"
continue
# Determine the subdirectory based on the file URL or other criteria
if "warning_images" in specific_file_url:
sub_dir = os.path.join("config", "warning_images", machine_id)
elif "ricette" in specific_file_url:
sub_dir = os.path.join("config", "csv_import", "auto_csv_import")
else:
sub_dir = "tmp" # Save others under the temporary folder
# Ensure the directory exists
download_path = os.path.join(base_download_path, sub_dir)
os.makedirs(download_path, exist_ok=True)
# Save the file to the determined directory
local_file_path = os.path.join(download_path, os.path.basename(specific_file_url))
with open(local_file_path, "wb") as f:
f.write(response.content)
self.log.info(f"File downloaded successfully: {local_file_path}")
downloaded_files.append(local_file_path)
except (requests.ConnectionError, requests.Timeout) as e:
self.log.warning(
f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
errors[specific_file_url] = "URL unreachable"
except Exception as e:
self.log.error( self.log.error(
f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}") f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
errors[specific_file_url] = "Unexpected error" errors[specific_file_url] = f"Unexpected status {response.status_code}"
continue
return { # Determine the subdirectory based on the file URL or other criteria
"downloaded_files": downloaded_files, if "warning_images" in specific_file_url:
"errors": errors sub_dir = os.path.join("config", "warning_images", machine_id)
} elif "ricette" in specific_file_url:
sub_dir = os.path.join("config", "csv_import", "auto_csv_import")
else:
sub_dir = "tmp" # Save others under the temporary folder
except Exception as e: # Ensure the directory exists
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}") download_path = os.path.join(base_download_path, sub_dir)
return {"error": "Unexpected error"} os.makedirs(download_path, exist_ok=True)
# Save the file to the determined directory
local_file_path = os.path.join(download_path, os.path.basename(specific_file_url))
with open(local_file_path, "wb") as f:
f.write(response.content)
self.log.info(f"File downloaded successfully: {local_file_path}")
downloaded_files.append(local_file_path)
except (requests.ConnectionError, requests.Timeout) as e:
self.log.warning(
f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
errors[specific_file_url] = "URL unreachable"
except Exception as e:
self.log.error(
f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
errors[specific_file_url] = "Unexpected error"
return {
"downloaded_files": downloaded_files,
"errors": errors
}
except Exception as e:
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
return {"error": "Unexpected error"}
def check_actions_to_do(self, machine_id: str) :
update_url = f"http://dev.r5portal.it/device/?machine-id={machine_id}"
response = requests.get(update_url)
if response.status_code == 200:
try:
#json_response = response.json()
#action_to_do = json_response.get("ACTIONS_TO_DO", "NO_ACTION")
#if action_to_do == "DOWNLOAD_FILES":
self.remote_fetch(machine_id)
#else:
# self.log.info(f"No action required: {json_response}")
except JSONDecodeError as e:
self.log.error(f"JSON decode error: {str(e)} - Content: {response.content}")
else:
self.log.error(f"Failed to update device info: {response.status_code} - {response.content}")