Refactor remote_fetch and add check_actions_to_do
Simplify the remote_fetch method by removing machine_status parameter and updating error handling. Introduce check_actions_to_do method to determine necessary actions for the machine, including invoking remote_fetch as a required action.
This commit is contained in:
parent
3f4f0aab82
commit
118c0367b9
@ -11,6 +11,8 @@ import traceback
|
||||
import requests
|
||||
from google.api_core.exceptions import Forbidden
|
||||
from google.cloud import storage
|
||||
from requests import JSONDecodeError
|
||||
|
||||
from lib.db import Archive, db
|
||||
from PyQt5.QtCore import QThread
|
||||
from requests.adapters import HTTPAdapter, Retry
|
||||
@ -59,7 +61,7 @@ class ArchiveSynchronizer(Component):
|
||||
|
||||
if "--dev-portal" in sys.argv:
|
||||
self.update_machine_status()
|
||||
self.remote_fetch(self.machine_status,self.machine_id)
|
||||
self.check_actions_to_do(self.machine_id)
|
||||
super()._get()
|
||||
|
||||
def update_machine_status(self):
|
||||
@ -152,123 +154,135 @@ class ArchiveSynchronizer(Component):
|
||||
self.log.info(f"id: {record.id}: stored remotely")
|
||||
return True
|
||||
|
||||
def remote_fetch(self, machine_status=None, machine_id=None):
|
||||
def remote_fetch(self, machine_id=None):
|
||||
"""
|
||||
Fetch and download files from the server.
|
||||
|
||||
:param machine_status: Status of the machine (e.g., 'logged-in').
|
||||
:param machine_id: ID of the machine.
|
||||
:return: A list of downloaded file paths or a dictionary with errors if any occur.
|
||||
"""
|
||||
# URLs endpoint from your Django server with machine_id as a query parameter
|
||||
if machine_id is None:
|
||||
raise ValueError("machine_id cannot be None")
|
||||
urls_endpoint = f"http://dev.r5portal.it/get-file-urls/?machine_id={machine_id}"
|
||||
|
||||
if machine_status == "logged-in":
|
||||
try:
|
||||
# Fetching the list of specific file URLs from the Django server
|
||||
if not self.simulate:
|
||||
with requests.Session() as s:
|
||||
self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
|
||||
try:
|
||||
if not self.simulate:
|
||||
with requests.Session() as s:
|
||||
self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
|
||||
|
||||
# Make the HTTP GET request to fetch the list of URLs
|
||||
response = s.get(urls_endpoint, timeout=5, verify=False)
|
||||
# Make the HTTP GET request to fetch the list of URLs
|
||||
response = s.get(urls_endpoint, timeout=5, verify=False)
|
||||
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
self.log.info(f"Response Content: {response.content}")
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
self.log.info(f"Response Content: {response.content}")
|
||||
|
||||
# Handle HTTP errors for the list of URLs fetch
|
||||
if response.status_code == 404:
|
||||
self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
|
||||
return {"error": "URLs endpoint not found"}
|
||||
elif response.status_code == 403:
|
||||
self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}")
|
||||
return {"error": "Access forbidden"}
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
|
||||
return {"error": "Unexpected HTTP response status"}
|
||||
# Handle HTTP errors for the list of URLs fetch
|
||||
if response.status_code == 404:
|
||||
self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
|
||||
return {"error": "URLs endpoint not found"}
|
||||
elif response.status_code == 403 or response.status_code == 401:
|
||||
self.log.warning(f"Access forbidden or not logged in for URLs endpoint: {urls_endpoint}")
|
||||
return {"error": "Access forbidden or not logged in"}
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
|
||||
return {"error": "Unexpected HTTP response status"}
|
||||
|
||||
# Parse the list of URLs (assuming the response contains a JSON array of URLs)
|
||||
# Parse the list of URLs (assuming the response contains a JSON array of URLs)
|
||||
try:
|
||||
file_urls = response.json()
|
||||
self.log.info(f"Fetched file URLs: {file_urls}")
|
||||
except ValueError as e:
|
||||
self.log.error(f"Error parsing JSON response: {str(e)}")
|
||||
self.log.error(f"Response Content: {response.content}")
|
||||
return {"error": "Failed to parse JSON response"}
|
||||
|
||||
downloaded_files = []
|
||||
errors = {}
|
||||
|
||||
# Setup the base download path
|
||||
base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1")
|
||||
|
||||
# Iterate over the list of URLs to download each file
|
||||
for specific_file_url in file_urls:
|
||||
try:
|
||||
file_urls = response.json()
|
||||
self.log.info(f"Fetched file URLs: {file_urls}")
|
||||
except ValueError as e:
|
||||
self.log.error(f"Error parsing JSON response: {str(e)}")
|
||||
self.log.error(f"Response Content: {response.content}")
|
||||
return {"error": "Failed to parse JSON response"}
|
||||
self.log.info(f"Attempting to fetch file from: {specific_file_url}")
|
||||
|
||||
downloaded_files = []
|
||||
errors = {}
|
||||
# Make the HTTP GET request to fetch the file URL
|
||||
response = s.get(specific_file_url, timeout=5, verify=False)
|
||||
|
||||
# Setup the base download path
|
||||
base_download_path = os.path.join(Path.home(), "PycharmProjects", "st-ten-1")
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
|
||||
# Iterate over the list of URLs to download each file
|
||||
for specific_file_url in file_urls:
|
||||
try:
|
||||
self.log.info(f"Attempting to fetch file from: {specific_file_url}")
|
||||
|
||||
# Make the HTTP GET request to fetch the file URL
|
||||
response = s.get(specific_file_url, timeout=5, verify=False)
|
||||
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
|
||||
# Handle HTTP errors appropriately for each file
|
||||
if response.status_code == 404:
|
||||
self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
|
||||
errors[specific_file_url] = "File not found"
|
||||
continue
|
||||
elif response.status_code == 403:
|
||||
self.log.warning(f"Access forbidden for file: {specific_file_url}")
|
||||
errors[specific_file_url] = "Access forbidden"
|
||||
continue
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
|
||||
errors[specific_file_url] = f"Unexpected status {response.status_code}"
|
||||
continue
|
||||
|
||||
# Determine the subdirectory based on the file URL or other criteria
|
||||
if "warning_images" in specific_file_url:
|
||||
sub_dir = os.path.join("config", "warning_images", machine_id)
|
||||
elif "ricette" in specific_file_url:
|
||||
sub_dir = os.path.join("config", "csv_import", "auto_csv_import")
|
||||
else:
|
||||
sub_dir = "tmp" # Save others under the temporary folder
|
||||
|
||||
# Ensure the directory exists
|
||||
download_path = os.path.join(base_download_path, sub_dir)
|
||||
os.makedirs(download_path, exist_ok=True)
|
||||
|
||||
# Save the file to the determined directory
|
||||
local_file_path = os.path.join(download_path, os.path.basename(specific_file_url))
|
||||
with open(local_file_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
self.log.info(f"File downloaded successfully: {local_file_path}")
|
||||
downloaded_files.append(local_file_path)
|
||||
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
self.log.warning(
|
||||
f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
|
||||
errors[specific_file_url] = "URL unreachable"
|
||||
except Exception as e:
|
||||
# Handle HTTP errors appropriately for each file
|
||||
if response.status_code == 404:
|
||||
self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
|
||||
errors[specific_file_url] = "File not found"
|
||||
continue
|
||||
elif response.status_code == 403:
|
||||
self.log.warning(f"Access forbidden for file: {specific_file_url}")
|
||||
errors[specific_file_url] = "Access forbidden"
|
||||
continue
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
|
||||
errors[specific_file_url] = "Unexpected error"
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
|
||||
errors[specific_file_url] = f"Unexpected status {response.status_code}"
|
||||
continue
|
||||
|
||||
return {
|
||||
"downloaded_files": downloaded_files,
|
||||
"errors": errors
|
||||
}
|
||||
# Determine the subdirectory based on the file URL or other criteria
|
||||
if "warning_images" in specific_file_url:
|
||||
sub_dir = os.path.join("config", "warning_images", machine_id)
|
||||
elif "ricette" in specific_file_url:
|
||||
sub_dir = os.path.join("config", "csv_import", "auto_csv_import")
|
||||
else:
|
||||
sub_dir = "tmp" # Save others under the temporary folder
|
||||
|
||||
except Exception as e:
|
||||
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
|
||||
return {"error": "Unexpected error"}
|
||||
# Ensure the directory exists
|
||||
download_path = os.path.join(base_download_path, sub_dir)
|
||||
os.makedirs(download_path, exist_ok=True)
|
||||
|
||||
# Save the file to the determined directory
|
||||
local_file_path = os.path.join(download_path, os.path.basename(specific_file_url))
|
||||
with open(local_file_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
self.log.info(f"File downloaded successfully: {local_file_path}")
|
||||
downloaded_files.append(local_file_path)
|
||||
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
self.log.warning(
|
||||
f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
|
||||
errors[specific_file_url] = "URL unreachable"
|
||||
except Exception as e:
|
||||
self.log.error(
|
||||
f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
|
||||
errors[specific_file_url] = "Unexpected error"
|
||||
|
||||
return {
|
||||
"downloaded_files": downloaded_files,
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
|
||||
return {"error": "Unexpected error"}
|
||||
|
||||
def check_actions_to_do(self, machine_id: str) :
|
||||
update_url = f"http://dev.r5portal.it/device/?machine-id={machine_id}"
|
||||
response = requests.get(update_url)
|
||||
|
||||
if response.status_code == 200:
|
||||
try:
|
||||
#json_response = response.json()
|
||||
#action_to_do = json_response.get("ACTIONS_TO_DO", "NO_ACTION")
|
||||
#if action_to_do == "DOWNLOAD_FILES":
|
||||
self.remote_fetch(machine_id)
|
||||
#else:
|
||||
# self.log.info(f"No action required: {json_response}")
|
||||
except JSONDecodeError as e:
|
||||
self.log.error(f"JSON decode error: {str(e)} - Content: {response.content}")
|
||||
else:
|
||||
self.log.error(f"Failed to update device info: {response.status_code} - {response.content}")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user