Eduardo_Scripniciuc/st-ten-1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev.r5portal remote_fetch

Browse Source 
ok
This commit is contained in:
edo-neo 2024-10-23 11:30:30 +02:00
parent 7261f2d1e0
commit 2c01810685
1 changed files with 84 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

92
src/components/archive_synchronizer.py
View File

@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
self.machine_id = self.config.machine_id self.machine_id = self.config.machine_id
if "--dev-portal" in sys.argv: if "--dev-portal" in sys.argv:
self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/" self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv" #self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
else: else:
self.archive_endpoint = self.config[self.name]["archive_endpoint"] self.archive_endpoint = self.config[self.name]["archive_endpoint"]
#self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"] #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
@ -57,7 +57,7 @@ class ArchiveSynchronizer(Component):
self.gcs_bucket = None self.gcs_bucket = None
if "--dev-portal" in sys.argv: if "--dev-portal" in sys.argv:
self.update_machine_status() self.update_machine_status()
self.remote_fetch(self.url) self.remote_fetch(self.machine_status)
super()._get() super()._get()
def update_machine_status(self): def update_machine_status(self):
@ -155,39 +155,79 @@ class ArchiveSynchronizer(Component):
self.log.info(f"id: {record.id}: stored remotely") self.log.info(f"id: {record.id}: stored remotely")
return True return True
def remote_fetch(self, machine_status): def remote_fetch(self, machine_status="logged-in"):
""" """
Fetch and download a specific file from the server. Fetch and download files from the server.
:param specific_file_url: The full URL of the file to download. :param machine_status: Status of the machine (e.g., 'logged-in').
:return: The downloaded file path or False if an error occurs. :return: A list of downloaded file paths or a dictionary with errors if any occur.
""" """
# URLs endpoint from your Django server
urls_endpoint = "http://dev.r5portal.it/get-file-urls/"
if machine_status == "logged-in": if machine_status == "logged-in":
try: try:
specific_file_url = "https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv" # Fetching the list of specific file URLs from the Django server
if not self.simulate: if not self.simulate:
with requests.Session() as s: with requests.Session() as s:
self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
# Make the HTTP GET request to fetch the list of URLs
response = s.get(urls_endpoint, timeout=5, verify=False)
# Log response details
self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}")
self.log.info(f"Response Content: {response.content}")
# Handle HTTP errors for the list of URLs fetch
if response.status_code == 404:
self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
return {"error": "URLs endpoint not found"}
elif response.status_code == 403:
self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}")
return {"error": "Access forbidden"}
elif response.status_code != 200:
self.log.error(
f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
return {"error": "Unexpected HTTP response status"}
# Parse the list of URLs (assuming the response contains a JSON array of URLs)
try:
file_urls = response.json()
self.log.info(f"Fetched file URLs: {file_urls}")
except ValueError as e:
self.log.error(f"Error parsing JSON response: {str(e)}")
self.log.error(f"Response Content: {response.content}")
return {"error": "Failed to parse JSON response"}
downloaded_files = []
errors = {}
# Iterate over the list of URLs to download each file
for specific_file_url in file_urls:
try:
self.log.info(f"Attempting to fetch file from: {specific_file_url}") self.log.info(f"Attempting to fetch file from: {specific_file_url}")
# Make the HTTP GET request to fetch the specific file URL # Make the HTTP GET request to fetch the file URL
response = s.get(specific_file_url, timeout=5, verify=False) response = s.get(specific_file_url, timeout=5, verify=False)
# Log response details # Log response details
self.log.info(f"HTTP Status Code: {response.status_code}") self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}") self.log.info(f"Response Headers: {response.headers}")
# Handle HTTP errors appropriately # Handle HTTP errors appropriately for each file
if response.status_code == 404: if response.status_code == 404:
self.log.warning( self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
f"File not found: {specific_file_url}. Please check the URL path and ensure the server has the expected file.") errors[specific_file_url] = "File not found"
return False continue
elif response.status_code == 403: elif response.status_code == 403:
self.log.warning(f"Access forbidden for file: {specific_file_url}") self.log.warning(f"Access forbidden for file: {specific_file_url}")
return False errors[specific_file_url] = "Access forbidden"
continue
elif response.status_code != 200: elif response.status_code != 200:
self.log.error( self.log.error(
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}") f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
return False errors[specific_file_url] = f"Unexpected status {response.status_code}"
continue
# Save the file to /tmp/ # Save the file to /tmp/
project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp" project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp"
@ -198,16 +238,24 @@ class ArchiveSynchronizer(Component):
f.write(response.content) f.write(response.content)
self.log.info(f"File downloaded successfully: {local_file_path}") self.log.info(f"File downloaded successfully: {local_file_path}")
return local_file_path downloaded_files.append(local_file_path)
except AssertionError as e:
self.log.warning(f"Failed to download file: {str(e)}")
return False
except (requests.ConnectionError, requests.Timeout) as e: except (requests.ConnectionError, requests.Timeout) as e:
self.log.warning(f"Failed to download file, the URL might be unreachable: {str(e)}") self.log.warning(
return False f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
errors[specific_file_url] = "URL unreachable"
except Exception as e: except Exception as e:
self.log.error(f"An unexpected error occurred: {str(e)}") self.log.error(
return False f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
errors[specific_file_url] = "Unexpected error"
# Return the list of downloaded files and any errors
return {
"downloaded_files": downloaded_files,
"errors": errors
}
except Exception as e:
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
return {"error": "Unexpected error"}