dev.r5portal remote_fetch
ok
This commit is contained in:
parent
7261f2d1e0
commit
2c01810685
@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
|
||||
self.machine_id = self.config.machine_id
|
||||
if "--dev-portal" in sys.argv:
|
||||
self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
|
||||
self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
|
||||
#self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
|
||||
else:
|
||||
self.archive_endpoint = self.config[self.name]["archive_endpoint"]
|
||||
#self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
|
||||
@ -57,7 +57,7 @@ class ArchiveSynchronizer(Component):
|
||||
self.gcs_bucket = None
|
||||
if "--dev-portal" in sys.argv:
|
||||
self.update_machine_status()
|
||||
self.remote_fetch(self.url)
|
||||
self.remote_fetch(self.machine_status)
|
||||
super()._get()
|
||||
|
||||
def update_machine_status(self):
|
||||
@ -155,59 +155,107 @@ class ArchiveSynchronizer(Component):
|
||||
self.log.info(f"id: {record.id}: stored remotely")
|
||||
return True
|
||||
|
||||
def remote_fetch(self, machine_status):
|
||||
def remote_fetch(self, machine_status="logged-in"):
|
||||
"""
|
||||
Fetch and download a specific file from the server.
|
||||
Fetch and download files from the server.
|
||||
|
||||
:param specific_file_url: The full URL of the file to download.
|
||||
:return: The downloaded file path or False if an error occurs.
|
||||
:param machine_status: Status of the machine (e.g., 'logged-in').
|
||||
:return: A list of downloaded file paths or a dictionary with errors if any occur.
|
||||
"""
|
||||
# URLs endpoint from your Django server
|
||||
urls_endpoint = "http://dev.r5portal.it/get-file-urls/"
|
||||
|
||||
if machine_status == "logged-in":
|
||||
try:
|
||||
specific_file_url = "https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
|
||||
# Fetching the list of specific file URLs from the Django server
|
||||
if not self.simulate:
|
||||
with requests.Session() as s:
|
||||
self.log.info(f"Attempting to fetch file from: {specific_file_url}")
|
||||
self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
|
||||
|
||||
# Make the HTTP GET request to fetch the specific file URL
|
||||
response = s.get(specific_file_url, timeout=5, verify=False)
|
||||
# Make the HTTP GET request to fetch the list of URLs
|
||||
response = s.get(urls_endpoint, timeout=5, verify=False)
|
||||
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
|
||||
# Handle HTTP errors appropriately
|
||||
self.log.info(f"Response Content: {response.content}")
|
||||
# Handle HTTP errors for the list of URLs fetch
|
||||
if response.status_code == 404:
|
||||
self.log.warning(
|
||||
f"File not found: {specific_file_url}. Please check the URL path and ensure the server has the expected file.")
|
||||
return False
|
||||
self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
|
||||
return {"error": "URLs endpoint not found"}
|
||||
elif response.status_code == 403:
|
||||
self.log.warning(f"Access forbidden for file: {specific_file_url}")
|
||||
return False
|
||||
self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}")
|
||||
return {"error": "Access forbidden"}
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
|
||||
return False
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
|
||||
return {"error": "Unexpected HTTP response status"}
|
||||
|
||||
# Save the file to /tmp/
|
||||
project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp"
|
||||
local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url))
|
||||
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
|
||||
# Parse the list of URLs (assuming the response contains a JSON array of URLs)
|
||||
try:
|
||||
file_urls = response.json()
|
||||
self.log.info(f"Fetched file URLs: {file_urls}")
|
||||
except ValueError as e:
|
||||
self.log.error(f"Error parsing JSON response: {str(e)}")
|
||||
self.log.error(f"Response Content: {response.content}")
|
||||
return {"error": "Failed to parse JSON response"}
|
||||
downloaded_files = []
|
||||
errors = {}
|
||||
|
||||
with open(local_file_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
# Iterate over the list of URLs to download each file
|
||||
for specific_file_url in file_urls:
|
||||
try:
|
||||
self.log.info(f"Attempting to fetch file from: {specific_file_url}")
|
||||
|
||||
self.log.info(f"File downloaded successfully: {local_file_path}")
|
||||
return local_file_path
|
||||
# Make the HTTP GET request to fetch the file URL
|
||||
response = s.get(specific_file_url, timeout=5, verify=False)
|
||||
|
||||
# Log response details
|
||||
self.log.info(f"HTTP Status Code: {response.status_code}")
|
||||
self.log.info(f"Response Headers: {response.headers}")
|
||||
|
||||
# Handle HTTP errors appropriately for each file
|
||||
if response.status_code == 404:
|
||||
self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
|
||||
errors[specific_file_url] = "File not found"
|
||||
continue
|
||||
elif response.status_code == 403:
|
||||
self.log.warning(f"Access forbidden for file: {specific_file_url}")
|
||||
errors[specific_file_url] = "Access forbidden"
|
||||
continue
|
||||
elif response.status_code != 200:
|
||||
self.log.error(
|
||||
f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
|
||||
errors[specific_file_url] = f"Unexpected status {response.status_code}"
|
||||
continue
|
||||
|
||||
# Save the file to /tmp/
|
||||
project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp"
|
||||
local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url))
|
||||
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
|
||||
|
||||
with open(local_file_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
self.log.info(f"File downloaded successfully: {local_file_path}")
|
||||
downloaded_files.append(local_file_path)
|
||||
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
self.log.warning(
|
||||
f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
|
||||
errors[specific_file_url] = "URL unreachable"
|
||||
except Exception as e:
|
||||
self.log.error(
|
||||
f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
|
||||
errors[specific_file_url] = "Unexpected error"
|
||||
|
||||
# Return the list of downloaded files and any errors
|
||||
return {
|
||||
"downloaded_files": downloaded_files,
|
||||
"errors": errors
|
||||
}
|
||||
|
||||
except AssertionError as e:
|
||||
self.log.warning(f"Failed to download file: {str(e)}")
|
||||
return False
|
||||
except (requests.ConnectionError, requests.Timeout) as e:
|
||||
self.log.warning(f"Failed to download file, the URL might be unreachable: {str(e)}")
|
||||
return False
|
||||
except Exception as e:
|
||||
self.log.error(f"An unexpected error occurred: {str(e)}")
|
||||
return False
|
||||
|
||||
self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
|
||||
return {"error": "Unexpected error"}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user