Eduardo_Scripniciuc/st-ten-1
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dev.r5portal upload system wip

Browse Source
This commit is contained in:
edo-neo 2024-10-22 12:11:56 +02:00
parent a54935fcad
commit 5464f9a9c4
1 changed files with 27 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/components/archive_synchronizer.py
View File

@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
self.machine_id = self.config.machine_id
if "--dev-portal" in sys.argv:
self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
self.base_media_url = f"https://dev.r5portal.it/media/uploads/"
self.base_media_url = f"https://dev.r5portal.it/media/uploads"
else:
self.archive_endpoint = self.config[self.name]["archive_endpoint"]
#self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
@ -155,7 +155,6 @@ class ArchiveSynchronizer(Component):
self.log.info(f"id: {record.id}: stored remotely")
return True
def remote_fetch(self, machine_id, subfolder="warning_images"):
"""
Fetch and download all files from the server's folder.
@ -171,24 +170,35 @@ class ArchiveSynchronizer(Component):
with requests.Session() as s:
s.mount("", HTTPAdapter(max_retries=Retry(total=0)))
# Get directory listing
self.log.info(f"Attempting to fetch directory listing from: {base_url}")
# Make the HTTP GET request to the directory URL
response = s.get(base_url, timeout=5, verify=False)
if response.status_code == 403:
# Log response details
self.log.info(f"HTTP Status Code: {response.status_code}")
self.log.info(f"Response Headers: {response.headers}")
# Handle HTTP errors appropriately
if response.status_code == 404:
self.log.warning(
f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.")
return False
elif response.status_code == 403:
self.log.warning(f"Access forbidden for directory listing: {base_url}")
return False
elif response.status_code != 200:
raise AssertionError(f"bad status response for directory listing: {base_url}")
self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}")
return False
# Parse the response content to get all file URLs.
soup = BeautifulSoup(response.content, 'html.parser')
# Manually parse the response content to get all file URLs
content = response.content.decode('iso-8859-1') # or appropriate charset
file_urls = []
for link in soup.find_all('a'):
href = link.get('href')
if href and re.search(r'\.(png|csv|svg)$', href,
re.IGNORECASE): # Adjust file extensions as needed
file_urls.append(f"{self.base_media_url}/{machine_id}/{href}")
matches = re.findall(r'href="([^"]+\.(?:png|csv|svg))"', content, re.IGNORECASE)
for href in matches:
file_urls.append(f"{base_url}{href}")
# Download each file
for file_url in file_urls:
file_response = s.get(file_url, timeout=5, verify=False)
@ -196,9 +206,11 @@ class ArchiveSynchronizer(Component):
self.log.warning(f"Access forbidden for file URL: {file_url}")
continue
elif file_response.status_code != 200:
raise AssertionError(f"bad status response for {file_url}")
self.log.error(
f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}")
continue
local_file_path = os.path.join("downloaded_files", os.path.basename(file_url))
local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url))
os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
with open(local_file_path, "wb") as f:
@ -215,4 +227,4 @@ class ArchiveSynchronizer(Component):
except Exception as e:
self.log.error(f"An unexpected error occurred: {str(e)}")
return False
return responses
return responses