initial commit dev.r5portal upload system

src/components/archive_synchronizer.py

@@ -6,7 +6,7 @@ import threading
 import time
 import requests
 import traceback
-
+from bs4 import BeautifulSoup
 import requests
 from google.api_core.exceptions import Forbidden
 from google.cloud import storage
@@ -28,11 +28,11 @@ class ArchiveSynchronizer(Component):
     def config_changed(self):
         self.machine_id = self.config.machine_id
         if "--dev-portal" in sys.argv:
-            self.archive_endpoint =f"https://dev.r5portal.it/api/st-ten-save/"
+            self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
+            self.base_media_url = f"https://dev.r5portal.it/media/uploads/"
         else:
             self.archive_endpoint = self.config[self.name]["archive_endpoint"]
-        #self.status_endpoint = f"https://dev.r5portal.it/api/device-info-update?machine-id={self.machine_id_endpoint}&status={self.status}"
-        # self.images_path = self.config[self.name]["images_path"]
+            #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
         self._do_set_period({"period": float(self.config[self.name]["poll_time"])})
         self.hold_time = round(float(self.config[self.name]["hold_time"]) * 1000)
         self.gcs_client = storage.Client.from_service_account_json(self.config[self.name]["service_account_json"])
@@ -57,6 +57,7 @@ class ArchiveSynchronizer(Component):
         self.gcs_bucket = None
         if "--dev-portal" in sys.argv:
             self.update_machine_status()
+            self.remote_fetch(self.machine_id)
         super()._get()
 
     def update_machine_status(self):
@@ -153,3 +154,65 @@ class ArchiveSynchronizer(Component):
                 self.log.error(f"id: {record.id}: failed to store remotely:\n{traceback.format_exc()}")
             self.log.info(f"id: {record.id}: stored remotely")
         return True
+
+
+    def remote_fetch(self, machine_id, subfolder="warning_images"):
+        """
+        Fetch and download all files from the server's folder.
+
+        :param machine_id: The machine ID used to construct the folder path.
+        :param subfolder: The subfolder within the machine-specific path to fetch files from.
+        :return: A list of downloaded file paths, or False if an error occurs.
+        """
+        base_url = f"{self.base_media_url}/{subfolder}/{machine_id}"
+        responses = []
+        try:
+            if not self.simulate:
+                with requests.Session() as s:
+                    s.mount("", HTTPAdapter(max_retries=Retry(total=0)))
+
+                    # Get directory listing
+                    response = s.get(base_url, timeout=5, verify=False)
+
+                    if response.status_code == 403:
+                        self.log.warning(f"Access forbidden for directory listing: {base_url}")
+                        return False
+                    elif response.status_code != 200:
+                        raise AssertionError(f"bad status response for directory listing: {base_url}")
+
+                    # Parse the response content to get all file URLs.
+                    soup = BeautifulSoup(response.content, 'html.parser')
+                    file_urls = []
+                    for link in soup.find_all('a'):
+                        href = link.get('href')
+                        if href and re.search(r'\.(png|csv|svg)$', href,
+                                              re.IGNORECASE):  # Adjust file extensions as needed
+                            file_urls.append(f"{self.base_media_url}/{machine_id}/{href}")
+
+                    for file_url in file_urls:
+                        file_response = s.get(file_url, timeout=5, verify=False)
+
+                        if file_response.status_code == 403:
+                            self.log.warning(f"Access forbidden for file URL: {file_url}")
+                            continue
+                        elif file_response.status_code != 200:
+                            raise AssertionError(f"bad status response for {file_url}")
+
+                        local_file_path = os.path.join("downloaded_files", os.path.basename(file_url))
+                        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+                        with open(local_file_path, "wb") as f:
+                            f.write(file_response.content)
+
+                        responses.append((file_url, local_file_path))
+
+        except AssertionError as e:
+            self.log.warning(f"Failed to download files: {str(e)}")
+            return False
+        except (requests.ConnectionError, requests.Timeout) as e:
+            self.log.warning(f"Failed to download files, base_media_url might be unreachable: {str(e)}")
+            return False
+        except Exception as e:
+            self.log.error(f"An unexpected error occurred: {str(e)}")
+            return False
+        return responses