dev.r5portal remote_fetch

ok

src/components/archive_synchronizer.py

@@ -29,7 +29,7 @@ class ArchiveSynchronizer(Component):
         self.machine_id = self.config.machine_id
         if "--dev-portal" in sys.argv:
             self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
-            self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
+            #self.url = f"https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
         else:
             self.archive_endpoint = self.config[self.name]["archive_endpoint"]
             #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
@@ -57,7 +57,7 @@ class ArchiveSynchronizer(Component):
         self.gcs_bucket = None
         if "--dev-portal" in sys.argv:
             self.update_machine_status()
-            self.remote_fetch(self.url)
+            self.remote_fetch(self.machine_status)
         super()._get()
 
     def update_machine_status(self):
@@ -155,59 +155,107 @@ class ArchiveSynchronizer(Component):
             self.log.info(f"id: {record.id}: stored remotely")
         return True
 
-    def remote_fetch(self, machine_status):
+    def remote_fetch(self, machine_status="logged-in"):
         """
-        Fetch and download a specific file from the server.
+        Fetch and download files from the server.
 
-        :param specific_file_url: The full URL of the file to download.
+        :param machine_status: Status of the machine (e.g., 'logged-in').
-        :return: The downloaded file path or False if an error occurs.
+        :return: A list of downloaded file paths or a dictionary with errors if any occur.
         """
+        # URLs endpoint from your Django server
+        urls_endpoint = "http://dev.r5portal.it/get-file-urls/"
+
         if machine_status == "logged-in":
             try:
-                specific_file_url = "https://dev.r5portal.it/media/uploads/warning_images/st-ten-10/st-ten-11.csv"
+                # Fetching the list of specific file URLs from the Django server
                 if not self.simulate:
                     with requests.Session() as s:
-                        self.log.info(f"Attempting to fetch file from: {specific_file_url}")
+                        self.log.info(f"Fetching list of file URLs from: {urls_endpoint}")
 
-                        # Make the HTTP GET request to fetch the specific file URL
+                        # Make the HTTP GET request to fetch the list of URLs
-                        response = s.get(specific_file_url, timeout=5, verify=False)
+                        response = s.get(urls_endpoint, timeout=5, verify=False)
 
                         # Log response details
                         self.log.info(f"HTTP Status Code: {response.status_code}")
                         self.log.info(f"Response Headers: {response.headers}")
-
+                        self.log.info(f"Response Content: {response.content}")
-                        # Handle HTTP errors appropriately
+                        # Handle HTTP errors for the list of URLs fetch
                         if response.status_code == 404:
-                            self.log.warning(
+                            self.log.warning(f"URLs endpoint not found: {urls_endpoint}. Please check the URL path.")
-                                f"File not found: {specific_file_url}. Please check the URL path and ensure the server has the expected file.")
+                            return {"error": "URLs endpoint not found"}
-                            return False
                         elif response.status_code == 403:
-                            self.log.warning(f"Access forbidden for file: {specific_file_url}")
+                            self.log.warning(f"Access forbidden for URLs endpoint: {urls_endpoint}")
-                            return False
+                            return {"error": "Access forbidden"}
                         elif response.status_code != 200:
                             self.log.error(
-                                f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
+                                f"Unexpected HTTP response status: {response.status_code} for URL: {urls_endpoint}")
-                            return False
+                            return {"error": "Unexpected HTTP response status"}
 
-                        # Save the file to /tmp/
+                        # Parse the list of URLs (assuming the response contains a JSON array of URLs)
-                        project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp"
+                        try:
-                        local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url))
+                            file_urls = response.json()
-                        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+                            self.log.info(f"Fetched file URLs: {file_urls}")
+                        except ValueError as e:
+                            self.log.error(f"Error parsing JSON response: {str(e)}")
+                            self.log.error(f"Response Content: {response.content}")
+                            return {"error": "Failed to parse JSON response"}
+                        downloaded_files = []
+                        errors = {}
 
-                        with open(local_file_path, "wb") as f:
+                        # Iterate over the list of URLs to download each file
-                            f.write(response.content)
+                        for specific_file_url in file_urls:
+                            try:
+                                self.log.info(f"Attempting to fetch file from: {specific_file_url}")
 
-                        self.log.info(f"File downloaded successfully: {local_file_path}")
+                                # Make the HTTP GET request to fetch the file URL
-                        return local_file_path
+                                response = s.get(specific_file_url, timeout=5, verify=False)
+
+                                # Log response details
+                                self.log.info(f"HTTP Status Code: {response.status_code}")
+                                self.log.info(f"Response Headers: {response.headers}")
+
+                                # Handle HTTP errors appropriately for each file
+                                if response.status_code == 404:
+                                    self.log.warning(f"File not found: {specific_file_url}. Please check the URL path.")
+                                    errors[specific_file_url] = "File not found"
+                                    continue
+                                elif response.status_code == 403:
+                                    self.log.warning(f"Access forbidden for file: {specific_file_url}")
+                                    errors[specific_file_url] = "Access forbidden"
+                                    continue
+                                elif response.status_code != 200:
+                                    self.log.error(
+                                        f"Unexpected HTTP response status: {response.status_code} for URL: {specific_file_url}")
+                                    errors[specific_file_url] = f"Unexpected status {response.status_code}"
+                                    continue
+
+                                # Save the file to /tmp/
+                                project_tmp_path = "/home/edo-neo/PycharmProjects/st-ten-1/tmp"
+                                local_file_path = os.path.join(project_tmp_path, os.path.basename(specific_file_url))
+                                os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+                                with open(local_file_path, "wb") as f:
+                                    f.write(response.content)
+
+                                self.log.info(f"File downloaded successfully: {local_file_path}")
+                                downloaded_files.append(local_file_path)
+
+                            except (requests.ConnectionError, requests.Timeout) as e:
+                                self.log.warning(
+                                    f"Failed to download file from {specific_file_url}, the URL might be unreachable: {str(e)}")
+                                errors[specific_file_url] = "URL unreachable"
+                            except Exception as e:
+                                self.log.error(
+                                    f"An unexpected error occurred while downloading {specific_file_url}: {str(e)}")
+                                errors[specific_file_url] = "Unexpected error"
+
+                        # Return the list of downloaded files and any errors
+                        return {
+                            "downloaded_files": downloaded_files,
+                            "errors": errors
+                        }
 
-            except AssertionError as e:
-                self.log.warning(f"Failed to download file: {str(e)}")
-                return False
-            except (requests.ConnectionError, requests.Timeout) as e:
-                self.log.warning(f"Failed to download file, the URL might be unreachable: {str(e)}")
-                return False
             except Exception as e:
-                self.log.error(f"An unexpected error occurred: {str(e)}")
+                self.log.error(f"An unexpected error occurred while fetching URLs: {str(e)}")
-                return False
+                return {"error": "Unexpected error"}
-