st-ten-1/src/components/archive_synchronizer.py

import json
import os
import re
import sys
import threading
import time
import requests
import traceback
from bs4 import BeautifulSoup
import requests
from google.api_core.exceptions import Forbidden
from google.cloud import storage
from lib.db import Archive, db
from PyQt5.QtCore import QThread
from requests.adapters import HTTPAdapter, Retry
from urllib3.exceptions import InsecureRequestWarning

from .component import Component
# Suppress insecure request warning
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)
class ArchiveSynchronizer(Component):
    machine_status = "offline"
    def __init__(self, config=None, name=None, period=1, lazy=True, paused=False, threaded=True):
        super().__init__(config=config, name=name, period=period, lazy=lazy, paused=paused, threaded=threaded)
        self.simulate = "--sim-archiver" in sys.argv
        self.status= ArchiveSynchronizer.machine_status

    def config_changed(self):
        self.machine_id = self.config.machine_id
        if "--dev-portal" in sys.argv:
            self.archive_endpoint = f"https://dev.r5portal.it/api/st-ten-save/"
            self.base_media_url = f"https://dev.r5portal.it"
        else:
            self.archive_endpoint = self.config[self.name]["archive_endpoint"]
            #self.browse_folder_endpoint = self.config[self.name]["browse_folder_endpoint"]
        self._do_set_period({"period": float(self.config[self.name]["poll_time"])})
        self.hold_time = round(float(self.config[self.name]["hold_time"]) * 1000)
        self.gcs_client = storage.Client.from_service_account_json(self.config[self.name]["service_account_json"])
        self.gcs_client._http.mount("", HTTPAdapter(max_retries=Retry(total=0)))  # this seems to be useless
        self.gcs_client._http.adapters.move_to_end("", last=False)  # this seems to be useless
        self.bucket_id = self.config[self.name]["bucket_id"]
        self.gcs_bucket = None

    @db.connection_context()
    def _get(self):
        for record in list(Archive.select().where((Archive.archived != True) | (Archive.uploaded != True))):  # using "is not True" breaks the query..  # list() forces the complete execution of the query unlocking the db unlike __enter__()
            if not self.simulate:
                if record.archived is not True:
                    record.archived = self.remote_archive(record) is True
                if record.uploaded is not True:
                    record.uploaded = self.remote_store(record) is True
            else:
                self.log.info("simulated archive synchronizer cycle")
            record.save()
            if self.hold_time > 0:
                QThread.msleep(self.hold_time)
        self.gcs_bucket = None
        if "--dev-portal" in sys.argv:
            self.update_machine_status()
            self.remote_fetch(self.machine_id)
        super()._get()

    def update_machine_status(self):
        self.status = ArchiveSynchronizer.machine_status
        #print(f"machine_status_global: {ArchiveSynchronizer.machine_status}")  # TESTING
        self.status_endpoint = f"https://dev.r5portal.it/api/device-info-update?machine-id={self.machine_id.upper()}&status={self.status}"

        if self.status not in ["working", "logged-in", "logged-out"]:
            status_dict = {"last_status": "offline"}
        else:
            status_dict = {"last_status": self.status}

        response = None
        try:
            if not self.simulate:
                with requests.Session() as s:
                    s.mount("", HTTPAdapter(max_retries=Retry(total=0)))  # this disables retries
                    response = requests.post(self.status_endpoint, json=status_dict, timeout=5, verify=False)
                    if response.status_code != 200:
                        raise AssertionError("bad status response")
        except AssertionError as e:
            self.log.warning(
                f"Status: {self.status}: failed to update machine status: {str(e)}: {response.status_code if response else 'no response'}: {response.content if response else 'no response'}"
            )
            return False
        except (requests.ConnectionError, requests.Timeout) as e:
            self.log.warning(
                f"Status: {self.status}: failed to update machine status, archive_endpoint might be unreachable: {str(e)}"
            )
            return False
        except Exception:
            self.log.error(
                f"Status: {self.status}: failed to update machine status:\n{traceback.format_exc()}:\n{response.status_code if response else 'no response'}: {response.content if response else 'no response'}"
            )
            return False

        self.log.info(f"Status: {self.status}: Machine Status Updated Successfully")
        return True
    def remote_archive(self, record):
        r = None
        try:
            if not self.simulate:
                with requests.Session() as s:
                    s.mount("", HTTPAdapter(max_retries=Retry(total=0)))  # this disables retries
                    r = requests.post(self.archive_endpoint, params={
                        "data": json.dumps(record.test_data),
                        "machine_id": self.machine_id,
                        "overridden": record.overridden,
                        "recipe": record.test_data.get("recipe", {}).get("name", None),
                        "result": "OK" if record.result else "KO",
                        "serial": record.id,
                        "time": record.time.isoformat(),
                        "user": record.user.username,
                    }, timeout=5, verify=False)
                    if r.status_code != 200:
                        raise AssertionError("bad status response")
        except AssertionError as e:
            self.log.warning(f"id: {record.id}: failed to archive remotely: {str(e)}: {r.status_code}: {r.content}")
            return False
        except (requests.ConnectionError, requests.Timeout) as e:
            self.log.warning(f"id: {record.id}: failed to archive remotely, archive_endpoint might be unreachable: {str(e)}")
            return False
        return True

    def remote_store(self, record):
        if "vision" not in record.test_data:
            return True
        try:
            self.gcs_bucket = self.gcs_client.get_bucket(self.bucket_id, timeout=(5, 30), retry=None)  # retry=None seems to be ignored
        except Exception:
            self.log.warning(f"id {record.id}: failed to connect to bucket {repr(self.bucket_id)}:\n{traceback.format_exc()}")
            self.gcs_bucket = None
        if self.gcs_bucket is None:
            return False
        for path in record.test_data["vision"]["0"]["files"]:
            dt = record.time
            # path_in = f"{self.images_path}/{dt.strftime('%Y')}/{dt.strftime('%m')}/{os.path.basename(path)}"
            path_in = path
            path_out = f"{self.machine_id}/{dt.strftime('%Y')}/{dt.strftime('%m')}/{os.path.basename(path)}"
            try:
                blob = self.gcs_bucket.blob(path_out)
                if not self.simulate:
                    blob.upload_from_filename(filename=path_in)
            except FileNotFoundError:
                self.log.error(f"id {record.id}: {path_in}: file not found.")
                return False
            except Forbidden as e:
                if re.match("^Object '.*?' is subject to bucket's retention policy and cannot be deleted, overwritten or archived until .*?$", e._response.json()["error"]["message"]) is not None:
                    self.log.info(f"id {record.id}: {path_in}: already stored.")
                else:
                    self.log.warning(f"id: {record.id}: failed to store remotely: {str(e)}: {e._response.json()}")
                    return False
            except Exception:
                self.log.error(f"id: {record.id}: failed to store remotely:\n{traceback.format_exc()}")
            self.log.info(f"id: {record.id}: stored remotely")
        return True

    def remote_fetch(self, machine_id, subfolder="warning_images"):
        """
        Fetch and download all files from the server's folder.

        :param machine_id: The machine ID used to construct the folder path.
        :param subfolder: The subfolder within the machine-specific path to fetch files from.
        :return: A list of downloaded file paths or False if an error occurs.
        """
        base_url = f"{self.base_media_url}/list-files/"
        responses = []
        try:
            if not self.simulate:
                with requests.Session() as s:
                    s.mount("", HTTPAdapter(max_retries=Retry(total=0)))

                    # Construct the request to the new API endpoint
                    params = {"machine_id": machine_id, "subfolder": subfolder}
                    self.log.info(f"Attempting to fetch file list from: {base_url} with params: {params}")

                    # Make the HTTP GET request to fetch the file URLs
                    response = s.get(base_url, params=params, timeout=5, verify=False)

                    # Log response details
                    self.log.info(f"HTTP Status Code: {response.status_code}")
                    self.log.info(f"Response Headers: {response.headers}")

                    # Handle HTTP errors appropriately
                    if response.status_code == 404:
                        self.log.warning(
                            f"Directory not found: {base_url}. Please check the URL path and ensure the server has the expected directory structure.")
                        return False
                    elif response.status_code == 403:
                        self.log.warning(f"Access forbidden for file list: {base_url}")
                        return False
                    elif response.status_code != 200:
                        self.log.error(f"Unexpected HTTP response status: {response.status_code} for URL: {base_url}")
                        return False

                    # Parse the JSON response to get the list of file URLs
                    file_data = response.json()
                    if "file_urls" not in file_data:
                        self.log.error(f"Invalid response structure: {file_data}")
                        return False

                    file_urls = file_data["file_urls"]

                    # Download each file
                    for file_url in file_urls:
                        file_response = s.get(file_url, timeout=5, verify=False)

                        if file_response.status_code == 403:
                            self.log.warning(f"Access forbidden for file URL: {file_url}")
                            continue
                        elif file_response.status_code != 200:
                            self.log.error(
                                f"Unexpected HTTP response status: {file_response.status_code} for file URL: {file_url}")
                            continue

                        local_file_path = os.path.join("downloaded_files", machine_id, os.path.basename(file_url)) #further adjust
                        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)

                        with open(local_file_path, "wb") as f:
                            f.write(file_response.content)

                        responses.append((file_url, local_file_path))

        except AssertionError as e:
            self.log.warning(f"Failed to download files: {str(e)}")
            return False
        except (requests.ConnectionError, requests.Timeout) as e:
            self.log.warning(f"Failed to download files, base_media_url might be unreachable: {str(e)}")
            return False
        except Exception as e:
            self.log.error(f"An unexpected error occurred: {str(e)}")
            return False
        return responses