Lagt till hantering av host_url där LLM:er körs.

smartassist/src/startservices.py

@@ -69,43 +69,24 @@ def configure():
 
     if isinstance(updated_config.get('endpoints'), list):  # Extract info on endpoint, model, url, provider et cetera from list
         global_state.set_endpoints(endpoints=updated_config.get('endpoints'))  # Extract and set list of endpoints
-        logger.debug("endpoints = \n{}".format(json.dumps(global_state.get_endpoints(), indent=4)))
+        # logger.debug("endpoints = \n{}".format(json.dumps(global_state.get_endpoints(), indent=4)))
         fetch_models_from_endpoints(global_state.get_endpoints(), global_state)  # Call the new function
+        endpoints = global_state.get_endpoints()
+        for endpoint in endpoints: # Set default LLM for each endpoint
+            available_llms = global_state.get_list_of_available_llms(endpoint=endpoint)
+            llm = next(iter(available_llms),None) # First available LLM or None. Default for AUTODETECT and requests for non-existing LLMs 
+            logger.debug(f"url {endpoint['url']} = {available_llms}")
+            if endpoint["model"] in available_llms: # Check if specific LLM requested
+                llm = endpoint["model"] 
+            endpoint["default_llm"] = llm
             
  
-    # TODO: Remove this section when not needed anymore
+        global_state.set_host_url(next(iter(endpoints),None)) # Set initial host to the first item in endpoints (or None)
-    if isinstance(updated_config.get('ollama'), dict): # Look for 'ollama' key
+        global_state.set_llm(llm) # Set which server and llm to use
-        if isinstance(updated_config['ollama'].get('model'), str): # Look for 'model' key
-            model_to_use = updated_config['ollama'].get('model')
-            global_state.set_llm(model_to_use)
-            logger.debug("configure(): LLM is set to: %s",global_state.get_llm())
 
     return updated_config            
 
 
-# def start_frontend(config):
-#     parsed_url = urllib.parse.urlparse(config['frontend']['url'])
-#     hostname = parsed_url.netloc.split(':')[0]  # Split by ':' and take the first part, i.e., 'localhost', IP, or domain name
-#     port = parsed_url.port  # This is the server port
-
-#     # Use the socket module in Python to check whether a port is in use, 
-#     # which would indicate that a server is already running on that port. 
-#     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
-#         try:
-#             s.bind((hostname, port))
-#             logger.debug("No server is running on %s -— starting one.", parsed_url.netloc)
-#             # Start frontend (web server) as a separate process
-#             subprocess.Popen(["python", "-m", "http.server", str(port)])
-#         except socket.error as e:
-#             if e.errno == 48:
-#                 logger.debug("A server is already running on %s -— will use this.", parsed_url.netloc)
-#             else:
-#                 raise  # Unexpected error, re-raise it so we can see the traceback
-#         except Exception as e:
-#             logger.error("Failed to start frontend: %s", str(e)) # Corresponds to print(f"Failed to start frontend: {e}")
-            
-
-
 def start_backend(config):
     parsed_url = urllib.parse.urlparse(config['backend']['url'])
     # hostname = parsed_url.netloc.split(':')[0]  # Split by ':' and take the first part, i.e., 'localhost', IP, or domain name
@@ -120,6 +101,6 @@ def start_backend(config):
 
 if __name__ == '__main__':
     conf = configure() # Read config from file and set up config dict
-    logger.debug('conf dictionary set to \n{}'.format(json.dumps(conf, indent=4)))
+    # logger.debug('conf dictionary set to \n{}'.format(json.dumps(conf, indent=4)))
     # start_frontend(config=conf) # Not needed as we are using Flask for backend now
     start_backend(config=conf)

smartassist/src/utils.py

@@ -40,10 +40,8 @@ def fetch_models_from_endpoints(endpoints, global_state):
             if isinstance(models, dict) and 'error' in models:
                 logger.error('Error fetching models from backend: %s', models['error'])
             else:
-                endpoint["models"] = models.get("models", [])  # get the list of models directly
+                endpoint["models"] = models.get("models", [])  # Get the list of models directly
-                logger.debug("models = \n{}".format(json.dumps(models, indent=4)))
+                # logger.debug("models = \n{}".format(json.dumps(models, indent=4)))
-                if endpoint["model"] is not "AUTODETECT":  # Check if specified model is available
-                    logger.debug("Asking for specific model")
 
 
 class GlobalState:
@@ -67,6 +65,7 @@ class GlobalState:
             cls._instance.logger.addHandler(handler)
             cls._instance.logger.setLevel(getattr(logging, cls._instance.log_level))  # Initialize root logger level
             cls._instance.logger.info(" __new__(cls): Logger in GlobalState created: %s", cls._instance.logger)
+            cls._instance.host_url = None # Currently used LLM host
             cls._instance.llm = "phi3:mini" # Default LLM for queries. TODO: Check with ollama server that it actually exists
             # cls._instance.backend_api_ep = "http://localhost:5005/api/chat" # Default backend API endpoint
             # Try making things more aligned with the outline of the yaml file
@@ -105,6 +104,14 @@ class GlobalState:
         logger = logging.getLogger(module_name)
         return logger
     
+    def set_host_url(self, url="http://localhost:11434"):
+        """Set the host url to which LLM requests are sent"""
+        self.host = url
+
+    def get_host_url(self):
+        """Get the url for the currently used host for LLMs"""
+        return self.host_url
+    
     def set_llm(self, model_name="phi3:mini"):
         """Set LLM for queries"""
         self.llm = model_name
@@ -114,11 +121,11 @@ class GlobalState:
         return self.llm
     
     def set_backend(self, backend=None):
-        """Set backend that web clients connect to"""
+        """Set backend server that web clients connect to"""
         self.backend  = backend
 
     def get_backend(self):
-        """Getter for backend that web clients connect to"""
+        """Getter for backend server that web clients connect to"""
         return self.backend
     
     def get_backend_api_ep(self):
@@ -135,3 +142,10 @@ class GlobalState:
     def get_endpoints(self):
         """Return the list of endpoints"""
         return self.endpoints
+    
+    def get_list_of_available_llms(self, endpoint=None):
+        """Return a sorted list of LLMs available at endpoint"""
+        llm_list = None
+        if isinstance(endpoint["models"], list):
+            llm_list = sorted([list_item['name'] for list_item in endpoint["models"]], key=str.lower)
+        return llm_list